@warmdrift/kgauto-compiler 2.0.0-alpha.61 → 2.0.0-alpha.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -12
- package/dist/brain-proxy.js +5 -1
- package/dist/brain-proxy.mjs +1 -1
- package/dist/{chunk-5TI6PNSK.mjs → chunk-3KQAID63.mjs} +5 -0
- package/dist/{chunk-IUWFML6Z.mjs → chunk-65ZMX5OT.mjs} +5 -1
- package/dist/{chunk-3RMLZCUK.mjs → chunk-ABFXMO73.mjs} +1 -1
- package/dist/{chunk-IIMPJZNH.mjs → chunk-YZRPNSSQ.mjs} +10 -0
- package/dist/dialect.d.mts +14 -2
- package/dist/dialect.d.ts +14 -2
- package/dist/dialect.js +5 -0
- package/dist/dialect.mjs +1 -1
- package/dist/glassbox/index.d.mts +3 -3
- package/dist/glassbox/index.d.ts +3 -3
- package/dist/glassbox-routes/format.d.mts +2 -2
- package/dist/glassbox-routes/format.d.ts +2 -2
- package/dist/glassbox-routes/index.d.mts +4 -4
- package/dist/glassbox-routes/index.d.ts +4 -4
- package/dist/glassbox-routes/index.js +10 -0
- package/dist/glassbox-routes/index.mjs +1 -1
- package/dist/glassbox-routes/react/index.d.mts +2 -2
- package/dist/glassbox-routes/react/index.d.ts +2 -2
- package/dist/index.d.mts +193 -6
- package/dist/index.d.ts +193 -6
- package/dist/index.js +681 -2
- package/dist/index.mjs +651 -4
- package/dist/{ir-DAKlQsVb.d.mts → ir-B2dRyJXO.d.mts} +74 -9
- package/dist/{ir-DmUuJsWc.d.ts → ir-ClU56aBc.d.ts} +74 -9
- package/dist/key-health.js +1 -1
- package/dist/key-health.mjs +1 -1
- package/dist/profiles.d.mts +1 -1
- package/dist/profiles.d.ts +1 -1
- package/dist/{types-B8X1Pyhx.d.ts → types-ByN3r0_7.d.ts} +1 -1
- package/dist/{types-DR62iPcO.d.ts → types-C-Wp7HpI.d.ts} +1 -1
- package/dist/{types-MRMBUqzY.d.mts → types-CSDweZsl.d.mts} +1 -1
- package/dist/{types-CssWqd0X.d.mts → types-axAX1Bg0.d.mts} +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -26,10 +26,11 @@ What this package delivers today, and nothing it doesn't:
|
|
|
26
26
|
|
|
27
27
|
**What it does not do (yet):** routing does not change itself from measured
|
|
28
28
|
outcomes. There is no automatic learning loop in the shipped package — model
|
|
29
|
-
promotion is evidence-*informed* but operator-*driven*.
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
29
|
+
promotion is evidence-*informed* but operator-*driven*. What DOES ship as of
|
|
30
|
+
alpha.62 is the eval spine (Stage 1 of that architecture): golden-set capture,
|
|
31
|
+
a pairwise judge, and a non-inferiority eval whose verdict lands as evidence
|
|
32
|
+
on the advisory surface — the eval runs, a human clicks promote. Automatic
|
|
33
|
+
promotion (Stages 2–3) returns to this README when it is code, not before.
|
|
33
34
|
|
|
34
35
|
Greenfield rewrite of `@warmdrift/kgauto` v1 (a behavioral patcher with
|
|
35
36
|
telemetry).
|
|
@@ -324,16 +325,33 @@ For staging without a dedicated brain, point consumers at the same Supabase
|
|
|
324
325
|
they already use — the schema is identical and migration to a dedicated brain
|
|
325
326
|
is a `pg_dump` away.
|
|
326
327
|
|
|
328
|
+
## Golden-set eval (alpha.62 — Stage 1, shipped)
|
|
329
|
+
|
|
330
|
+
The eval-driven quality-floor-guarded swap architecture (specced 2026-07-17
|
|
331
|
+
after an adversarial promise review) began shipping in alpha.62:
|
|
332
|
+
|
|
333
|
+
- **Capture:** set `KGAUTO_GOLDEN_CAPTURE=0.1` on a consumer and successful
|
|
334
|
+
`call()`s sample their full IR + served output into a per-surface replay
|
|
335
|
+
corpus. Explicit opt-in — golden rows carry raw prompts (documented posture
|
|
336
|
+
change; consent provenance on every row; reads strictly app-scoped). Or
|
|
337
|
+
hand-curate with `scripts/seed-golden-set.mjs`.
|
|
338
|
+
- **Eval:** `scripts/run-golden-eval.mjs --app X --archetype Y --candidate M`
|
|
339
|
+
replays the corpus on incumbent AND candidate (fresh outputs both sides),
|
|
340
|
+
applies mechanical hard floors (empty / schema / candidate-error / latency),
|
|
341
|
+
and runs an order-swapped pairwise judge (`judge` archetype, per-archetype
|
|
342
|
+
rubrics). Non-inferiority rule: wins-or-ties ≥ 80% + zero floor violations
|
|
343
|
+
→ `promote-ready`.
|
|
344
|
+
- **Evidence:** every run lands as brain rows (`kgauto_golden_eval_runs` +
|
|
345
|
+
per-case `probe_outcomes` `golden-replay` rows) and a `golden-eval-verdict`
|
|
346
|
+
advisory on the consumer's advisory surface.
|
|
347
|
+
|
|
327
348
|
## What's next
|
|
328
349
|
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
another on a surface only by demonstrating same-or-better quality on that
|
|
335
|
-
surface's real workload. Each stage lands in this README **when it ships**,
|
|
336
|
-
with its gate evidence — the promise follows the code.
|
|
350
|
+
Stages 2–3 of the trust ladder (auto-promote downswaps behind a 7-day
|
|
351
|
+
rollback guard, then upswaps) and trigger wiring (model-release-watch /
|
|
352
|
+
price-truth-watch / live-telemetry drift → auto-eval), then the same harness
|
|
353
|
+
pointed at compile variants (Tier D). Each stage lands in this README **when
|
|
354
|
+
it ships**, with its gate evidence — the promise follows the code.
|
|
337
355
|
|
|
338
356
|
## Why this exists
|
|
339
357
|
|
package/dist/brain-proxy.js
CHANGED
|
@@ -27,7 +27,11 @@ var SEGMENT_TO_TABLE = {
|
|
|
27
27
|
outcomes: "compile_outcomes",
|
|
28
28
|
compile_outcome_advisories: "compile_outcome_advisories",
|
|
29
29
|
compile_outcome_quality: "compile_outcome_quality",
|
|
30
|
-
probe_outcomes: "probe_outcomes"
|
|
30
|
+
probe_outcomes: "probe_outcomes",
|
|
31
|
+
// alpha.62 (eval spine): golden-set capture. Only receives rows when the
|
|
32
|
+
// consumer has explicitly opted in via KGAUTO_GOLDEN_CAPTURE — the segment
|
|
33
|
+
// existing here does not by itself store anything.
|
|
34
|
+
golden_irs: "kgauto_golden_irs"
|
|
31
35
|
};
|
|
32
36
|
var KNOWN_SEGMENTS = Object.keys(SEGMENT_TO_TABLE);
|
|
33
37
|
var JSON_HEADERS = { "Content-Type": "application/json" };
|
package/dist/brain-proxy.mjs
CHANGED
|
@@ -45,6 +45,11 @@ var INTENT_ARCHETYPES = {
|
|
|
45
45
|
name: "transform",
|
|
46
46
|
description: "Change format or style while preserving content",
|
|
47
47
|
examples: ["markdown \u2192 html", "formal \u2192 casual", "translate"]
|
|
48
|
+
},
|
|
49
|
+
judge: {
|
|
50
|
+
name: "judge",
|
|
51
|
+
description: "Pairwise comparison of two candidate outputs for the same input",
|
|
52
|
+
examples: ["golden-set eval A/B verdict", "model-swap non-inferiority check", "response bake-off"]
|
|
48
53
|
}
|
|
49
54
|
};
|
|
50
55
|
var ALL_ARCHETYPES = Object.keys(INTENT_ARCHETYPES);
|
|
@@ -3,7 +3,11 @@ var SEGMENT_TO_TABLE = {
|
|
|
3
3
|
outcomes: "compile_outcomes",
|
|
4
4
|
compile_outcome_advisories: "compile_outcome_advisories",
|
|
5
5
|
compile_outcome_quality: "compile_outcome_quality",
|
|
6
|
-
probe_outcomes: "probe_outcomes"
|
|
6
|
+
probe_outcomes: "probe_outcomes",
|
|
7
|
+
// alpha.62 (eval spine): golden-set capture. Only receives rows when the
|
|
8
|
+
// consumer has explicitly opted in via KGAUTO_GOLDEN_CAPTURE — the segment
|
|
9
|
+
// existing here does not by itself store anything.
|
|
10
|
+
golden_irs: "kgauto_golden_irs"
|
|
7
11
|
};
|
|
8
12
|
var KNOWN_SEGMENTS = Object.keys(SEGMENT_TO_TABLE);
|
|
9
13
|
var JSON_HEADERS = { "Content-Type": "application/json" };
|
|
@@ -333,6 +333,16 @@ var STARTER_CHAINS_GROUNDED = {
|
|
|
333
333
|
{ id: "gemini-2.5-pro", grounding: "judgment", reason: "Cross-provider anchor in similar quality bracket" },
|
|
334
334
|
{ id: "gpt-5.5", grounding: "judgment", reason: "alpha.16: third-provider frontier-tier floor (archetypePerf=9)" }
|
|
335
335
|
],
|
|
336
|
+
// alpha.62 (eval spine) — pairwise output comparison. Same reasoning-floor
|
|
337
|
+
// posture as critique (a verdict from a weak judge is worse than no
|
|
338
|
+
// verdict); tier 0 anchored on the CURRENT opus (4-8, s64 roster). Strict
|
|
339
|
+
// JSON output, so every tier declares native structured output.
|
|
340
|
+
judge: [
|
|
341
|
+
{ id: "claude-opus-4-8", grounding: "judgment", reason: "Highest reasoning bar for pairwise verdicts \u2014 current opus (s64 roster)" },
|
|
342
|
+
{ id: "claude-sonnet-4-6", grounding: "judgment", reason: "Same-provider walk-down on 429" },
|
|
343
|
+
{ id: "gemini-2.5-pro", grounding: "judgment", reason: "Cross-provider anchor in similar quality bracket" },
|
|
344
|
+
{ id: "gpt-5.5", grounding: "judgment", reason: "Third-provider frontier-tier floor" }
|
|
345
|
+
],
|
|
336
346
|
// Reasoning matters — Sonnet primary; walk UP to Opus on 429.
|
|
337
347
|
plan: [
|
|
338
348
|
{ id: "claude-sonnet-4-6", grounding: "judgment", reason: "Reasoning + cost balance \u2014 engineer pick" },
|
package/dist/dialect.d.mts
CHANGED
|
@@ -12,8 +12,15 @@
|
|
|
12
12
|
* The brain learns by `(archetype, model, shape)` tuples. Apps that declare
|
|
13
13
|
* the same tuple inherit each other's mutations.
|
|
14
14
|
*
|
|
15
|
-
* v1 is intentionally small (
|
|
16
|
-
* arrive
|
|
15
|
+
* v1 is intentionally small (10 archetypes, 5 shape dimensions). New entries
|
|
16
|
+
* arrive from real usage signals — never speculatively.
|
|
17
|
+
*
|
|
18
|
+
* Versioning note (alpha.62): `judge` was added as an ADDITIVE 10th entry
|
|
19
|
+
* under dialect-v1 rather than bumping to v2. Bumping the version string
|
|
20
|
+
* would fragment every existing learning key (`v1::…` → `v2::…`) for a
|
|
21
|
+
* purely additive vocabulary change; existing keys are untouched and judge
|
|
22
|
+
* keys are new, so no collision is possible. A v2 bump is reserved for
|
|
23
|
+
* changes that alter the MEANING of existing entries or the shape hash.
|
|
17
24
|
*/
|
|
18
25
|
declare const DIALECT_VERSION: "v1";
|
|
19
26
|
declare const INTENT_ARCHETYPES: {
|
|
@@ -62,6 +69,11 @@ declare const INTENT_ARCHETYPES: {
|
|
|
62
69
|
readonly description: "Change format or style while preserving content";
|
|
63
70
|
readonly examples: ["markdown → html", "formal → casual", "translate"];
|
|
64
71
|
};
|
|
72
|
+
readonly judge: {
|
|
73
|
+
readonly name: "judge";
|
|
74
|
+
readonly description: "Pairwise comparison of two candidate outputs for the same input";
|
|
75
|
+
readonly examples: ["golden-set eval A/B verdict", "model-swap non-inferiority check", "response bake-off"];
|
|
76
|
+
};
|
|
65
77
|
};
|
|
66
78
|
type IntentArchetypeName = keyof typeof INTENT_ARCHETYPES;
|
|
67
79
|
declare const ALL_ARCHETYPES: IntentArchetypeName[];
|
package/dist/dialect.d.ts
CHANGED
|
@@ -12,8 +12,15 @@
|
|
|
12
12
|
* The brain learns by `(archetype, model, shape)` tuples. Apps that declare
|
|
13
13
|
* the same tuple inherit each other's mutations.
|
|
14
14
|
*
|
|
15
|
-
* v1 is intentionally small (
|
|
16
|
-
* arrive
|
|
15
|
+
* v1 is intentionally small (10 archetypes, 5 shape dimensions). New entries
|
|
16
|
+
* arrive from real usage signals — never speculatively.
|
|
17
|
+
*
|
|
18
|
+
* Versioning note (alpha.62): `judge` was added as an ADDITIVE 10th entry
|
|
19
|
+
* under dialect-v1 rather than bumping to v2. Bumping the version string
|
|
20
|
+
* would fragment every existing learning key (`v1::…` → `v2::…`) for a
|
|
21
|
+
* purely additive vocabulary change; existing keys are untouched and judge
|
|
22
|
+
* keys are new, so no collision is possible. A v2 bump is reserved for
|
|
23
|
+
* changes that alter the MEANING of existing entries or the shape hash.
|
|
17
24
|
*/
|
|
18
25
|
declare const DIALECT_VERSION: "v1";
|
|
19
26
|
declare const INTENT_ARCHETYPES: {
|
|
@@ -62,6 +69,11 @@ declare const INTENT_ARCHETYPES: {
|
|
|
62
69
|
readonly description: "Change format or style while preserving content";
|
|
63
70
|
readonly examples: ["markdown → html", "formal → casual", "translate"];
|
|
64
71
|
};
|
|
72
|
+
readonly judge: {
|
|
73
|
+
readonly name: "judge";
|
|
74
|
+
readonly description: "Pairwise comparison of two candidate outputs for the same input";
|
|
75
|
+
readonly examples: ["golden-set eval A/B verdict", "model-swap non-inferiority check", "response bake-off"];
|
|
76
|
+
};
|
|
65
77
|
};
|
|
66
78
|
type IntentArchetypeName = keyof typeof INTENT_ARCHETYPES;
|
|
67
79
|
declare const ALL_ARCHETYPES: IntentArchetypeName[];
|
package/dist/dialect.js
CHANGED
|
@@ -77,6 +77,11 @@ var INTENT_ARCHETYPES = {
|
|
|
77
77
|
name: "transform",
|
|
78
78
|
description: "Change format or style while preserving content",
|
|
79
79
|
examples: ["markdown \u2192 html", "formal \u2192 casual", "translate"]
|
|
80
|
+
},
|
|
81
|
+
judge: {
|
|
82
|
+
name: "judge",
|
|
83
|
+
description: "Pairwise comparison of two candidate outputs for the same input",
|
|
84
|
+
examples: ["golden-set eval A/B verdict", "model-swap non-inferiority check", "response bake-off"]
|
|
80
85
|
}
|
|
81
86
|
};
|
|
82
87
|
var ALL_ARCHETYPES = Object.keys(INTENT_ARCHETYPES);
|
package/dist/dialect.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { G as GlassboxEvent } from '../types-
|
|
2
|
-
export { A as AdvisoryFiredData, C as CompileDoneData, a as CompileStartData, E as ExecuteAttemptData, b as ExecuteSuccessData, F as FallbackWalkedData, c as GLASSBOX_STREAM_TTL_MS, d as GlassboxEventKind, e as GlassboxPubSub } from '../types-
|
|
3
|
-
import '../ir-
|
|
1
|
+
import { G as GlassboxEvent } from '../types-axAX1Bg0.mjs';
|
|
2
|
+
export { A as AdvisoryFiredData, C as CompileDoneData, a as CompileStartData, E as ExecuteAttemptData, b as ExecuteSuccessData, F as FallbackWalkedData, c as GLASSBOX_STREAM_TTL_MS, d as GlassboxEventKind, e as GlassboxPubSub } from '../types-axAX1Bg0.mjs';
|
|
3
|
+
import '../ir-B2dRyJXO.mjs';
|
|
4
4
|
import '../dialect.mjs';
|
|
5
5
|
|
|
6
6
|
/**
|
package/dist/glassbox/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { G as GlassboxEvent } from '../types-
|
|
2
|
-
export { A as AdvisoryFiredData, C as CompileDoneData, a as CompileStartData, E as ExecuteAttemptData, b as ExecuteSuccessData, F as FallbackWalkedData, c as GLASSBOX_STREAM_TTL_MS, d as GlassboxEventKind, e as GlassboxPubSub } from '../types-
|
|
3
|
-
import '../ir-
|
|
1
|
+
import { G as GlassboxEvent } from '../types-C-Wp7HpI.js';
|
|
2
|
+
export { A as AdvisoryFiredData, C as CompileDoneData, a as CompileStartData, E as ExecuteAttemptData, b as ExecuteSuccessData, F as FallbackWalkedData, c as GLASSBOX_STREAM_TTL_MS, d as GlassboxEventKind, e as GlassboxPubSub } from '../types-C-Wp7HpI.js';
|
|
3
|
+
import '../ir-ClU56aBc.js';
|
|
4
4
|
import '../dialect.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { G as GlassboxEvent } from '../types-
|
|
2
|
-
import { a as TraceDetail, b as TraceSummary, c as TraceCounterfactual } from '../types-
|
|
3
|
-
export { A as AdvisoryRecord, T as TraceHealth, d as TraceSectionRewrite } from '../types-
|
|
4
|
-
import '../ir-
|
|
1
|
+
import { G as GlassboxEvent } from '../types-axAX1Bg0.mjs';
|
|
2
|
+
import { a as TraceDetail, b as TraceSummary, c as TraceCounterfactual } from '../types-CSDweZsl.mjs';
|
|
3
|
+
export { A as AdvisoryRecord, T as TraceHealth, d as TraceSectionRewrite } from '../types-CSDweZsl.mjs';
|
|
4
|
+
import '../ir-B2dRyJXO.mjs';
|
|
5
5
|
import '../dialect.mjs';
|
|
6
6
|
|
|
7
7
|
/**
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { G as GlassboxEvent } from '../types-
|
|
2
|
-
import { a as TraceDetail, b as TraceSummary, c as TraceCounterfactual } from '../types-
|
|
3
|
-
export { A as AdvisoryRecord, T as TraceHealth, d as TraceSectionRewrite } from '../types-
|
|
4
|
-
import '../ir-
|
|
1
|
+
import { G as GlassboxEvent } from '../types-C-Wp7HpI.js';
|
|
2
|
+
import { a as TraceDetail, b as TraceSummary, c as TraceCounterfactual } from '../types-ByN3r0_7.js';
|
|
3
|
+
export { A as AdvisoryRecord, T as TraceHealth, d as TraceSectionRewrite } from '../types-ByN3r0_7.js';
|
|
4
|
+
import '../ir-ClU56aBc.js';
|
|
5
5
|
import '../dialect.js';
|
|
6
6
|
|
|
7
7
|
/**
|
|
@@ -1650,6 +1650,16 @@ var STARTER_CHAINS_GROUNDED = {
|
|
|
1650
1650
|
{ id: "gemini-2.5-pro", grounding: "judgment", reason: "Cross-provider anchor in similar quality bracket" },
|
|
1651
1651
|
{ id: "gpt-5.5", grounding: "judgment", reason: "alpha.16: third-provider frontier-tier floor (archetypePerf=9)" }
|
|
1652
1652
|
],
|
|
1653
|
+
// alpha.62 (eval spine) — pairwise output comparison. Same reasoning-floor
|
|
1654
|
+
// posture as critique (a verdict from a weak judge is worse than no
|
|
1655
|
+
// verdict); tier 0 anchored on the CURRENT opus (4-8, s64 roster). Strict
|
|
1656
|
+
// JSON output, so every tier declares native structured output.
|
|
1657
|
+
judge: [
|
|
1658
|
+
{ id: "claude-opus-4-8", grounding: "judgment", reason: "Highest reasoning bar for pairwise verdicts \u2014 current opus (s64 roster)" },
|
|
1659
|
+
{ id: "claude-sonnet-4-6", grounding: "judgment", reason: "Same-provider walk-down on 429" },
|
|
1660
|
+
{ id: "gemini-2.5-pro", grounding: "judgment", reason: "Cross-provider anchor in similar quality bracket" },
|
|
1661
|
+
{ id: "gpt-5.5", grounding: "judgment", reason: "Third-provider frontier-tier floor" }
|
|
1662
|
+
],
|
|
1653
1663
|
// Reasoning matters — Sonnet primary; walk UP to Opus on 429.
|
|
1654
1664
|
plan: [
|
|
1655
1665
|
{ id: "claude-sonnet-4-6", grounding: "judgment", reason: "Reasoning + cost balance \u2014 engineer pick" },
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as react_jsx_runtime from 'react/jsx-runtime';
|
|
2
|
-
import { a as TraceDetail } from '../../types-
|
|
3
|
-
import '../../ir-
|
|
2
|
+
import { a as TraceDetail } from '../../types-CSDweZsl.mjs';
|
|
3
|
+
import '../../ir-B2dRyJXO.mjs';
|
|
4
4
|
import '../../dialect.mjs';
|
|
5
5
|
|
|
6
6
|
/**
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as react_jsx_runtime from 'react/jsx-runtime';
|
|
2
|
-
import { a as TraceDetail } from '../../types-
|
|
3
|
-
import '../../ir-
|
|
2
|
+
import { a as TraceDetail } from '../../types-ByN3r0_7.js';
|
|
3
|
+
import '../../ir-ClU56aBc.js';
|
|
4
4
|
import '../../dialect.js';
|
|
5
5
|
|
|
6
6
|
/**
|
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { C as CompilePolicy, N as NormalizedResponse, A as ApiKeys, P as ProviderOverrides, a as CompiledRequest, b as PromptIR, c as CallOptions, d as CallResult, S as SystemModelMessage, e as CompileResult, B as BestPracticeAdvisory, f as SectionRewrite, R as RecordInput, g as RecordOutcomeInput, O as OutcomeResult, h as OracleScore, i as Adapter, j as PerAxisMetrics, k as Provider, l as ChainEntry, G as Grounding } from './ir-
|
|
2
|
-
export { m as CallAttempt, n as CallError, o as ChainModelEntry, p as ChainWithGrounding, q as Constraints, E as EffortLevel, F as FallbackReason, H as HistoryCachePolicy, I as IntentDeclaration, M as Message,
|
|
1
|
+
import { C as CompilePolicy, N as NormalizedResponse, A as ApiKeys, P as ProviderOverrides, a as CompiledRequest, b as PromptIR, c as CallOptions, d as CallResult, S as SystemModelMessage, e as CompileResult, B as BestPracticeAdvisory, f as SectionRewrite, R as RecordInput, g as RecordOutcomeInput, O as OutcomeResult, h as OracleScore, i as Adapter, j as PerAxisMetrics, k as Provider, l as ChainEntry, G as Grounding } from './ir-B2dRyJXO.mjs';
|
|
2
|
+
export { m as CallAttempt, n as CallError, o as ChainModelEntry, p as ChainWithGrounding, q as Constraints, E as EffortLevel, F as FallbackReason, r as GoldenCaptureOptions, H as HistoryCachePolicy, I as IntentDeclaration, M as Message, s as MutationApplied, t as NormalizedTokens, u as OutcomeKind, v as PerAxisMetricsByModel, w as PromptSection, x as SectionKind, y as ShadowProbeConfig, T as ToolCall, z as ToolDefinition, D as captureGoldenIr, J as parseGoldenCaptureRate, K as resolveGoldenCaptureRate, L as shouldCaptureGolden } from './ir-B2dRyJXO.mjs';
|
|
3
3
|
import { ModelProfile, ArchetypeConvention } from './profiles.mjs';
|
|
4
4
|
export { ALIASES, CacheStrategy, CliffRule, LATENCY_TIER_MS, LatencyTier, LoweringSpec, RecoveryRule, StructuredOutputCapability, SystemPromptMode, allProfiles, getProfile, latencyTierOf, profilesByProvider, tryGetProfile } from './profiles.mjs';
|
|
5
5
|
export { BrainForwardConfig, BrainForwardRoutes, createBrainForwardRoutes } from './brain-proxy.mjs';
|
|
@@ -462,7 +462,7 @@ interface BrainHealthSnapshot {
|
|
|
462
462
|
}
|
|
463
463
|
interface BrainDeadLetterEntry {
|
|
464
464
|
/** Brain route segment the payload was bound for. */
|
|
465
|
-
route: 'outcomes' | 'compile_outcome_quality' | 'probe_outcomes' | 'compile_outcome_advisories';
|
|
465
|
+
route: 'outcomes' | 'compile_outcome_quality' | 'probe_outcomes' | 'compile_outcome_advisories' | 'golden_irs';
|
|
466
466
|
/**
|
|
467
467
|
* Endpoint the payload was originally bound for. Flush replays ONLY entries
|
|
468
468
|
* matching the currently-configured endpoint — a reconfigure must never
|
|
@@ -686,6 +686,193 @@ declare function buildShadowProbeRow(input: ShadowProbeRecordInput): {
|
|
|
686
686
|
* `/probe_outcomes` the same way they forward `/outcomes`.
|
|
687
687
|
*/
|
|
688
688
|
declare function recordShadowProbe(input: ShadowProbeRecordInput): Promise<void>;
|
|
689
|
+
/**
|
|
690
|
+
* Golden-IR capture input. UNLIKE every other brain write, this row carries the
|
|
691
|
+
* RAW full PromptIR — an explicit, documented posture change from the s51
|
|
692
|
+
* "brain holds no raw consumer prompts" rule, gated on consumer opt-in
|
|
693
|
+
* (`KGAUTO_GOLDEN_CAPTURE` env / `CallOptions.goldenCapture`). The `consent`
|
|
694
|
+
* field records the provenance of that opt-in at capture time.
|
|
695
|
+
*/
|
|
696
|
+
interface GoldenIrRecordInput {
|
|
697
|
+
appId: string;
|
|
698
|
+
archetype: string;
|
|
699
|
+
/** hashShape() of the call at compile time (peek before record() pops). */
|
|
700
|
+
shapeKey?: string;
|
|
701
|
+
/** The full PromptIR, verbatim. Plain JSON by construction (ir.ts). */
|
|
702
|
+
ir: unknown;
|
|
703
|
+
incumbentModel: string;
|
|
704
|
+
incumbentOutput?: string;
|
|
705
|
+
incumbentLatencyMs?: number;
|
|
706
|
+
incumbentTokensIn?: number;
|
|
707
|
+
incumbentTokensOut?: number;
|
|
708
|
+
/** 'sampled' (capture flag fired on live traffic) | 'curated' (operator). */
|
|
709
|
+
source: 'sampled' | 'curated';
|
|
710
|
+
/** Consent provenance, e.g. 'KGAUTO_GOLDEN_CAPTURE=0.1 (consumer env opt-in)'. */
|
|
711
|
+
consent: string;
|
|
712
|
+
/** Optional link back to the compile handle of the captured call. */
|
|
713
|
+
outcomeHandle?: string;
|
|
714
|
+
}
|
|
715
|
+
/** Pure builder for the `kgauto_golden_irs` row (snake_case wire shape). */
|
|
716
|
+
declare function buildGoldenIrRow(input: GoldenIrRecordInput): {
|
|
717
|
+
app_id: string;
|
|
718
|
+
intent_archetype: string;
|
|
719
|
+
shape_key: string | null;
|
|
720
|
+
ir: unknown;
|
|
721
|
+
incumbent_model: string;
|
|
722
|
+
incumbent_output: string | null;
|
|
723
|
+
incumbent_latency_ms: number | null;
|
|
724
|
+
incumbent_tokens_in: number | null;
|
|
725
|
+
incumbent_tokens_out: number | null;
|
|
726
|
+
source: 'sampled' | 'curated';
|
|
727
|
+
consent: string;
|
|
728
|
+
outcome_handle: string | null;
|
|
729
|
+
};
|
|
730
|
+
/**
|
|
731
|
+
* Persist one golden IR to `kgauto_golden_irs`. Same G1 write-leg discipline as
|
|
732
|
+
* every other brain route (redirect check, ledger, dead-letter); fire-and-forget
|
|
733
|
+
* by default, honors `BrainConfig.sync`. Never throws.
|
|
734
|
+
*/
|
|
735
|
+
declare function recordGoldenIr(input: GoldenIrRecordInput): Promise<void>;
|
|
736
|
+
|
|
737
|
+
/**
|
|
738
|
+
* Golden-set eval engine (alpha.62, eval spine — design brief 2026-07-17).
|
|
739
|
+
*
|
|
740
|
+
* Implements the quality-floor-guarded model-swap contract:
|
|
741
|
+
*
|
|
742
|
+
* > A model replaces another on a surface only by demonstrating
|
|
743
|
+
* > same-or-better quality on that surface's real workload — never by
|
|
744
|
+
* > price, recency, or judgment alone.
|
|
745
|
+
*
|
|
746
|
+
* One eval run: load the active golden set for (app, archetype), replay every
|
|
747
|
+
* IR on BOTH the incumbent and the candidate (fresh outputs on both sides —
|
|
748
|
+
* fair comparison, no stale stored output), apply mechanical hard floors,
|
|
749
|
+
* then a pairwise LLM judge (order-swapped to cancel position bias), and
|
|
750
|
+
* aggregate under the non-inferiority rule:
|
|
751
|
+
*
|
|
752
|
+
* promote-ready ⇔ wins-or-ties ≥ threshold (default 80%)
|
|
753
|
+
* AND zero hard-floor violations
|
|
754
|
+
* AND candidate p50 latency ≤ ratio × incumbent p50
|
|
755
|
+
*
|
|
756
|
+
* Evidence lands as brain rows: one `kgauto_golden_eval_runs` aggregate + one
|
|
757
|
+
* `probe_outcomes` row per case (`replay_source='golden-replay'`, full
|
|
758
|
+
* fidelity by construction) + an evidence-attached advisory on the consumer's
|
|
759
|
+
* advisory surface (trust-ladder Stage 1: eval runs, human clicks promote).
|
|
760
|
+
*
|
|
761
|
+
* Judging is PAIRWISE, not absolute — it operationalizes the quality-floor
|
|
762
|
+
* Common Pattern directly ("same or better, never less"). Each pair is judged
|
|
763
|
+
* twice with A/B order swapped; disagreement between the two orders is scored
|
|
764
|
+
* a tie (the honest reading of positional noise).
|
|
765
|
+
*
|
|
766
|
+
* This module is operator-grade: it talks to the brain via direct PostgREST
|
|
767
|
+
* with a service key (the watchers' convention), never through consumer
|
|
768
|
+
* proxies, and it deliberately does NOT call record() — eval traffic must not
|
|
769
|
+
* pollute compile_outcomes.
|
|
770
|
+
*/
|
|
771
|
+
|
|
772
|
+
/**
|
|
773
|
+
* What "better" means per archetype. Fed verbatim into the pairwise judge
|
|
774
|
+
* prompt. Kept deliberately short — the judge sees the full request, so the
|
|
775
|
+
* rubric only needs to name the failure modes that matter for the archetype.
|
|
776
|
+
*/
|
|
777
|
+
declare const JUDGE_RUBRICS: Record<string, string>;
|
|
778
|
+
declare function rubricFor(archetype: string): string;
|
|
779
|
+
/** Render the golden IR's request for the judge (system sections + user turn). */
|
|
780
|
+
declare function renderIrForJudge(ir: PromptIR): string;
|
|
781
|
+
declare function buildPairwiseJudgePrompt(args: {
|
|
782
|
+
archetype: string;
|
|
783
|
+
renderedInput: string;
|
|
784
|
+
outputA: string;
|
|
785
|
+
outputB: string;
|
|
786
|
+
}): string;
|
|
787
|
+
type PairwiseWinner = 'A' | 'B' | 'tie';
|
|
788
|
+
/** Parse the judge's JSON verdict. Returns undefined when unparseable. */
|
|
789
|
+
declare function parseJudgeVerdict(raw: string): {
|
|
790
|
+
winner: PairwiseWinner;
|
|
791
|
+
rationale?: string;
|
|
792
|
+
} | undefined;
|
|
793
|
+
/**
|
|
794
|
+
* Combine the two order-swapped verdicts into a case verdict. Both verdicts
|
|
795
|
+
* are expressed in incumbent/candidate terms BEFORE this call (the caller
|
|
796
|
+
* un-swaps run 2). Agreement → that verdict; disagreement → 'tied' (positional
|
|
797
|
+
* noise reads as a tie, never as a win).
|
|
798
|
+
*/
|
|
799
|
+
declare function combineOrderSwappedVerdicts(run1: 'candidate-better' | 'current-better' | 'tied', run2: 'candidate-better' | 'current-better' | 'tied'): 'candidate-better' | 'current-better' | 'tied';
|
|
800
|
+
interface GoldenEvalOptions {
|
|
801
|
+
appId: string;
|
|
802
|
+
archetype: string;
|
|
803
|
+
candidateModel: string;
|
|
804
|
+
/**
|
|
805
|
+
* Incumbent to compare against. Default: the most frequent incumbent_model
|
|
806
|
+
* across the loaded golden set (the surface's de-facto leader).
|
|
807
|
+
*/
|
|
808
|
+
incumbentModel?: string;
|
|
809
|
+
/** Judge model id. Default 'claude-opus-4-8' (judge chain tier 0). */
|
|
810
|
+
judgeModel?: string;
|
|
811
|
+
/** Max golden cases to load (newest first). Default 50. */
|
|
812
|
+
limit?: number;
|
|
813
|
+
/** Non-inferiority threshold on wins-or-ties. Default 0.8. */
|
|
814
|
+
winOrTieThreshold?: number;
|
|
815
|
+
/** Latency hard floor: candidate p50 ≤ ratio × incumbent p50. Default 3. */
|
|
816
|
+
latencyFloorRatio?: number;
|
|
817
|
+
/** Minimum judged cases for a non-inconclusive verdict. Default 5. */
|
|
818
|
+
minJudgeableCases?: number;
|
|
819
|
+
/** What fired the eval. Default 'manual' (Stage 1). */
|
|
820
|
+
triggerSource?: 'manual' | 'supply' | 'price' | 'demand';
|
|
821
|
+
/** Brain access — direct PostgREST (operator convention). */
|
|
822
|
+
supabaseUrl: string;
|
|
823
|
+
serviceKey: string;
|
|
824
|
+
/** Provider keys for the replay + judge calls. Default: process.env. */
|
|
825
|
+
apiKeys?: ApiKeys;
|
|
826
|
+
fetchImpl?: typeof fetch;
|
|
827
|
+
/** Replay + judge but write NOTHING to the brain. */
|
|
828
|
+
dryRun?: boolean;
|
|
829
|
+
onProgress?: (message: string) => void;
|
|
830
|
+
}
|
|
831
|
+
interface GoldenEvalCase {
|
|
832
|
+
goldenIrId: number;
|
|
833
|
+
verdict: 'candidate-better' | 'tied' | 'current-better' | 'inconclusive';
|
|
834
|
+
judgeRationale?: string;
|
|
835
|
+
/** Hard-floor violations on the candidate side for this case. */
|
|
836
|
+
floorViolations: Array<'empty' | 'schema' | 'candidate_error'>;
|
|
837
|
+
errorClass?: string;
|
|
838
|
+
incumbent?: {
|
|
839
|
+
latencyMs: number;
|
|
840
|
+
tokensIn: number;
|
|
841
|
+
tokensOut: number;
|
|
842
|
+
text: string;
|
|
843
|
+
};
|
|
844
|
+
candidate?: {
|
|
845
|
+
latencyMs: number;
|
|
846
|
+
tokensIn: number;
|
|
847
|
+
tokensOut: number;
|
|
848
|
+
text: string;
|
|
849
|
+
};
|
|
850
|
+
/** Case excluded from the denominator (incumbent replay/judge failed). */
|
|
851
|
+
excludedReason?: string;
|
|
852
|
+
}
|
|
853
|
+
interface GoldenEvalRunResult {
|
|
854
|
+
verdict: 'promote-ready' | 'not-inferior' | 'inconclusive';
|
|
855
|
+
runId?: number;
|
|
856
|
+
appId: string;
|
|
857
|
+
archetype: string;
|
|
858
|
+
incumbentModel: string;
|
|
859
|
+
candidateModel: string;
|
|
860
|
+
judgeModel: string;
|
|
861
|
+
nCases: number;
|
|
862
|
+
wins: number;
|
|
863
|
+
ties: number;
|
|
864
|
+
losses: number;
|
|
865
|
+
floorViolations: number;
|
|
866
|
+
floorDetail: Record<string, number>;
|
|
867
|
+
winOrTieRatio: number | null;
|
|
868
|
+
latencyRatio: number | null;
|
|
869
|
+
costIncumbentUsd: number | null;
|
|
870
|
+
costCandidateUsd: number | null;
|
|
871
|
+
advisoryOutcomeId?: number;
|
|
872
|
+
cases: GoldenEvalCase[];
|
|
873
|
+
notes: string[];
|
|
874
|
+
}
|
|
875
|
+
declare function runGoldenEval(opts: GoldenEvalOptions): Promise<GoldenEvalRunResult>;
|
|
689
876
|
|
|
690
877
|
/**
|
|
691
878
|
* alpha.60 — the library's own version, baked as a constant so Edge/Worker
|
|
@@ -700,7 +887,7 @@ declare function recordShadowProbe(input: ShadowProbeRecordInput): Promise<void>
|
|
|
700
887
|
* guard in `tests/version.test.ts` fails the suite (and therefore
|
|
701
888
|
* `prepublishOnly`) when they diverge — a stale constant cannot reach npm.
|
|
702
889
|
*/
|
|
703
|
-
declare const LIBRARY_VERSION = "2.0.0-alpha.
|
|
890
|
+
declare const LIBRARY_VERSION = "2.0.0-alpha.62";
|
|
704
891
|
|
|
705
892
|
/**
|
|
706
893
|
* Oracle contract — how an app tells the brain whether a response was good.
|
|
@@ -1886,7 +2073,7 @@ declare function getSequentialStarterChainWithGrounding(archetype: IntentArchety
|
|
|
1886
2073
|
* Sync reader for the brain-driven chains map. Returns bundled
|
|
1887
2074
|
* STARTER_CHAINS when brain-query is disabled, cold, or unreachable.
|
|
1888
2075
|
*/
|
|
1889
|
-
declare const loadChainsFromBrain: () => Record<"ask" | "hunt" | "classify" | "summarize" | "generate" | "extract" | "plan" | "critique" | "transform", string[]>;
|
|
2076
|
+
declare const loadChainsFromBrain: () => Record<"ask" | "hunt" | "classify" | "summarize" | "generate" | "extract" | "plan" | "critique" | "transform" | "judge", string[]>;
|
|
1890
2077
|
|
|
1891
2078
|
/**
|
|
1892
2079
|
* archetype-perf-brain — alpha.11 KG-12 adapter.
|
|
@@ -2489,4 +2676,4 @@ declare function markPromoteReadyHandled(opts: MarkPromoteReadyHandledOptions):
|
|
|
2489
2676
|
*/
|
|
2490
2677
|
declare function compile(ir: PromptIR, opts?: CompileOptions): CompileResult;
|
|
2491
2678
|
|
|
2492
|
-
export { ABSOLUTE_FLOOR, type AISDKConvertedMessage, ARCHETYPE_FAMILY_FITS, ARCHETYPE_FLOOR_DEFAULT, type ActionableAdvisory, Adapter, type AdvisoryResolutionSource, type AdvisorySeverity, type AdvisoryStatus, type AdvisorySuggestedFix, ApiKeys, type AppOracle, type ApplySectionRewritesArgs, type ApplySectionRewritesResult, ArchetypeConvention, type ArchetypeFamilyFit, type ArchetypePerfMap, type ArchetypePerfNMap, type ArchetypePerfScoreResult, type AttachCacheControlResult, BestPracticeAdvisory, type BrainConfig, type BrainDeadLetterEntry, type BrainHealthSnapshot, type BrainQueryConfig, type BrainReadEnv, CallOptions, CallResult, ChainEntry, type CompatibilityIntent, type CompileForAISDKv6Result, type CompileOptions, CompilePolicy, CompileResult, CompiledRequest, DEFAULT_FINDINGS_ENDPOINT, type ExclusionFindingRow, type ExclusionResolutionSource, type ExecuteErr, type ExecuteOk, type ExecuteOptions, type ExecuteResult, type FallbackPosture, FamilyResolutionError, type GetActionableAdvisoriesOptions, type GetDefaultFallbackChainOpts, type GetPerAxisMetricsOpts, type GetRecommendedPrimaryOptions, Grounding, IntentArchetypeName, LIBRARY_VERSION, type LLMJudgeOptions, MEASURED_GROUNDING_MIN_N, type MarkAdvisoryResolvedOptions, type MarkExclusionFindingHandledOptions, type MarkPromoteReadyHandledOptions, type ModelBrainRow, type ModelCompatibility, ModelProfile, NormalizedResponse, type OracleContext, OracleScore, type OutcomePayload, OutcomeResult, PRODUCER_OWNED_RULE_CODES, PROVIDER_ENV_KEYS, PerAxisMetrics, type PricingRow, type ProbeShadowOptions, type ProbeShadowServed, type ProfileToRowOptions, type PromoteReadyFindingRow, type PromoteReadyResolution, PromptIR, Provider, ProviderOverrides, type ProviderReachability, RULE_SEQUENTIAL_TOOL_CLIFF, type ReachabilityOpts, RecordInput, RecordOutcomeInput, type RunAdvisorPhase2Context, SectionRewrite, type ShadowProbeRecordInput, type SupportedProvider, SystemModelMessage, TRANSLATOR_FLOOR, applyArchetypeConvention, applySectionRewrites, attachCacheControlToStreamTextInput, brainHealth, buildLLMJudge, buildShadowProbeRow, call, clearBrain, compile, compileForAISDKv6, configureBrain, countTokens, deriveFamilyFromModelId, deriveOwnership, execute, findBetterFit, flushBrainDeadLetter, getActionableAdvisories, getAllStarterChains, getAllStarterChainsWithGrounding, getArchetypePerfScore, getDefaultFallbackChain, getDefaultFallbackChainWithGrounding, getModelCompatibility, getPerAxisMetrics, getReachabilityDiagnostic, getRecommendedPrimary, getSequentialStarterChain, getSequentialStarterChainWithGrounding, getStaleExclusionFindings, getStarterChain, getStarterChainWithGrounding, isBrainQueryActiveFor, isBrainSync, isExclusionFindingsBrainActive, isModelReachable, isProviderReachable, loadAliasesFromBrain, loadArchetypePerfFromBrain, loadArchetypePerfNFromBrain, loadChainsFromBrain, loadModelsFromBrain, loadPricingFromBrain, markAdvisoryResolved, markExclusionFindingHandled, markPromoteReadyHandled, peekBrainDeadLetter, probeShadow, profileToRow, readBrainReadEnv, record, recordOutcome, recordShadowProbe, resetTokenizer, resolveConventionsForProfile, resolvePricingAt, resolveProviderKey, runAdvisor, setTokenizer };
|
|
2679
|
+
export { ABSOLUTE_FLOOR, type AISDKConvertedMessage, ARCHETYPE_FAMILY_FITS, ARCHETYPE_FLOOR_DEFAULT, type ActionableAdvisory, Adapter, type AdvisoryResolutionSource, type AdvisorySeverity, type AdvisoryStatus, type AdvisorySuggestedFix, ApiKeys, type AppOracle, type ApplySectionRewritesArgs, type ApplySectionRewritesResult, ArchetypeConvention, type ArchetypeFamilyFit, type ArchetypePerfMap, type ArchetypePerfNMap, type ArchetypePerfScoreResult, type AttachCacheControlResult, BestPracticeAdvisory, type BrainConfig, type BrainDeadLetterEntry, type BrainHealthSnapshot, type BrainQueryConfig, type BrainReadEnv, CallOptions, CallResult, ChainEntry, type CompatibilityIntent, type CompileForAISDKv6Result, type CompileOptions, CompilePolicy, CompileResult, CompiledRequest, DEFAULT_FINDINGS_ENDPOINT, type ExclusionFindingRow, type ExclusionResolutionSource, type ExecuteErr, type ExecuteOk, type ExecuteOptions, type ExecuteResult, type FallbackPosture, FamilyResolutionError, type GetActionableAdvisoriesOptions, type GetDefaultFallbackChainOpts, type GetPerAxisMetricsOpts, type GetRecommendedPrimaryOptions, type GoldenEvalCase, type GoldenEvalOptions, type GoldenEvalRunResult, type GoldenIrRecordInput, Grounding, IntentArchetypeName, JUDGE_RUBRICS, LIBRARY_VERSION, type LLMJudgeOptions, MEASURED_GROUNDING_MIN_N, type MarkAdvisoryResolvedOptions, type MarkExclusionFindingHandledOptions, type MarkPromoteReadyHandledOptions, type ModelBrainRow, type ModelCompatibility, ModelProfile, NormalizedResponse, type OracleContext, OracleScore, type OutcomePayload, OutcomeResult, PRODUCER_OWNED_RULE_CODES, PROVIDER_ENV_KEYS, PerAxisMetrics, type PricingRow, type ProbeShadowOptions, type ProbeShadowServed, type ProfileToRowOptions, type PromoteReadyFindingRow, type PromoteReadyResolution, PromptIR, Provider, ProviderOverrides, type ProviderReachability, RULE_SEQUENTIAL_TOOL_CLIFF, type ReachabilityOpts, RecordInput, RecordOutcomeInput, type RunAdvisorPhase2Context, SectionRewrite, type ShadowProbeRecordInput, type SupportedProvider, SystemModelMessage, TRANSLATOR_FLOOR, applyArchetypeConvention, applySectionRewrites, attachCacheControlToStreamTextInput, brainHealth, buildGoldenIrRow, buildLLMJudge, buildPairwiseJudgePrompt, buildShadowProbeRow, call, clearBrain, combineOrderSwappedVerdicts, compile, compileForAISDKv6, configureBrain, countTokens, deriveFamilyFromModelId, deriveOwnership, execute, findBetterFit, flushBrainDeadLetter, getActionableAdvisories, getAllStarterChains, getAllStarterChainsWithGrounding, getArchetypePerfScore, getDefaultFallbackChain, getDefaultFallbackChainWithGrounding, getModelCompatibility, getPerAxisMetrics, getReachabilityDiagnostic, getRecommendedPrimary, getSequentialStarterChain, getSequentialStarterChainWithGrounding, getStaleExclusionFindings, getStarterChain, getStarterChainWithGrounding, isBrainQueryActiveFor, isBrainSync, isExclusionFindingsBrainActive, isModelReachable, isProviderReachable, loadAliasesFromBrain, loadArchetypePerfFromBrain, loadArchetypePerfNFromBrain, loadChainsFromBrain, loadModelsFromBrain, loadPricingFromBrain, markAdvisoryResolved, markExclusionFindingHandled, markPromoteReadyHandled, parseJudgeVerdict, peekBrainDeadLetter, probeShadow, profileToRow, readBrainReadEnv, record, recordGoldenIr, recordOutcome, recordShadowProbe, renderIrForJudge, resetTokenizer, resolveConventionsForProfile, resolvePricingAt, resolveProviderKey, rubricFor, runAdvisor, runGoldenEval, setTokenizer };
|