@warmdrift/kgauto-compiler 2.0.0-alpha.69 → 2.0.0-alpha.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-EZXVJUK7.mjs → chunk-FR4DNGLW.mjs} +1 -1
- package/dist/index.d.mts +171 -2
- package/dist/index.d.ts +171 -2
- package/dist/index.js +254 -18
- package/dist/index.mjs +243 -17
- package/dist/key-health.js +1 -1
- package/dist/key-health.mjs +1 -1
- package/package.json +1 -1
package/dist/index.d.mts
CHANGED
|
@@ -425,6 +425,17 @@ interface BrainQueryConfig {
|
|
|
425
425
|
* `https://kgauto-dashboard.vercel.app/api/kgauto-v2/promotions`.
|
|
426
426
|
*/
|
|
427
427
|
promotionsEndpoint?: string;
|
|
428
|
+
/**
|
|
429
|
+
* alpha.70 — hard opt-out of the measured-failure gate fetch layer.
|
|
430
|
+
* Default true (the gate is opt-OUT, see `BrainConfig.measuredFailureGate`).
|
|
431
|
+
*/
|
|
432
|
+
measuredFailure?: boolean;
|
|
433
|
+
/**
|
|
434
|
+
* alpha.70 — override the measured-failure endpoint. Library appends
|
|
435
|
+
* `?app_id=X`. Defaults to
|
|
436
|
+
* `https://kgauto-dashboard.vercel.app/api/kgauto-v2/measured-failure`.
|
|
437
|
+
*/
|
|
438
|
+
measuredFailureEndpoint?: string;
|
|
428
439
|
}
|
|
429
440
|
interface BrainConfig {
|
|
430
441
|
/** Brain HTTP endpoint base URL (e.g., https://kgauto-brain.vercel.app/api). */
|
|
@@ -495,6 +506,25 @@ interface BrainConfig {
|
|
|
495
506
|
* promotion entirely.
|
|
496
507
|
*/
|
|
497
508
|
autoPromote?: boolean;
|
|
509
|
+
/**
|
|
510
|
+
* alpha.70 — the measured-failure quality gate. **Default ON** wherever
|
|
511
|
+
* the brain is configured; set `false` (or `KGAUTO_MEASURED_FAILURE_GATE=0`)
|
|
512
|
+
* to disable. An explicit value here wins over the env var.
|
|
513
|
+
*
|
|
514
|
+
* Why this is opt-OUT while `autoPromote` is opt-IN: a promotion silently
|
|
515
|
+
* changes WHICH model serves you on kgauto's evidence, so it needs
|
|
516
|
+
* consent. This gate only ever DE-RANKS a model that the consumer's own
|
|
517
|
+
* outcome rows show failing them — the failures are already being paid
|
|
518
|
+
* for on their bill and their users' latency, and the model stays
|
|
519
|
+
* available as a graceful fallback. Requiring an opt-in to stop paying
|
|
520
|
+
* would be the wrong default.
|
|
521
|
+
*
|
|
522
|
+
* Consumer sovereignty is unchanged: `forceModel` still wins. Unlike a
|
|
523
|
+
* promotion, `policy.preferredModels` does NOT defer the gate — a
|
|
524
|
+
* preference says which model you would like to lead, not that a
|
|
525
|
+
* measurably-failing one works.
|
|
526
|
+
*/
|
|
527
|
+
measuredFailureGate?: boolean;
|
|
498
528
|
}
|
|
499
529
|
declare function configureBrain(config: BrainConfig): void;
|
|
500
530
|
declare function clearBrain(): void;
|
|
@@ -973,7 +1003,7 @@ declare function runGoldenEval(opts: GoldenEvalOptions): Promise<GoldenEvalRunRe
|
|
|
973
1003
|
* guard in `tests/version.test.ts` fails the suite (and therefore
|
|
974
1004
|
* `prepublishOnly`) when they diverge — a stale constant cannot reach npm.
|
|
975
1005
|
*/
|
|
976
|
-
declare const LIBRARY_VERSION = "2.0.0-alpha.
|
|
1006
|
+
declare const LIBRARY_VERSION = "2.0.0-alpha.70";
|
|
977
1007
|
|
|
978
1008
|
/**
|
|
979
1009
|
* Oracle contract — how an app tells the brain whether a response was good.
|
|
@@ -2833,6 +2863,145 @@ declare function _testResetPromotions(): void;
|
|
|
2833
2863
|
/** Wait for any in-flight refresh to settle. */
|
|
2834
2864
|
declare function _testWaitForPromotionsRefresh(): Promise<void>;
|
|
2835
2865
|
|
|
2866
|
+
/**
|
|
2867
|
+
* measured-failure-brain — the CREATE-a-gate direction of
|
|
2868
|
+
* measured-supersedes-judgment (alpha.70).
|
|
2869
|
+
*
|
|
2870
|
+
* alpha.49 gives kgauto a quality gate that fires off a model profile's
|
|
2871
|
+
* DECLARED conventions (`archetypeConventions.structuredOutputHint: 'avoid'`).
|
|
2872
|
+
* alpha.64 let a MEASURED eval verdict SUPPRESS that gate
|
|
2873
|
+
* (`kgauto_promotions.suppress_quality_gate`). Nothing could go the other
|
|
2874
|
+
* way — a model whose declarations are wrong could not have a gate CREATED
|
|
2875
|
+
* for it, which is precisely the case where declarations cannot be trusted.
|
|
2876
|
+
*
|
|
2877
|
+
* The originating evidence: `claude-haiku-4-5` on playbacksam/summarize
|
|
2878
|
+
* declares `structuredOutput: 'grammar'` + `archetypePerf.summarize: 8`, so
|
|
2879
|
+
* every quality signal reads green and it wins the surface on price — while
|
|
2880
|
+
* failing **12 of 12 in-window attempts** (2026-07-14..25, seven in one
|
|
2881
|
+
* day). The brain held that evidence and nothing could act on it.
|
|
2882
|
+
*
|
|
2883
|
+
* ── Design notes ───────────────────────────────────────────────────────
|
|
2884
|
+
*
|
|
2885
|
+
* **No stored decision.** The gate is DERIVED from a rolling 28d window
|
|
2886
|
+
* (`kgauto_surface_model_failure_v`), never written down. That kills the
|
|
2887
|
+
* flap problem by construction — there is no state to flap — and it means
|
|
2888
|
+
* a gated model un-gates itself once its failures age out. Combined with
|
|
2889
|
+
* the gate being a DE-RANK (out of leadership, retained as graceful
|
|
2890
|
+
* fallback) rather than a hard block, the model keeps earning evidence, so
|
|
2891
|
+
* this cannot become the exclusion-lock-in trap where a blocked model can
|
|
2892
|
+
* never prove itself again.
|
|
2893
|
+
*
|
|
2894
|
+
* **The threshold is a confidence bound, not a fixed sample floor.** A flat
|
|
2895
|
+
* `minSample: 20` (promotion-guard's rate floor) would NOT have fired on
|
|
2896
|
+
* haiku's n=12 — the consumer would keep paying while we waited for a
|
|
2897
|
+
* rounder number. But distinguishing 100% from 5% needs far fewer trials
|
|
2898
|
+
* than distinguishing 2% from 6%. Gating on the Wilson score lower bound
|
|
2899
|
+
* makes the sample requirement adapt to the effect size automatically,
|
|
2900
|
+
* which is the alpha.67 insight ("the sample floor for a median can be far
|
|
2901
|
+
* lower than for a rate") generalised properly.
|
|
2902
|
+
*
|
|
2903
|
+
* **Direct app-scoped read, not the config-endpoint cache.** Modelled on
|
|
2904
|
+
* `promotions-brain.ts`, deliberately NOT `createBrainQueryCache` — that
|
|
2905
|
+
* routes through the dashboard `/v2/config` projection, which is the third
|
|
2906
|
+
* layer in the L-073 "executable knowledge is dropped at every transport
|
|
2907
|
+
* boundary" chain. One fewer place to forget a field.
|
|
2908
|
+
*/
|
|
2909
|
+
|
|
2910
|
+
/**
|
|
2911
|
+
* One row of the rolling-window evidence view, per (app, archetype, model).
|
|
2912
|
+
* Raw counts only — the brain serves facts, this module decides.
|
|
2913
|
+
*/
|
|
2914
|
+
interface SurfaceFailureRow {
|
|
2915
|
+
archetype: string;
|
|
2916
|
+
model: string;
|
|
2917
|
+
/** Total attempts in-window (served + quality walkaways). */
|
|
2918
|
+
n: number;
|
|
2919
|
+
/** Attempts that failed on the quality axis. */
|
|
2920
|
+
nFail: number;
|
|
2921
|
+
}
|
|
2922
|
+
declare const MEASURED_FAILURE_CFG: {
|
|
2923
|
+
/**
|
|
2924
|
+
* Hard minimum attempts before ANY gate may be created. Guards against
|
|
2925
|
+
* pathological tiny samples that the confidence bound alone would let
|
|
2926
|
+
* through in edge cases. At 5-for-5 the bound clears the threshold; at
|
|
2927
|
+
* 3-for-3 it does not, which is the behaviour we want (three failures is
|
|
2928
|
+
* a bad day, five in a row is a pattern).
|
|
2929
|
+
*/
|
|
2930
|
+
readonly minSample: 5;
|
|
2931
|
+
/**
|
|
2932
|
+
* Gate when we are 95% confident the model fails MORE OFTEN THAN IT
|
|
2933
|
+
* SUCCEEDS on this surface. Deliberately unarguable rather than tuned —
|
|
2934
|
+
* a model that probably fails the majority of the time has no business
|
|
2935
|
+
* leading a surface, whatever its declared scores say.
|
|
2936
|
+
*/
|
|
2937
|
+
readonly lowerBoundThreshold: 0.5;
|
|
2938
|
+
/** 95% one-sided-ish confidence (standard two-sided z at α=0.05). */
|
|
2939
|
+
readonly z: 1.96;
|
|
2940
|
+
/** Must match the view's window. Documented here for the advisory text. */
|
|
2941
|
+
readonly windowDays: 28;
|
|
2942
|
+
};
|
|
2943
|
+
/**
|
|
2944
|
+
* Wilson score interval, lower bound. Preferred over the normal
|
|
2945
|
+
* approximation because it stays sane at the extremes — at p̂ = 1 the
|
|
2946
|
+
* normal approximation gives a zero-width interval (it would gate on a
|
|
2947
|
+
* single failure), while Wilson correctly returns a bound that tightens
|
|
2948
|
+
* with n.
|
|
2949
|
+
*
|
|
2950
|
+
* Returns 0 for n <= 0.
|
|
2951
|
+
*/
|
|
2952
|
+
declare function wilsonLowerBound(failures: number, n: number, z?: number): number;
|
|
2953
|
+
interface MeasuredFailureVerdict {
|
|
2954
|
+
/** Whether the measured-failure quality gate fires for this tuple. */
|
|
2955
|
+
gated: boolean;
|
|
2956
|
+
/** Observed failure rate in-window. */
|
|
2957
|
+
rate: number;
|
|
2958
|
+
/** 95% lower confidence bound on that rate — what the gate tests. */
|
|
2959
|
+
lowerBound: number;
|
|
2960
|
+
/** Attempts backing the verdict. */
|
|
2961
|
+
n: number;
|
|
2962
|
+
nFail: number;
|
|
2963
|
+
}
|
|
2964
|
+
/**
|
|
2965
|
+
* Pure decision, exported for tests and for the advisory text. Separated
|
|
2966
|
+
* from the I/O so the policy can be exercised without a brain.
|
|
2967
|
+
*/
|
|
2968
|
+
declare function judgeMeasuredFailure(row: SurfaceFailureRow | undefined, cfg?: typeof MEASURED_FAILURE_CFG): MeasuredFailureVerdict | undefined;
|
|
2969
|
+
interface MeasuredFailureRuntime {
|
|
2970
|
+
/** Endpoint base URL. The library appends `?app_id=<id>`. */
|
|
2971
|
+
endpoint: string;
|
|
2972
|
+
/** Stale-while-revalidate window in ms. */
|
|
2973
|
+
ttlMs: number;
|
|
2974
|
+
fetchImpl: typeof fetch;
|
|
2975
|
+
onError?: (err: unknown) => void;
|
|
2976
|
+
}
|
|
2977
|
+
/** Default endpoint on the kgauto-dashboard. Mirrors the promotions convention. */
|
|
2978
|
+
declare const DEFAULT_MEASURED_FAILURE_ENDPOINT = "https://kgauto-dashboard.vercel.app/api/kgauto-v2/measured-failure";
|
|
2979
|
+
/**
|
|
2980
|
+
* Opt-out env switch. UNLIKE auto-promote this defaults ON wherever the
|
|
2981
|
+
* brain is configured, because the gate is protective: it only ever
|
|
2982
|
+
* de-ranks a model the consumer's OWN traffic shows to be failing them, and
|
|
2983
|
+
* the failure is already being paid for. Set `KGAUTO_MEASURED_FAILURE_GATE=0`
|
|
2984
|
+
* (or 'false') to disable.
|
|
2985
|
+
*/
|
|
2986
|
+
declare function isMeasuredFailureGateEnabledFromEnv(envSource?: Record<string, string | undefined>): boolean;
|
|
2987
|
+
declare function configureMeasuredFailureBrain(rt: MeasuredFailureRuntime | undefined): void;
|
|
2988
|
+
declare function isMeasuredFailureBrainActive(): boolean;
|
|
2989
|
+
interface GetMeasuredFailureOpts {
|
|
2990
|
+
appId: string;
|
|
2991
|
+
archetype: IntentArchetypeName | string;
|
|
2992
|
+
model: string;
|
|
2993
|
+
}
|
|
2994
|
+
/**
|
|
2995
|
+
* Sync reader. Returns the verdict for `(appId, archetype, model)` or
|
|
2996
|
+
* undefined (not configured / cold / below minSample / brain down).
|
|
2997
|
+
*
|
|
2998
|
+
* NEVER throws. Cold start returns undefined and warms in the background —
|
|
2999
|
+
* same posture as every other brain-driven lever in compile().
|
|
3000
|
+
*/
|
|
3001
|
+
declare function getMeasuredFailureVerdict(opts: GetMeasuredFailureOpts): MeasuredFailureVerdict | undefined;
|
|
3002
|
+
declare function _testResetMeasuredFailure(): void;
|
|
3003
|
+
declare function _testWaitForMeasuredFailureRefresh(): Promise<void>;
|
|
3004
|
+
|
|
2836
3005
|
/**
|
|
2837
3006
|
* @warmdrift/kgauto v2 — prompt compiler + central learning brain.
|
|
2838
3007
|
*
|
|
@@ -2879,4 +3048,4 @@ declare function _testWaitForPromotionsRefresh(): Promise<void>;
|
|
|
2879
3048
|
*/
|
|
2880
3049
|
declare function compile(ir: PromptIR, opts?: CompileOptions): CompileResult;
|
|
2881
3050
|
|
|
2882
|
-
export { ABSOLUTE_FLOOR, type AISDKConvertedMessage, ARCHETYPE_FAMILY_FITS, ARCHETYPE_FLOOR_DEFAULT, type ActionableAdvisory, Adapter, type AdvisoryResolutionSource, type AdvisorySeverity, type AdvisoryStatus, type AdvisorySuggestedFix, ApiKeys, type AppOracle, type ApplySectionRewritesArgs, type ApplySectionRewritesResult, ArchetypeConvention, type ArchetypeFamilyFit, type ArchetypePerfMap, type ArchetypePerfNMap, type ArchetypePerfScoreResult, type AttachCacheControlResult, BestPracticeAdvisory, type BrainConfig, type BrainDeadLetterEntry, type BrainHealthSnapshot, type BrainQueryConfig, type BrainReadEnv, CallOptions, CallResult, ChainEntry, type CompatibilityIntent, type CompileForAISDKv6Result, type CompileOptions, CompilePolicy, CompileResult, CompiledRequest, DEFAULT_FINDINGS_ENDPOINT, DEFAULT_PROMOTIONS_ENDPOINT, type ExclusionFindingRow, type ExclusionResolutionSource, type ExecuteErr, type ExecuteOk, type ExecuteOptions, type ExecuteResult, type FallbackPosture, FamilyResolutionError, type GetActionableAdvisoriesOptions, type GetApplicablePromotionOpts, type GetDefaultFallbackChainOpts, type GetPerAxisMetricsOpts, type GetRecommendedPrimaryOptions, type GoldenEvalCase, type GoldenEvalOptions, type GoldenEvalRunResult, type GoldenIrRecordInput, Grounding, IntentArchetypeName, JUDGE_RUBRICS, LIBRARY_VERSION, type LLMJudgeOptions, MEASURED_GROUNDING_MIN_N, type MarkAdvisoryResolvedOptions, type MarkExclusionFindingHandledOptions, type MarkPromoteReadyHandledOptions, type ModelBrainRow, type ModelCompatibility, ModelProfile, NormalizedResponse, type OracleContext, OracleScore, type OutcomePayload, OutcomeResult, OutputMode, PRODUCER_OWNED_RULE_CODES, PROVIDER_ENV_KEYS, PerAxisMetrics, type PricingRow, type ProbeShadowOptions, type ProbeShadowServed, type ProfileToRowOptions, type PromoteReadyFindingRow, type PromoteReadyResolution, type PromotionRow, type PromotionsRuntime, PromptIR, Provider, ProviderOverrides, type ProviderReachability, RULE_SEQUENTIAL_TOOL_CLIFF, type ReachabilityOpts, RecordInput, RecordOutcomeInput, type RunAdvisorPhase2Context, SectionRewrite, type ShadowProbeRecordInput, type SupportedProvider, SystemModelMessage, TRANSLATOR_FLOOR, _testResetPromotions, _testWaitForPromotionsRefresh, applyArchetypeConvention, applySectionRewrites, attachCacheControlToStreamTextInput, brainHealth, buildGoldenIrRow, buildLLMJudge, buildPairwiseJudgePrompt, buildShadowProbeRow, call, clearBrain, combineOrderSwappedVerdicts, compile, compileForAISDKv6, configureBrain, configurePromotionsBrain, countTokens, deriveFamilyFromModelId, deriveOwnership, execute, findBetterFit, flushBrainDeadLetter, getActionableAdvisories, getAllStarterChains, getAllStarterChainsWithGrounding, getApplicablePromotion, getArchetypePerfScore, getDefaultFallbackChain, getDefaultFallbackChainWithGrounding, getModelCompatibility, getPerAxisMetrics, getReachabilityDiagnostic, getRecommendedPrimary, getSequentialStarterChain, getSequentialStarterChainWithGrounding, getStaleExclusionFindings, getStarterChain, getStarterChainWithGrounding, isAutoPromoteEnabledFromEnv, isBrainQueryActiveFor, isBrainSync, isExclusionFindingsBrainActive, isModelReachable, isPromotionsBrainActive, isProviderReachable, loadAliasesFromBrain, loadArchetypePerfFromBrain, loadArchetypePerfNFromBrain, loadChainsFromBrain, loadModelsFromBrain, loadPricingFromBrain, markAdvisoryResolved, markExclusionFindingHandled, markPromoteReadyHandled, parseJudgeVerdict, peekBrainDeadLetter, probeShadow, profileToRow, readBrainReadEnv, record, recordGoldenIr, recordOutcome, recordShadowProbe, renderIrForJudge, resetTokenizer, resolveConventionsForProfile, resolvePricingAt, resolveProviderKey, rubricFor, runAdvisor, runGoldenEval, setTokenizer };
|
|
3051
|
+
export { ABSOLUTE_FLOOR, type AISDKConvertedMessage, ARCHETYPE_FAMILY_FITS, ARCHETYPE_FLOOR_DEFAULT, type ActionableAdvisory, Adapter, type AdvisoryResolutionSource, type AdvisorySeverity, type AdvisoryStatus, type AdvisorySuggestedFix, ApiKeys, type AppOracle, type ApplySectionRewritesArgs, type ApplySectionRewritesResult, ArchetypeConvention, type ArchetypeFamilyFit, type ArchetypePerfMap, type ArchetypePerfNMap, type ArchetypePerfScoreResult, type AttachCacheControlResult, BestPracticeAdvisory, type BrainConfig, type BrainDeadLetterEntry, type BrainHealthSnapshot, type BrainQueryConfig, type BrainReadEnv, CallOptions, CallResult, ChainEntry, type CompatibilityIntent, type CompileForAISDKv6Result, type CompileOptions, CompilePolicy, CompileResult, CompiledRequest, DEFAULT_FINDINGS_ENDPOINT, DEFAULT_MEASURED_FAILURE_ENDPOINT, DEFAULT_PROMOTIONS_ENDPOINT, type ExclusionFindingRow, type ExclusionResolutionSource, type ExecuteErr, type ExecuteOk, type ExecuteOptions, type ExecuteResult, type FallbackPosture, FamilyResolutionError, type GetActionableAdvisoriesOptions, type GetApplicablePromotionOpts, type GetDefaultFallbackChainOpts, type GetMeasuredFailureOpts, type GetPerAxisMetricsOpts, type GetRecommendedPrimaryOptions, type GoldenEvalCase, type GoldenEvalOptions, type GoldenEvalRunResult, type GoldenIrRecordInput, Grounding, IntentArchetypeName, JUDGE_RUBRICS, LIBRARY_VERSION, type LLMJudgeOptions, MEASURED_FAILURE_CFG, MEASURED_GROUNDING_MIN_N, type MarkAdvisoryResolvedOptions, type MarkExclusionFindingHandledOptions, type MarkPromoteReadyHandledOptions, type MeasuredFailureRuntime, type MeasuredFailureVerdict, type ModelBrainRow, type ModelCompatibility, ModelProfile, NormalizedResponse, type OracleContext, OracleScore, type OutcomePayload, OutcomeResult, OutputMode, PRODUCER_OWNED_RULE_CODES, PROVIDER_ENV_KEYS, PerAxisMetrics, type PricingRow, type ProbeShadowOptions, type ProbeShadowServed, type ProfileToRowOptions, type PromoteReadyFindingRow, type PromoteReadyResolution, type PromotionRow, type PromotionsRuntime, PromptIR, Provider, ProviderOverrides, type ProviderReachability, RULE_SEQUENTIAL_TOOL_CLIFF, type ReachabilityOpts, RecordInput, RecordOutcomeInput, type RunAdvisorPhase2Context, SectionRewrite, type ShadowProbeRecordInput, type SupportedProvider, type SurfaceFailureRow, SystemModelMessage, TRANSLATOR_FLOOR, _testResetMeasuredFailure, _testResetPromotions, _testWaitForMeasuredFailureRefresh, _testWaitForPromotionsRefresh, applyArchetypeConvention, applySectionRewrites, attachCacheControlToStreamTextInput, brainHealth, buildGoldenIrRow, buildLLMJudge, buildPairwiseJudgePrompt, buildShadowProbeRow, call, clearBrain, combineOrderSwappedVerdicts, compile, compileForAISDKv6, configureBrain, configureMeasuredFailureBrain, configurePromotionsBrain, countTokens, deriveFamilyFromModelId, deriveOwnership, execute, findBetterFit, flushBrainDeadLetter, getActionableAdvisories, getAllStarterChains, getAllStarterChainsWithGrounding, getApplicablePromotion, getArchetypePerfScore, getDefaultFallbackChain, getDefaultFallbackChainWithGrounding, getMeasuredFailureVerdict, getModelCompatibility, getPerAxisMetrics, getReachabilityDiagnostic, getRecommendedPrimary, getSequentialStarterChain, getSequentialStarterChainWithGrounding, getStaleExclusionFindings, getStarterChain, getStarterChainWithGrounding, isAutoPromoteEnabledFromEnv, isBrainQueryActiveFor, isBrainSync, isExclusionFindingsBrainActive, isMeasuredFailureBrainActive, isMeasuredFailureGateEnabledFromEnv, isModelReachable, isPromotionsBrainActive, isProviderReachable, judgeMeasuredFailure, loadAliasesFromBrain, loadArchetypePerfFromBrain, loadArchetypePerfNFromBrain, loadChainsFromBrain, loadModelsFromBrain, loadPricingFromBrain, markAdvisoryResolved, markExclusionFindingHandled, markPromoteReadyHandled, parseJudgeVerdict, peekBrainDeadLetter, probeShadow, profileToRow, readBrainReadEnv, record, recordGoldenIr, recordOutcome, recordShadowProbe, renderIrForJudge, resetTokenizer, resolveConventionsForProfile, resolvePricingAt, resolveProviderKey, rubricFor, runAdvisor, runGoldenEval, setTokenizer, wilsonLowerBound };
|
package/dist/index.d.ts
CHANGED
|
@@ -425,6 +425,17 @@ interface BrainQueryConfig {
|
|
|
425
425
|
* `https://kgauto-dashboard.vercel.app/api/kgauto-v2/promotions`.
|
|
426
426
|
*/
|
|
427
427
|
promotionsEndpoint?: string;
|
|
428
|
+
/**
|
|
429
|
+
* alpha.70 — hard opt-out of the measured-failure gate fetch layer.
|
|
430
|
+
* Default true (the gate is opt-OUT, see `BrainConfig.measuredFailureGate`).
|
|
431
|
+
*/
|
|
432
|
+
measuredFailure?: boolean;
|
|
433
|
+
/**
|
|
434
|
+
* alpha.70 — override the measured-failure endpoint. Library appends
|
|
435
|
+
* `?app_id=X`. Defaults to
|
|
436
|
+
* `https://kgauto-dashboard.vercel.app/api/kgauto-v2/measured-failure`.
|
|
437
|
+
*/
|
|
438
|
+
measuredFailureEndpoint?: string;
|
|
428
439
|
}
|
|
429
440
|
interface BrainConfig {
|
|
430
441
|
/** Brain HTTP endpoint base URL (e.g., https://kgauto-brain.vercel.app/api). */
|
|
@@ -495,6 +506,25 @@ interface BrainConfig {
|
|
|
495
506
|
* promotion entirely.
|
|
496
507
|
*/
|
|
497
508
|
autoPromote?: boolean;
|
|
509
|
+
/**
|
|
510
|
+
* alpha.70 — the measured-failure quality gate. **Default ON** wherever
|
|
511
|
+
* the brain is configured; set `false` (or `KGAUTO_MEASURED_FAILURE_GATE=0`)
|
|
512
|
+
* to disable. An explicit value here wins over the env var.
|
|
513
|
+
*
|
|
514
|
+
* Why this is opt-OUT while `autoPromote` is opt-IN: a promotion silently
|
|
515
|
+
* changes WHICH model serves you on kgauto's evidence, so it needs
|
|
516
|
+
* consent. This gate only ever DE-RANKS a model that the consumer's own
|
|
517
|
+
* outcome rows show failing them — the failures are already being paid
|
|
518
|
+
* for on their bill and their users' latency, and the model stays
|
|
519
|
+
* available as a graceful fallback. Requiring an opt-in to stop paying
|
|
520
|
+
* would be the wrong default.
|
|
521
|
+
*
|
|
522
|
+
* Consumer sovereignty is unchanged: `forceModel` still wins. Unlike a
|
|
523
|
+
* promotion, `policy.preferredModels` does NOT defer the gate — a
|
|
524
|
+
* preference says which model you would like to lead, not that a
|
|
525
|
+
* measurably-failing one works.
|
|
526
|
+
*/
|
|
527
|
+
measuredFailureGate?: boolean;
|
|
498
528
|
}
|
|
499
529
|
declare function configureBrain(config: BrainConfig): void;
|
|
500
530
|
declare function clearBrain(): void;
|
|
@@ -973,7 +1003,7 @@ declare function runGoldenEval(opts: GoldenEvalOptions): Promise<GoldenEvalRunRe
|
|
|
973
1003
|
* guard in `tests/version.test.ts` fails the suite (and therefore
|
|
974
1004
|
* `prepublishOnly`) when they diverge — a stale constant cannot reach npm.
|
|
975
1005
|
*/
|
|
976
|
-
declare const LIBRARY_VERSION = "2.0.0-alpha.
|
|
1006
|
+
declare const LIBRARY_VERSION = "2.0.0-alpha.70";
|
|
977
1007
|
|
|
978
1008
|
/**
|
|
979
1009
|
* Oracle contract — how an app tells the brain whether a response was good.
|
|
@@ -2833,6 +2863,145 @@ declare function _testResetPromotions(): void;
|
|
|
2833
2863
|
/** Wait for any in-flight refresh to settle. */
|
|
2834
2864
|
declare function _testWaitForPromotionsRefresh(): Promise<void>;
|
|
2835
2865
|
|
|
2866
|
+
/**
|
|
2867
|
+
* measured-failure-brain — the CREATE-a-gate direction of
|
|
2868
|
+
* measured-supersedes-judgment (alpha.70).
|
|
2869
|
+
*
|
|
2870
|
+
* alpha.49 gives kgauto a quality gate that fires off a model profile's
|
|
2871
|
+
* DECLARED conventions (`archetypeConventions.structuredOutputHint: 'avoid'`).
|
|
2872
|
+
* alpha.64 let a MEASURED eval verdict SUPPRESS that gate
|
|
2873
|
+
* (`kgauto_promotions.suppress_quality_gate`). Nothing could go the other
|
|
2874
|
+
* way — a model whose declarations are wrong could not have a gate CREATED
|
|
2875
|
+
* for it, which is precisely the case where declarations cannot be trusted.
|
|
2876
|
+
*
|
|
2877
|
+
* The originating evidence: `claude-haiku-4-5` on playbacksam/summarize
|
|
2878
|
+
* declares `structuredOutput: 'grammar'` + `archetypePerf.summarize: 8`, so
|
|
2879
|
+
* every quality signal reads green and it wins the surface on price — while
|
|
2880
|
+
* failing **12 of 12 in-window attempts** (2026-07-14..25, seven in one
|
|
2881
|
+
* day). The brain held that evidence and nothing could act on it.
|
|
2882
|
+
*
|
|
2883
|
+
* ── Design notes ───────────────────────────────────────────────────────
|
|
2884
|
+
*
|
|
2885
|
+
* **No stored decision.** The gate is DERIVED from a rolling 28d window
|
|
2886
|
+
* (`kgauto_surface_model_failure_v`), never written down. That kills the
|
|
2887
|
+
* flap problem by construction — there is no state to flap — and it means
|
|
2888
|
+
* a gated model un-gates itself once its failures age out. Combined with
|
|
2889
|
+
* the gate being a DE-RANK (out of leadership, retained as graceful
|
|
2890
|
+
* fallback) rather than a hard block, the model keeps earning evidence, so
|
|
2891
|
+
* this cannot become the exclusion-lock-in trap where a blocked model can
|
|
2892
|
+
* never prove itself again.
|
|
2893
|
+
*
|
|
2894
|
+
* **The threshold is a confidence bound, not a fixed sample floor.** A flat
|
|
2895
|
+
* `minSample: 20` (promotion-guard's rate floor) would NOT have fired on
|
|
2896
|
+
* haiku's n=12 — the consumer would keep paying while we waited for a
|
|
2897
|
+
* rounder number. But distinguishing 100% from 5% needs far fewer trials
|
|
2898
|
+
* than distinguishing 2% from 6%. Gating on the Wilson score lower bound
|
|
2899
|
+
* makes the sample requirement adapt to the effect size automatically,
|
|
2900
|
+
* which is the alpha.67 insight ("the sample floor for a median can be far
|
|
2901
|
+
* lower than for a rate") generalised properly.
|
|
2902
|
+
*
|
|
2903
|
+
* **Direct app-scoped read, not the config-endpoint cache.** Modelled on
|
|
2904
|
+
* `promotions-brain.ts`, deliberately NOT `createBrainQueryCache` — that
|
|
2905
|
+
* routes through the dashboard `/v2/config` projection, which is the third
|
|
2906
|
+
* layer in the L-073 "executable knowledge is dropped at every transport
|
|
2907
|
+
* boundary" chain. One fewer place to forget a field.
|
|
2908
|
+
*/
|
|
2909
|
+
|
|
2910
|
+
/**
|
|
2911
|
+
* One row of the rolling-window evidence view, per (app, archetype, model).
|
|
2912
|
+
* Raw counts only — the brain serves facts, this module decides.
|
|
2913
|
+
*/
|
|
2914
|
+
interface SurfaceFailureRow {
|
|
2915
|
+
archetype: string;
|
|
2916
|
+
model: string;
|
|
2917
|
+
/** Total attempts in-window (served + quality walkaways). */
|
|
2918
|
+
n: number;
|
|
2919
|
+
/** Attempts that failed on the quality axis. */
|
|
2920
|
+
nFail: number;
|
|
2921
|
+
}
|
|
2922
|
+
declare const MEASURED_FAILURE_CFG: {
|
|
2923
|
+
/**
|
|
2924
|
+
* Hard minimum attempts before ANY gate may be created. Guards against
|
|
2925
|
+
* pathological tiny samples that the confidence bound alone would let
|
|
2926
|
+
* through in edge cases. At 5-for-5 the bound clears the threshold; at
|
|
2927
|
+
* 3-for-3 it does not, which is the behaviour we want (three failures is
|
|
2928
|
+
* a bad day, five in a row is a pattern).
|
|
2929
|
+
*/
|
|
2930
|
+
readonly minSample: 5;
|
|
2931
|
+
/**
|
|
2932
|
+
* Gate when we are 95% confident the model fails MORE OFTEN THAN IT
|
|
2933
|
+
* SUCCEEDS on this surface. Deliberately unarguable rather than tuned —
|
|
2934
|
+
* a model that probably fails the majority of the time has no business
|
|
2935
|
+
* leading a surface, whatever its declared scores say.
|
|
2936
|
+
*/
|
|
2937
|
+
readonly lowerBoundThreshold: 0.5;
|
|
2938
|
+
/** 95% one-sided-ish confidence (standard two-sided z at α=0.05). */
|
|
2939
|
+
readonly z: 1.96;
|
|
2940
|
+
/** Must match the view's window. Documented here for the advisory text. */
|
|
2941
|
+
readonly windowDays: 28;
|
|
2942
|
+
};
|
|
2943
|
+
/**
|
|
2944
|
+
* Wilson score interval, lower bound. Preferred over the normal
|
|
2945
|
+
* approximation because it stays sane at the extremes — at p̂ = 1 the
|
|
2946
|
+
* normal approximation gives a zero-width interval (it would gate on a
|
|
2947
|
+
* single failure), while Wilson correctly returns a bound that tightens
|
|
2948
|
+
* with n.
|
|
2949
|
+
*
|
|
2950
|
+
* Returns 0 for n <= 0.
|
|
2951
|
+
*/
|
|
2952
|
+
declare function wilsonLowerBound(failures: number, n: number, z?: number): number;
|
|
2953
|
+
interface MeasuredFailureVerdict {
|
|
2954
|
+
/** Whether the measured-failure quality gate fires for this tuple. */
|
|
2955
|
+
gated: boolean;
|
|
2956
|
+
/** Observed failure rate in-window. */
|
|
2957
|
+
rate: number;
|
|
2958
|
+
/** 95% lower confidence bound on that rate — what the gate tests. */
|
|
2959
|
+
lowerBound: number;
|
|
2960
|
+
/** Attempts backing the verdict. */
|
|
2961
|
+
n: number;
|
|
2962
|
+
nFail: number;
|
|
2963
|
+
}
|
|
2964
|
+
/**
|
|
2965
|
+
* Pure decision, exported for tests and for the advisory text. Separated
|
|
2966
|
+
* from the I/O so the policy can be exercised without a brain.
|
|
2967
|
+
*/
|
|
2968
|
+
declare function judgeMeasuredFailure(row: SurfaceFailureRow | undefined, cfg?: typeof MEASURED_FAILURE_CFG): MeasuredFailureVerdict | undefined;
|
|
2969
|
+
interface MeasuredFailureRuntime {
|
|
2970
|
+
/** Endpoint base URL. The library appends `?app_id=<id>`. */
|
|
2971
|
+
endpoint: string;
|
|
2972
|
+
/** Stale-while-revalidate window in ms. */
|
|
2973
|
+
ttlMs: number;
|
|
2974
|
+
fetchImpl: typeof fetch;
|
|
2975
|
+
onError?: (err: unknown) => void;
|
|
2976
|
+
}
|
|
2977
|
+
/** Default endpoint on the kgauto-dashboard. Mirrors the promotions convention. */
|
|
2978
|
+
declare const DEFAULT_MEASURED_FAILURE_ENDPOINT = "https://kgauto-dashboard.vercel.app/api/kgauto-v2/measured-failure";
|
|
2979
|
+
/**
|
|
2980
|
+
* Opt-out env switch. UNLIKE auto-promote this defaults ON wherever the
|
|
2981
|
+
* brain is configured, because the gate is protective: it only ever
|
|
2982
|
+
* de-ranks a model the consumer's OWN traffic shows to be failing them, and
|
|
2983
|
+
* the failure is already being paid for. Set `KGAUTO_MEASURED_FAILURE_GATE=0`
|
|
2984
|
+
* (or 'false') to disable.
|
|
2985
|
+
*/
|
|
2986
|
+
declare function isMeasuredFailureGateEnabledFromEnv(envSource?: Record<string, string | undefined>): boolean;
|
|
2987
|
+
declare function configureMeasuredFailureBrain(rt: MeasuredFailureRuntime | undefined): void;
|
|
2988
|
+
declare function isMeasuredFailureBrainActive(): boolean;
|
|
2989
|
+
interface GetMeasuredFailureOpts {
|
|
2990
|
+
appId: string;
|
|
2991
|
+
archetype: IntentArchetypeName | string;
|
|
2992
|
+
model: string;
|
|
2993
|
+
}
|
|
2994
|
+
/**
|
|
2995
|
+
* Sync reader. Returns the verdict for `(appId, archetype, model)` or
|
|
2996
|
+
* undefined (not configured / cold / below minSample / brain down).
|
|
2997
|
+
*
|
|
2998
|
+
* NEVER throws. Cold start returns undefined and warms in the background —
|
|
2999
|
+
* same posture as every other brain-driven lever in compile().
|
|
3000
|
+
*/
|
|
3001
|
+
declare function getMeasuredFailureVerdict(opts: GetMeasuredFailureOpts): MeasuredFailureVerdict | undefined;
|
|
3002
|
+
declare function _testResetMeasuredFailure(): void;
|
|
3003
|
+
declare function _testWaitForMeasuredFailureRefresh(): Promise<void>;
|
|
3004
|
+
|
|
2836
3005
|
/**
|
|
2837
3006
|
* @warmdrift/kgauto v2 — prompt compiler + central learning brain.
|
|
2838
3007
|
*
|
|
@@ -2879,4 +3048,4 @@ declare function _testWaitForPromotionsRefresh(): Promise<void>;
|
|
|
2879
3048
|
*/
|
|
2880
3049
|
declare function compile(ir: PromptIR, opts?: CompileOptions): CompileResult;
|
|
2881
3050
|
|
|
2882
|
-
export { ABSOLUTE_FLOOR, type AISDKConvertedMessage, ARCHETYPE_FAMILY_FITS, ARCHETYPE_FLOOR_DEFAULT, type ActionableAdvisory, Adapter, type AdvisoryResolutionSource, type AdvisorySeverity, type AdvisoryStatus, type AdvisorySuggestedFix, ApiKeys, type AppOracle, type ApplySectionRewritesArgs, type ApplySectionRewritesResult, ArchetypeConvention, type ArchetypeFamilyFit, type ArchetypePerfMap, type ArchetypePerfNMap, type ArchetypePerfScoreResult, type AttachCacheControlResult, BestPracticeAdvisory, type BrainConfig, type BrainDeadLetterEntry, type BrainHealthSnapshot, type BrainQueryConfig, type BrainReadEnv, CallOptions, CallResult, ChainEntry, type CompatibilityIntent, type CompileForAISDKv6Result, type CompileOptions, CompilePolicy, CompileResult, CompiledRequest, DEFAULT_FINDINGS_ENDPOINT, DEFAULT_PROMOTIONS_ENDPOINT, type ExclusionFindingRow, type ExclusionResolutionSource, type ExecuteErr, type ExecuteOk, type ExecuteOptions, type ExecuteResult, type FallbackPosture, FamilyResolutionError, type GetActionableAdvisoriesOptions, type GetApplicablePromotionOpts, type GetDefaultFallbackChainOpts, type GetPerAxisMetricsOpts, type GetRecommendedPrimaryOptions, type GoldenEvalCase, type GoldenEvalOptions, type GoldenEvalRunResult, type GoldenIrRecordInput, Grounding, IntentArchetypeName, JUDGE_RUBRICS, LIBRARY_VERSION, type LLMJudgeOptions, MEASURED_GROUNDING_MIN_N, type MarkAdvisoryResolvedOptions, type MarkExclusionFindingHandledOptions, type MarkPromoteReadyHandledOptions, type ModelBrainRow, type ModelCompatibility, ModelProfile, NormalizedResponse, type OracleContext, OracleScore, type OutcomePayload, OutcomeResult, OutputMode, PRODUCER_OWNED_RULE_CODES, PROVIDER_ENV_KEYS, PerAxisMetrics, type PricingRow, type ProbeShadowOptions, type ProbeShadowServed, type ProfileToRowOptions, type PromoteReadyFindingRow, type PromoteReadyResolution, type PromotionRow, type PromotionsRuntime, PromptIR, Provider, ProviderOverrides, type ProviderReachability, RULE_SEQUENTIAL_TOOL_CLIFF, type ReachabilityOpts, RecordInput, RecordOutcomeInput, type RunAdvisorPhase2Context, SectionRewrite, type ShadowProbeRecordInput, type SupportedProvider, SystemModelMessage, TRANSLATOR_FLOOR, _testResetPromotions, _testWaitForPromotionsRefresh, applyArchetypeConvention, applySectionRewrites, attachCacheControlToStreamTextInput, brainHealth, buildGoldenIrRow, buildLLMJudge, buildPairwiseJudgePrompt, buildShadowProbeRow, call, clearBrain, combineOrderSwappedVerdicts, compile, compileForAISDKv6, configureBrain, configurePromotionsBrain, countTokens, deriveFamilyFromModelId, deriveOwnership, execute, findBetterFit, flushBrainDeadLetter, getActionableAdvisories, getAllStarterChains, getAllStarterChainsWithGrounding, getApplicablePromotion, getArchetypePerfScore, getDefaultFallbackChain, getDefaultFallbackChainWithGrounding, getModelCompatibility, getPerAxisMetrics, getReachabilityDiagnostic, getRecommendedPrimary, getSequentialStarterChain, getSequentialStarterChainWithGrounding, getStaleExclusionFindings, getStarterChain, getStarterChainWithGrounding, isAutoPromoteEnabledFromEnv, isBrainQueryActiveFor, isBrainSync, isExclusionFindingsBrainActive, isModelReachable, isPromotionsBrainActive, isProviderReachable, loadAliasesFromBrain, loadArchetypePerfFromBrain, loadArchetypePerfNFromBrain, loadChainsFromBrain, loadModelsFromBrain, loadPricingFromBrain, markAdvisoryResolved, markExclusionFindingHandled, markPromoteReadyHandled, parseJudgeVerdict, peekBrainDeadLetter, probeShadow, profileToRow, readBrainReadEnv, record, recordGoldenIr, recordOutcome, recordShadowProbe, renderIrForJudge, resetTokenizer, resolveConventionsForProfile, resolvePricingAt, resolveProviderKey, rubricFor, runAdvisor, runGoldenEval, setTokenizer };
|
|
3051
|
+
export { ABSOLUTE_FLOOR, type AISDKConvertedMessage, ARCHETYPE_FAMILY_FITS, ARCHETYPE_FLOOR_DEFAULT, type ActionableAdvisory, Adapter, type AdvisoryResolutionSource, type AdvisorySeverity, type AdvisoryStatus, type AdvisorySuggestedFix, ApiKeys, type AppOracle, type ApplySectionRewritesArgs, type ApplySectionRewritesResult, ArchetypeConvention, type ArchetypeFamilyFit, type ArchetypePerfMap, type ArchetypePerfNMap, type ArchetypePerfScoreResult, type AttachCacheControlResult, BestPracticeAdvisory, type BrainConfig, type BrainDeadLetterEntry, type BrainHealthSnapshot, type BrainQueryConfig, type BrainReadEnv, CallOptions, CallResult, ChainEntry, type CompatibilityIntent, type CompileForAISDKv6Result, type CompileOptions, CompilePolicy, CompileResult, CompiledRequest, DEFAULT_FINDINGS_ENDPOINT, DEFAULT_MEASURED_FAILURE_ENDPOINT, DEFAULT_PROMOTIONS_ENDPOINT, type ExclusionFindingRow, type ExclusionResolutionSource, type ExecuteErr, type ExecuteOk, type ExecuteOptions, type ExecuteResult, type FallbackPosture, FamilyResolutionError, type GetActionableAdvisoriesOptions, type GetApplicablePromotionOpts, type GetDefaultFallbackChainOpts, type GetMeasuredFailureOpts, type GetPerAxisMetricsOpts, type GetRecommendedPrimaryOptions, type GoldenEvalCase, type GoldenEvalOptions, type GoldenEvalRunResult, type GoldenIrRecordInput, Grounding, IntentArchetypeName, JUDGE_RUBRICS, LIBRARY_VERSION, type LLMJudgeOptions, MEASURED_FAILURE_CFG, MEASURED_GROUNDING_MIN_N, type MarkAdvisoryResolvedOptions, type MarkExclusionFindingHandledOptions, type MarkPromoteReadyHandledOptions, type MeasuredFailureRuntime, type MeasuredFailureVerdict, type ModelBrainRow, type ModelCompatibility, ModelProfile, NormalizedResponse, type OracleContext, OracleScore, type OutcomePayload, OutcomeResult, OutputMode, PRODUCER_OWNED_RULE_CODES, PROVIDER_ENV_KEYS, PerAxisMetrics, type PricingRow, type ProbeShadowOptions, type ProbeShadowServed, type ProfileToRowOptions, type PromoteReadyFindingRow, type PromoteReadyResolution, type PromotionRow, type PromotionsRuntime, PromptIR, Provider, ProviderOverrides, type ProviderReachability, RULE_SEQUENTIAL_TOOL_CLIFF, type ReachabilityOpts, RecordInput, RecordOutcomeInput, type RunAdvisorPhase2Context, SectionRewrite, type ShadowProbeRecordInput, type SupportedProvider, type SurfaceFailureRow, SystemModelMessage, TRANSLATOR_FLOOR, _testResetMeasuredFailure, _testResetPromotions, _testWaitForMeasuredFailureRefresh, _testWaitForPromotionsRefresh, applyArchetypeConvention, applySectionRewrites, attachCacheControlToStreamTextInput, brainHealth, buildGoldenIrRow, buildLLMJudge, buildPairwiseJudgePrompt, buildShadowProbeRow, call, clearBrain, combineOrderSwappedVerdicts, compile, compileForAISDKv6, configureBrain, configureMeasuredFailureBrain, configurePromotionsBrain, countTokens, deriveFamilyFromModelId, deriveOwnership, execute, findBetterFit, flushBrainDeadLetter, getActionableAdvisories, getAllStarterChains, getAllStarterChainsWithGrounding, getApplicablePromotion, getArchetypePerfScore, getDefaultFallbackChain, getDefaultFallbackChainWithGrounding, getMeasuredFailureVerdict, getModelCompatibility, getPerAxisMetrics, getReachabilityDiagnostic, getRecommendedPrimary, getSequentialStarterChain, getSequentialStarterChainWithGrounding, getStaleExclusionFindings, getStarterChain, getStarterChainWithGrounding, isAutoPromoteEnabledFromEnv, isBrainQueryActiveFor, isBrainSync, isExclusionFindingsBrainActive, isMeasuredFailureBrainActive, isMeasuredFailureGateEnabledFromEnv, isModelReachable, isPromotionsBrainActive, isProviderReachable, judgeMeasuredFailure, loadAliasesFromBrain, loadArchetypePerfFromBrain, loadArchetypePerfNFromBrain, loadChainsFromBrain, loadModelsFromBrain, loadPricingFromBrain, markAdvisoryResolved, markExclusionFindingHandled, markPromoteReadyHandled, parseJudgeVerdict, peekBrainDeadLetter, probeShadow, profileToRow, readBrainReadEnv, record, recordGoldenIr, recordOutcome, recordShadowProbe, renderIrForJudge, resetTokenizer, resolveConventionsForProfile, resolvePricingAt, resolveProviderKey, rubricFor, runAdvisor, runGoldenEval, setTokenizer, wilsonLowerBound };
|
package/dist/index.js
CHANGED
|
@@ -27,6 +27,7 @@ __export(index_exports, {
|
|
|
27
27
|
ARCHETYPE_FLOOR_DEFAULT: () => ARCHETYPE_FLOOR_DEFAULT,
|
|
28
28
|
CallError: () => CallError,
|
|
29
29
|
DEFAULT_FINDINGS_ENDPOINT: () => DEFAULT_FINDINGS_ENDPOINT,
|
|
30
|
+
DEFAULT_MEASURED_FAILURE_ENDPOINT: () => DEFAULT_MEASURED_FAILURE_ENDPOINT,
|
|
30
31
|
DEFAULT_PROMOTIONS_ENDPOINT: () => DEFAULT_PROMOTIONS_ENDPOINT,
|
|
31
32
|
DIALECT_VERSION: () => DIALECT_VERSION,
|
|
32
33
|
FamilyResolutionError: () => FamilyResolutionError,
|
|
@@ -34,12 +35,15 @@ __export(index_exports, {
|
|
|
34
35
|
JUDGE_RUBRICS: () => JUDGE_RUBRICS,
|
|
35
36
|
LATENCY_TIER_MS: () => LATENCY_TIER_MS,
|
|
36
37
|
LIBRARY_VERSION: () => LIBRARY_VERSION,
|
|
38
|
+
MEASURED_FAILURE_CFG: () => MEASURED_FAILURE_CFG,
|
|
37
39
|
MEASURED_GROUNDING_MIN_N: () => MEASURED_GROUNDING_MIN_N,
|
|
38
40
|
PRODUCER_OWNED_RULE_CODES: () => PRODUCER_OWNED_RULE_CODES,
|
|
39
41
|
PROVIDER_ENV_KEYS: () => PROVIDER_ENV_KEYS,
|
|
40
42
|
RULE_SEQUENTIAL_TOOL_CLIFF: () => RULE_SEQUENTIAL_TOOL_CLIFF,
|
|
41
43
|
TRANSLATOR_FLOOR: () => TRANSLATOR_FLOOR,
|
|
44
|
+
_testResetMeasuredFailure: () => _testResetMeasuredFailure,
|
|
42
45
|
_testResetPromotions: () => _testResetPromotions,
|
|
46
|
+
_testWaitForMeasuredFailureRefresh: () => _testWaitForMeasuredFailureRefresh,
|
|
43
47
|
_testWaitForPromotionsRefresh: () => _testWaitForPromotionsRefresh,
|
|
44
48
|
allProfiles: () => allProfiles,
|
|
45
49
|
applyArchetypeConvention: () => applyArchetypeConvention,
|
|
@@ -60,6 +64,7 @@ __export(index_exports, {
|
|
|
60
64
|
compile: () => compile2,
|
|
61
65
|
compileForAISDKv6: () => compileForAISDKv6,
|
|
62
66
|
configureBrain: () => configureBrain,
|
|
67
|
+
configureMeasuredFailureBrain: () => configureMeasuredFailureBrain,
|
|
63
68
|
configurePromotionsBrain: () => configurePromotionsBrain,
|
|
64
69
|
countTokens: () => countTokens,
|
|
65
70
|
createBrainForwardRoutes: () => createBrainForwardRoutes,
|
|
@@ -76,6 +81,7 @@ __export(index_exports, {
|
|
|
76
81
|
getArchetypePerfScore: () => getArchetypePerfScore,
|
|
77
82
|
getDefaultFallbackChain: () => getDefaultFallbackChain,
|
|
78
83
|
getDefaultFallbackChainWithGrounding: () => getDefaultFallbackChainWithGrounding,
|
|
84
|
+
getMeasuredFailureVerdict: () => getMeasuredFailureVerdict,
|
|
79
85
|
getModelCompatibility: () => getModelCompatibility,
|
|
80
86
|
getPerAxisMetrics: () => getPerAxisMetrics,
|
|
81
87
|
getProfile: () => getProfile,
|
|
@@ -92,9 +98,12 @@ __export(index_exports, {
|
|
|
92
98
|
isBrainQueryActiveFor: () => isBrainQueryActiveFor,
|
|
93
99
|
isBrainSync: () => isBrainSync,
|
|
94
100
|
isExclusionFindingsBrainActive: () => isExclusionFindingsBrainActive,
|
|
101
|
+
isMeasuredFailureBrainActive: () => isMeasuredFailureBrainActive,
|
|
102
|
+
isMeasuredFailureGateEnabledFromEnv: () => isMeasuredFailureGateEnabledFromEnv,
|
|
95
103
|
isModelReachable: () => isModelReachable,
|
|
96
104
|
isPromotionsBrainActive: () => isPromotionsBrainActive,
|
|
97
105
|
isProviderReachable: () => isProviderReachable,
|
|
106
|
+
judgeMeasuredFailure: () => judgeMeasuredFailure,
|
|
98
107
|
latencyTierOf: () => latencyTierOf,
|
|
99
108
|
learningKey: () => learningKey,
|
|
100
109
|
loadAliasesFromBrain: () => loadAliasesFromBrain,
|
|
@@ -128,7 +137,8 @@ __export(index_exports, {
|
|
|
128
137
|
runGoldenEval: () => runGoldenEval,
|
|
129
138
|
setTokenizer: () => setTokenizer,
|
|
130
139
|
shouldCaptureGolden: () => shouldCaptureGolden,
|
|
131
|
-
tryGetProfile: () => tryGetProfile
|
|
140
|
+
tryGetProfile: () => tryGetProfile,
|
|
141
|
+
wilsonLowerBound: () => wilsonLowerBound
|
|
132
142
|
});
|
|
133
143
|
module.exports = __toCommonJS(index_exports);
|
|
134
144
|
|
|
@@ -2849,11 +2859,14 @@ function passScoreTargets(ir, opts) {
|
|
|
2849
2859
|
);
|
|
2850
2860
|
if (schemaWeak) qualityGatePenalty = QUALITY_GATE_PENALTY;
|
|
2851
2861
|
}
|
|
2862
|
+
const measuredGate = opts.measuredFailureGates?.get(modelId);
|
|
2863
|
+
if (measuredGate) qualityGatePenalty = QUALITY_GATE_PENALTY;
|
|
2852
2864
|
const isPromoted = promotion?.promotedModel === modelId;
|
|
2853
|
-
if (isPromoted && promotion.suppressQualityGate) {
|
|
2865
|
+
if (isPromoted && promotion.suppressQualityGate && !measuredGate) {
|
|
2854
2866
|
qualityGatePenalty = 0;
|
|
2855
2867
|
}
|
|
2856
|
-
const
|
|
2868
|
+
const promotionNeutralized = isPromoted && !!measuredGate;
|
|
2869
|
+
const promotionBoost = isPromoted && !measuredGate ? PROMOTION_BOOST : 0;
|
|
2857
2870
|
const rank = qualityScore + callerOrderBoost - costPenalty - reasons.length * 10 + preferredBoost - latencyPenalty - qualityGatePenalty + promotionBoost;
|
|
2858
2871
|
scores.push({
|
|
2859
2872
|
modelId,
|
|
@@ -2897,14 +2910,31 @@ function passScoreTargets(ir, opts) {
|
|
|
2897
2910
|
});
|
|
2898
2911
|
}
|
|
2899
2912
|
if (qualityGatePenalty > 0) {
|
|
2913
|
+
if (measuredGate) {
|
|
2914
|
+
const pct = (x) => `${(x * 100).toFixed(0)}%`;
|
|
2915
|
+
policyMutations.push({
|
|
2916
|
+
id: `quality-gate-measured-${modelId}`,
|
|
2917
|
+
source: "quality_gate",
|
|
2918
|
+
passName: "score_targets",
|
|
2919
|
+
description: `Model ${modelId} gated below the quality floor for archetype '${ir.intent.archetype}' by MEASURED evidence from this app's own outcomes \u2014 ${measuredGate.nFail} of ${measuredGate.n} attempts failed on the quality axis in the trailing window (${pct(measuredGate.rate)}; 95% lower bound ${pct(measuredGate.lowerBound)} > 50%). Down-ranked out of leadership; retained as graceful fallback only. The gate is derived, not stored \u2014 it lifts on its own once the failures age out of the window.`
|
|
2920
|
+
});
|
|
2921
|
+
} else {
|
|
2922
|
+
policyMutations.push({
|
|
2923
|
+
id: `quality-gate-structured-${modelId}`,
|
|
2924
|
+
source: "quality_gate",
|
|
2925
|
+
passName: "score_targets",
|
|
2926
|
+
description: `Model ${modelId} gated below the quality floor for archetype '${ir.intent.archetype}' \u2014 declared structuredOutput + kgauto convention flags it schema-weak for this contract (structuredOutputHint:'avoid'). Down-ranked out of leadership; retained as graceful fallback only.`
|
|
2927
|
+
});
|
|
2928
|
+
}
|
|
2929
|
+
}
|
|
2930
|
+
if (promotionNeutralized && reasons.length === 0) {
|
|
2900
2931
|
policyMutations.push({
|
|
2901
|
-
id: `
|
|
2902
|
-
source: "
|
|
2932
|
+
id: `promotion-neutralized-${modelId}`,
|
|
2933
|
+
source: "surface_promotion",
|
|
2903
2934
|
passName: "score_targets",
|
|
2904
|
-
description: `
|
|
2935
|
+
description: `Active promotion #${promotion.id} for ${modelId} NOT applied \u2014 the surface's own measured evidence contradicts it (${measuredGate.nFail}/${measuredGate.n} attempts failed in the trailing window). A promotion rests on an eval at promote time; this rests on what the surface is doing now, and the fresher evidence wins. The 7-day rollback guard is expected to retire this promotion on its next run.`
|
|
2905
2936
|
});
|
|
2906
|
-
}
|
|
2907
|
-
if (isPromoted && reasons.length === 0) {
|
|
2937
|
+
} else if (isPromoted && reasons.length === 0) {
|
|
2908
2938
|
policyMutations.push({
|
|
2909
2939
|
id: `promotion-applied-${modelId}`,
|
|
2910
2940
|
source: "surface_promotion",
|
|
@@ -4538,6 +4568,171 @@ async function _testWaitForPromotionsRefresh() {
|
|
|
4538
4568
|
if (pending.length > 0) await Promise.all(pending);
|
|
4539
4569
|
}
|
|
4540
4570
|
|
|
4571
|
+
// src/measured-failure-brain.ts
|
|
4572
|
+
function coerceCount(v) {
|
|
4573
|
+
if (typeof v === "number") return Number.isFinite(v) ? v : null;
|
|
4574
|
+
if (typeof v === "string") {
|
|
4575
|
+
const n = Number(v);
|
|
4576
|
+
return Number.isFinite(n) ? n : null;
|
|
4577
|
+
}
|
|
4578
|
+
return null;
|
|
4579
|
+
}
|
|
4580
|
+
function isRawFailureRow(x) {
|
|
4581
|
+
if (!x || typeof x !== "object") return false;
|
|
4582
|
+
const r = x;
|
|
4583
|
+
return typeof r.intent_archetype === "string" && typeof r.model === "string" && (typeof r.n === "number" || typeof r.n === "string");
|
|
4584
|
+
}
|
|
4585
|
+
function mapRows(rows) {
|
|
4586
|
+
const out = [];
|
|
4587
|
+
for (const row of rows) {
|
|
4588
|
+
if (!isRawFailureRow(row)) continue;
|
|
4589
|
+
const n = coerceCount(row.n);
|
|
4590
|
+
const nFail = coerceCount(row.n_fail) ?? 0;
|
|
4591
|
+
if (n === null || n <= 0) continue;
|
|
4592
|
+
out.push({
|
|
4593
|
+
archetype: row.intent_archetype,
|
|
4594
|
+
model: row.model,
|
|
4595
|
+
n,
|
|
4596
|
+
nFail
|
|
4597
|
+
});
|
|
4598
|
+
}
|
|
4599
|
+
return out;
|
|
4600
|
+
}
|
|
4601
|
+
var MEASURED_FAILURE_CFG = {
|
|
4602
|
+
/**
|
|
4603
|
+
* Hard minimum attempts before ANY gate may be created. Guards against
|
|
4604
|
+
* pathological tiny samples that the confidence bound alone would let
|
|
4605
|
+
* through in edge cases. At 5-for-5 the bound clears the threshold; at
|
|
4606
|
+
* 3-for-3 it does not, which is the behaviour we want (three failures is
|
|
4607
|
+
* a bad day, five in a row is a pattern).
|
|
4608
|
+
*/
|
|
4609
|
+
minSample: 5,
|
|
4610
|
+
/**
|
|
4611
|
+
* Gate when we are 95% confident the model fails MORE OFTEN THAN IT
|
|
4612
|
+
* SUCCEEDS on this surface. Deliberately unarguable rather than tuned —
|
|
4613
|
+
* a model that probably fails the majority of the time has no business
|
|
4614
|
+
* leading a surface, whatever its declared scores say.
|
|
4615
|
+
*/
|
|
4616
|
+
lowerBoundThreshold: 0.5,
|
|
4617
|
+
/** 95% one-sided-ish confidence (standard two-sided z at α=0.05). */
|
|
4618
|
+
z: 1.96,
|
|
4619
|
+
/** Must match the view's window. Documented here for the advisory text. */
|
|
4620
|
+
windowDays: 28
|
|
4621
|
+
};
|
|
4622
|
+
function wilsonLowerBound(failures, n, z = MEASURED_FAILURE_CFG.z) {
|
|
4623
|
+
if (n <= 0) return 0;
|
|
4624
|
+
const p = failures / n;
|
|
4625
|
+
const z2 = z * z;
|
|
4626
|
+
const denom = 1 + z2 / n;
|
|
4627
|
+
const centre = p + z2 / (2 * n);
|
|
4628
|
+
const margin = z * Math.sqrt(p * (1 - p) / n + z2 / (4 * n * n));
|
|
4629
|
+
const lower2 = (centre - margin) / denom;
|
|
4630
|
+
return lower2 < 0 ? 0 : lower2;
|
|
4631
|
+
}
|
|
4632
|
+
function judgeMeasuredFailure(row, cfg = MEASURED_FAILURE_CFG) {
|
|
4633
|
+
if (!row || row.n < cfg.minSample) return void 0;
|
|
4634
|
+
const lowerBound = wilsonLowerBound(row.nFail, row.n, cfg.z);
|
|
4635
|
+
return {
|
|
4636
|
+
gated: lowerBound > cfg.lowerBoundThreshold,
|
|
4637
|
+
rate: row.nFail / row.n,
|
|
4638
|
+
lowerBound,
|
|
4639
|
+
n: row.n,
|
|
4640
|
+
nFail: row.nFail
|
|
4641
|
+
};
|
|
4642
|
+
}
|
|
4643
|
+
var snapshots5 = /* @__PURE__ */ new Map();
|
|
4644
|
+
var runtime6;
|
|
4645
|
+
var warnedOnce5 = false;
|
|
4646
|
+
var DEFAULT_MEASURED_FAILURE_ENDPOINT = "https://kgauto-dashboard.vercel.app/api/kgauto-v2/measured-failure";
|
|
4647
|
+
function isMeasuredFailureGateEnabledFromEnv(envSource) {
|
|
4648
|
+
const env = envSource ?? (typeof process !== "undefined" && process.env ? process.env : {});
|
|
4649
|
+
const raw = (env.KGAUTO_MEASURED_FAILURE_GATE ?? "").trim().toLowerCase();
|
|
4650
|
+
return !(raw === "0" || raw === "false");
|
|
4651
|
+
}
|
|
4652
|
+
function configureMeasuredFailureBrain(rt) {
|
|
4653
|
+
runtime6 = rt;
|
|
4654
|
+
snapshots5.clear();
|
|
4655
|
+
warnedOnce5 = false;
|
|
4656
|
+
}
|
|
4657
|
+
function isMeasuredFailureBrainActive() {
|
|
4658
|
+
return runtime6 !== void 0;
|
|
4659
|
+
}
|
|
4660
|
+
function getMeasuredFailureVerdict(opts) {
|
|
4661
|
+
const rt = runtime6;
|
|
4662
|
+
if (!rt) return void 0;
|
|
4663
|
+
const { appId, archetype, model } = opts;
|
|
4664
|
+
if (!appId || !archetype || !model) return void 0;
|
|
4665
|
+
let snap = snapshots5.get(appId);
|
|
4666
|
+
if (!snap) {
|
|
4667
|
+
snap = { data: [], expiresAt: 0, refreshing: false };
|
|
4668
|
+
snapshots5.set(appId, snap);
|
|
4669
|
+
}
|
|
4670
|
+
const now = Date.now();
|
|
4671
|
+
if (snap.expiresAt <= now && !snap.refreshing) {
|
|
4672
|
+
snap.refreshing = true;
|
|
4673
|
+
void asyncRefresh6(rt, appId);
|
|
4674
|
+
}
|
|
4675
|
+
const row = snap.data.find(
|
|
4676
|
+
(r) => r.archetype === archetype && r.model === model
|
|
4677
|
+
);
|
|
4678
|
+
return judgeMeasuredFailure(row);
|
|
4679
|
+
}
|
|
4680
|
+
var pendingRefreshes5 = /* @__PURE__ */ new Map();
|
|
4681
|
+
async function asyncRefresh6(rt, appId) {
|
|
4682
|
+
const promise = doRefresh6(rt, appId);
|
|
4683
|
+
pendingRefreshes5.set(appId, promise);
|
|
4684
|
+
try {
|
|
4685
|
+
await promise;
|
|
4686
|
+
} finally {
|
|
4687
|
+
if (pendingRefreshes5.get(appId) === promise) {
|
|
4688
|
+
pendingRefreshes5.delete(appId);
|
|
4689
|
+
}
|
|
4690
|
+
}
|
|
4691
|
+
}
|
|
4692
|
+
async function doRefresh6(rt, appId) {
|
|
4693
|
+
const url = `${rt.endpoint}?app_id=${encodeURIComponent(appId)}`;
|
|
4694
|
+
let snap = snapshots5.get(appId);
|
|
4695
|
+
if (!snap) {
|
|
4696
|
+
snap = { data: [], expiresAt: 0, refreshing: false };
|
|
4697
|
+
snapshots5.set(appId, snap);
|
|
4698
|
+
}
|
|
4699
|
+
try {
|
|
4700
|
+
const res = await rt.fetchImpl(url, { method: "GET" });
|
|
4701
|
+
if (!res.ok) {
|
|
4702
|
+
throw new Error(`measured-failure ${res.status}: ${res.statusText}`);
|
|
4703
|
+
}
|
|
4704
|
+
const body = await res.json();
|
|
4705
|
+
if (runtime6 !== rt) return;
|
|
4706
|
+
snap.data = Array.isArray(body) ? mapRows(body) : [];
|
|
4707
|
+
snap.expiresAt = Date.now() + rt.ttlMs;
|
|
4708
|
+
snap.refreshing = false;
|
|
4709
|
+
} catch (err) {
|
|
4710
|
+
if (runtime6 !== rt) return;
|
|
4711
|
+
snap.refreshing = false;
|
|
4712
|
+
snap.expiresAt = Date.now() + rt.ttlMs;
|
|
4713
|
+
if (!warnedOnce5) {
|
|
4714
|
+
warnedOnce5 = true;
|
|
4715
|
+
(rt.onError ?? defaultOnError6)(err);
|
|
4716
|
+
}
|
|
4717
|
+
}
|
|
4718
|
+
}
|
|
4719
|
+
function defaultOnError6(err) {
|
|
4720
|
+
console.warn(
|
|
4721
|
+
"[kgauto] measured-failure fetch failed (gate inactive until next refresh):",
|
|
4722
|
+
err
|
|
4723
|
+
);
|
|
4724
|
+
}
|
|
4725
|
+
function _testResetMeasuredFailure() {
|
|
4726
|
+
runtime6 = void 0;
|
|
4727
|
+
snapshots5.clear();
|
|
4728
|
+
pendingRefreshes5 = /* @__PURE__ */ new Map();
|
|
4729
|
+
warnedOnce5 = false;
|
|
4730
|
+
}
|
|
4731
|
+
async function _testWaitForMeasuredFailureRefresh() {
|
|
4732
|
+
const pending = Array.from(pendingRefreshes5.values());
|
|
4733
|
+
if (pending.length > 0) await Promise.all(pending);
|
|
4734
|
+
}
|
|
4735
|
+
|
|
4541
4736
|
// src/compile.ts
|
|
4542
4737
|
var counter = 0;
|
|
4543
4738
|
function makeHandle() {
|
|
@@ -4574,12 +4769,31 @@ function compile(ir, opts = {}) {
|
|
|
4574
4769
|
evalRunId: activePromotion.evalRunId,
|
|
4575
4770
|
suppressQualityGate: activePromotion.suppressQualityGate
|
|
4576
4771
|
} : void 0;
|
|
4772
|
+
const measuredFailureGates = /* @__PURE__ */ new Map();
|
|
4773
|
+
for (const entry of workingIR.models ?? []) {
|
|
4774
|
+
const modelId = typeof entry === "string" ? entry : void 0;
|
|
4775
|
+
if (!modelId) continue;
|
|
4776
|
+
const verdict = getMeasuredFailureVerdict({
|
|
4777
|
+
appId: ir.appId,
|
|
4778
|
+
archetype: ir.intent.archetype,
|
|
4779
|
+
model: modelId
|
|
4780
|
+
});
|
|
4781
|
+
if (verdict?.gated) {
|
|
4782
|
+
measuredFailureGates.set(modelId, {
|
|
4783
|
+
rate: verdict.rate,
|
|
4784
|
+
lowerBound: verdict.lowerBound,
|
|
4785
|
+
n: verdict.n,
|
|
4786
|
+
nFail: verdict.nFail
|
|
4787
|
+
});
|
|
4788
|
+
}
|
|
4789
|
+
}
|
|
4577
4790
|
const inputTokens = estimateInputTokens(workingIR);
|
|
4578
4791
|
const scores = passScoreTargets(workingIR, {
|
|
4579
4792
|
estimatedInputTokens: inputTokens,
|
|
4580
4793
|
profilesById: resolver,
|
|
4581
4794
|
policy: opts.policy,
|
|
4582
|
-
promotion
|
|
4795
|
+
promotion,
|
|
4796
|
+
measuredFailureGates
|
|
4583
4797
|
});
|
|
4584
4798
|
accumulatedMutations.push(...scores.mutations);
|
|
4585
4799
|
const target = pickTarget(workingIR, scores.value);
|
|
@@ -4940,12 +5154,24 @@ function configureBrain(config) {
|
|
|
4940
5154
|
} else {
|
|
4941
5155
|
configurePromotionsBrain(void 0);
|
|
4942
5156
|
}
|
|
5157
|
+
const measuredFailureConsent = config.measuredFailureGate ?? isMeasuredFailureGateEnabledFromEnv();
|
|
5158
|
+
if (measuredFailureConsent && bq.measuredFailure !== false) {
|
|
5159
|
+
configureMeasuredFailureBrain({
|
|
5160
|
+
endpoint: bq.measuredFailureEndpoint ?? DEFAULT_MEASURED_FAILURE_ENDPOINT,
|
|
5161
|
+
ttlMs: bq.cacheTtlMs ?? 3e5,
|
|
5162
|
+
fetchImpl: config.fetchImpl ?? fetch,
|
|
5163
|
+
onError: config.onError
|
|
5164
|
+
});
|
|
5165
|
+
} else {
|
|
5166
|
+
configureMeasuredFailureBrain(void 0);
|
|
5167
|
+
}
|
|
4943
5168
|
}
|
|
4944
5169
|
function clearBrain() {
|
|
4945
5170
|
activeConfig = void 0;
|
|
4946
5171
|
configureBrainQuery(void 0);
|
|
4947
5172
|
configureExclusionFindingsBrain(void 0);
|
|
4948
5173
|
configurePromotionsBrain(void 0);
|
|
5174
|
+
configureMeasuredFailureBrain(void 0);
|
|
4949
5175
|
}
|
|
4950
5176
|
var DEAD_LETTER_MAX_ENTRIES = 50;
|
|
4951
5177
|
var DEAD_LETTER_MAX_ATTEMPTS = 3;
|
|
@@ -5062,7 +5288,7 @@ async function flushBrainDeadLetter() {
|
|
|
5062
5288
|
if (entry.attempts >= DEAD_LETTER_MAX_ATTEMPTS) {
|
|
5063
5289
|
const idx = deadLetter.indexOf(entry);
|
|
5064
5290
|
if (idx !== -1) deadLetter.splice(idx, 1);
|
|
5065
|
-
(config.onError ??
|
|
5291
|
+
(config.onError ?? defaultOnError7)(
|
|
5066
5292
|
new Error(
|
|
5067
5293
|
`brain dead-letter: dropping ${entry.route} payload after ${entry.attempts} attempts \u2014 last error: ${entry.lastError}`
|
|
5068
5294
|
)
|
|
@@ -5220,7 +5446,7 @@ async function record(input) {
|
|
|
5220
5446
|
} catch (err) {
|
|
5221
5447
|
noteFailure(err);
|
|
5222
5448
|
pushDeadLetter("outcomes", config.endpoint, payload, err);
|
|
5223
|
-
(config.onError ??
|
|
5449
|
+
(config.onError ?? defaultOnError7)(err);
|
|
5224
5450
|
return;
|
|
5225
5451
|
}
|
|
5226
5452
|
maybeAutoFlush();
|
|
@@ -5248,7 +5474,7 @@ async function record(input) {
|
|
|
5248
5474
|
} catch (err) {
|
|
5249
5475
|
noteFailure(err);
|
|
5250
5476
|
pushDeadLetter("compile_outcome_advisories", config.endpoint, advisoryPayload, err);
|
|
5251
|
-
(config.onError ??
|
|
5477
|
+
(config.onError ?? defaultOnError7)(err);
|
|
5252
5478
|
}
|
|
5253
5479
|
};
|
|
5254
5480
|
if (config.sync) {
|
|
@@ -5257,7 +5483,7 @@ async function record(input) {
|
|
|
5257
5483
|
void send();
|
|
5258
5484
|
}
|
|
5259
5485
|
}
|
|
5260
|
-
function
|
|
5486
|
+
function defaultOnError7(err) {
|
|
5261
5487
|
console.warn("[kgauto] brain record failed:", err);
|
|
5262
5488
|
}
|
|
5263
5489
|
function describeBrainWriteFailure(status, route, body) {
|
|
@@ -5422,7 +5648,7 @@ async function recordOutcome(input) {
|
|
|
5422
5648
|
} catch (err) {
|
|
5423
5649
|
noteFailure(err);
|
|
5424
5650
|
pushDeadLetter("compile_outcome_quality", config.endpoint, payload, err);
|
|
5425
|
-
(config.onError ??
|
|
5651
|
+
(config.onError ?? defaultOnError7)(err);
|
|
5426
5652
|
return { ok: false, reason: "persistence_failed" };
|
|
5427
5653
|
}
|
|
5428
5654
|
};
|
|
@@ -5493,7 +5719,7 @@ async function recordShadowProbe(input) {
|
|
|
5493
5719
|
} catch (err) {
|
|
5494
5720
|
noteFailure(err);
|
|
5495
5721
|
pushDeadLetter("probe_outcomes", config.endpoint, row, err);
|
|
5496
|
-
(config.onError ??
|
|
5722
|
+
(config.onError ?? defaultOnError7)(err);
|
|
5497
5723
|
}
|
|
5498
5724
|
};
|
|
5499
5725
|
if (config.sync) {
|
|
@@ -5548,7 +5774,7 @@ async function recordGoldenIr(input) {
|
|
|
5548
5774
|
} catch (err) {
|
|
5549
5775
|
noteFailure(err);
|
|
5550
5776
|
pushDeadLetter("golden_irs", config.endpoint, row, err);
|
|
5551
|
-
(config.onError ??
|
|
5777
|
+
(config.onError ?? defaultOnError7)(err);
|
|
5552
5778
|
}
|
|
5553
5779
|
};
|
|
5554
5780
|
if (config.sync) {
|
|
@@ -8376,7 +8602,7 @@ function createBrainForwardRoutes(config) {
|
|
|
8376
8602
|
}
|
|
8377
8603
|
|
|
8378
8604
|
// src/version.ts
|
|
8379
|
-
var LIBRARY_VERSION = "2.0.0-alpha.
|
|
8605
|
+
var LIBRARY_VERSION = "2.0.0-alpha.70";
|
|
8380
8606
|
|
|
8381
8607
|
// src/key-health.ts
|
|
8382
8608
|
var JSON_HEADERS2 = { "Content-Type": "application/json" };
|
|
@@ -9009,6 +9235,7 @@ function compile2(ir, opts) {
|
|
|
9009
9235
|
ARCHETYPE_FLOOR_DEFAULT,
|
|
9010
9236
|
CallError,
|
|
9011
9237
|
DEFAULT_FINDINGS_ENDPOINT,
|
|
9238
|
+
DEFAULT_MEASURED_FAILURE_ENDPOINT,
|
|
9012
9239
|
DEFAULT_PROMOTIONS_ENDPOINT,
|
|
9013
9240
|
DIALECT_VERSION,
|
|
9014
9241
|
FamilyResolutionError,
|
|
@@ -9016,12 +9243,15 @@ function compile2(ir, opts) {
|
|
|
9016
9243
|
JUDGE_RUBRICS,
|
|
9017
9244
|
LATENCY_TIER_MS,
|
|
9018
9245
|
LIBRARY_VERSION,
|
|
9246
|
+
MEASURED_FAILURE_CFG,
|
|
9019
9247
|
MEASURED_GROUNDING_MIN_N,
|
|
9020
9248
|
PRODUCER_OWNED_RULE_CODES,
|
|
9021
9249
|
PROVIDER_ENV_KEYS,
|
|
9022
9250
|
RULE_SEQUENTIAL_TOOL_CLIFF,
|
|
9023
9251
|
TRANSLATOR_FLOOR,
|
|
9252
|
+
_testResetMeasuredFailure,
|
|
9024
9253
|
_testResetPromotions,
|
|
9254
|
+
_testWaitForMeasuredFailureRefresh,
|
|
9025
9255
|
_testWaitForPromotionsRefresh,
|
|
9026
9256
|
allProfiles,
|
|
9027
9257
|
applyArchetypeConvention,
|
|
@@ -9042,6 +9272,7 @@ function compile2(ir, opts) {
|
|
|
9042
9272
|
compile,
|
|
9043
9273
|
compileForAISDKv6,
|
|
9044
9274
|
configureBrain,
|
|
9275
|
+
configureMeasuredFailureBrain,
|
|
9045
9276
|
configurePromotionsBrain,
|
|
9046
9277
|
countTokens,
|
|
9047
9278
|
createBrainForwardRoutes,
|
|
@@ -9058,6 +9289,7 @@ function compile2(ir, opts) {
|
|
|
9058
9289
|
getArchetypePerfScore,
|
|
9059
9290
|
getDefaultFallbackChain,
|
|
9060
9291
|
getDefaultFallbackChainWithGrounding,
|
|
9292
|
+
getMeasuredFailureVerdict,
|
|
9061
9293
|
getModelCompatibility,
|
|
9062
9294
|
getPerAxisMetrics,
|
|
9063
9295
|
getProfile,
|
|
@@ -9074,9 +9306,12 @@ function compile2(ir, opts) {
|
|
|
9074
9306
|
isBrainQueryActiveFor,
|
|
9075
9307
|
isBrainSync,
|
|
9076
9308
|
isExclusionFindingsBrainActive,
|
|
9309
|
+
isMeasuredFailureBrainActive,
|
|
9310
|
+
isMeasuredFailureGateEnabledFromEnv,
|
|
9077
9311
|
isModelReachable,
|
|
9078
9312
|
isPromotionsBrainActive,
|
|
9079
9313
|
isProviderReachable,
|
|
9314
|
+
judgeMeasuredFailure,
|
|
9080
9315
|
latencyTierOf,
|
|
9081
9316
|
learningKey,
|
|
9082
9317
|
loadAliasesFromBrain,
|
|
@@ -9110,5 +9345,6 @@ function compile2(ir, opts) {
|
|
|
9110
9345
|
runGoldenEval,
|
|
9111
9346
|
setTokenizer,
|
|
9112
9347
|
shouldCaptureGolden,
|
|
9113
|
-
tryGetProfile
|
|
9348
|
+
tryGetProfile,
|
|
9349
|
+
wilsonLowerBound
|
|
9114
9350
|
});
|
package/dist/index.mjs
CHANGED
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
import {
|
|
17
17
|
LIBRARY_VERSION,
|
|
18
18
|
createKeyHealthRoute
|
|
19
|
-
} from "./chunk-
|
|
19
|
+
} from "./chunk-FR4DNGLW.mjs";
|
|
20
20
|
import {
|
|
21
21
|
ABSOLUTE_FLOOR,
|
|
22
22
|
ARCHETYPE_FLOOR_DEFAULT,
|
|
@@ -826,11 +826,14 @@ function passScoreTargets(ir, opts) {
|
|
|
826
826
|
);
|
|
827
827
|
if (schemaWeak) qualityGatePenalty = QUALITY_GATE_PENALTY;
|
|
828
828
|
}
|
|
829
|
+
const measuredGate = opts.measuredFailureGates?.get(modelId);
|
|
830
|
+
if (measuredGate) qualityGatePenalty = QUALITY_GATE_PENALTY;
|
|
829
831
|
const isPromoted = promotion?.promotedModel === modelId;
|
|
830
|
-
if (isPromoted && promotion.suppressQualityGate) {
|
|
832
|
+
if (isPromoted && promotion.suppressQualityGate && !measuredGate) {
|
|
831
833
|
qualityGatePenalty = 0;
|
|
832
834
|
}
|
|
833
|
-
const
|
|
835
|
+
const promotionNeutralized = isPromoted && !!measuredGate;
|
|
836
|
+
const promotionBoost = isPromoted && !measuredGate ? PROMOTION_BOOST : 0;
|
|
834
837
|
const rank = qualityScore + callerOrderBoost - costPenalty - reasons.length * 10 + preferredBoost - latencyPenalty - qualityGatePenalty + promotionBoost;
|
|
835
838
|
scores.push({
|
|
836
839
|
modelId,
|
|
@@ -874,14 +877,31 @@ function passScoreTargets(ir, opts) {
|
|
|
874
877
|
});
|
|
875
878
|
}
|
|
876
879
|
if (qualityGatePenalty > 0) {
|
|
880
|
+
if (measuredGate) {
|
|
881
|
+
const pct = (x) => `${(x * 100).toFixed(0)}%`;
|
|
882
|
+
policyMutations.push({
|
|
883
|
+
id: `quality-gate-measured-${modelId}`,
|
|
884
|
+
source: "quality_gate",
|
|
885
|
+
passName: "score_targets",
|
|
886
|
+
description: `Model ${modelId} gated below the quality floor for archetype '${ir.intent.archetype}' by MEASURED evidence from this app's own outcomes \u2014 ${measuredGate.nFail} of ${measuredGate.n} attempts failed on the quality axis in the trailing window (${pct(measuredGate.rate)}; 95% lower bound ${pct(measuredGate.lowerBound)} > 50%). Down-ranked out of leadership; retained as graceful fallback only. The gate is derived, not stored \u2014 it lifts on its own once the failures age out of the window.`
|
|
887
|
+
});
|
|
888
|
+
} else {
|
|
889
|
+
policyMutations.push({
|
|
890
|
+
id: `quality-gate-structured-${modelId}`,
|
|
891
|
+
source: "quality_gate",
|
|
892
|
+
passName: "score_targets",
|
|
893
|
+
description: `Model ${modelId} gated below the quality floor for archetype '${ir.intent.archetype}' \u2014 declared structuredOutput + kgauto convention flags it schema-weak for this contract (structuredOutputHint:'avoid'). Down-ranked out of leadership; retained as graceful fallback only.`
|
|
894
|
+
});
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
if (promotionNeutralized && reasons.length === 0) {
|
|
877
898
|
policyMutations.push({
|
|
878
|
-
id: `
|
|
879
|
-
source: "
|
|
899
|
+
id: `promotion-neutralized-${modelId}`,
|
|
900
|
+
source: "surface_promotion",
|
|
880
901
|
passName: "score_targets",
|
|
881
|
-
description: `
|
|
902
|
+
description: `Active promotion #${promotion.id} for ${modelId} NOT applied \u2014 the surface's own measured evidence contradicts it (${measuredGate.nFail}/${measuredGate.n} attempts failed in the trailing window). A promotion rests on an eval at promote time; this rests on what the surface is doing now, and the fresher evidence wins. The 7-day rollback guard is expected to retire this promotion on its next run.`
|
|
882
903
|
});
|
|
883
|
-
}
|
|
884
|
-
if (isPromoted && reasons.length === 0) {
|
|
904
|
+
} else if (isPromoted && reasons.length === 0) {
|
|
885
905
|
policyMutations.push({
|
|
886
906
|
id: `promotion-applied-${modelId}`,
|
|
887
907
|
source: "surface_promotion",
|
|
@@ -2515,6 +2535,171 @@ async function _testWaitForPromotionsRefresh() {
|
|
|
2515
2535
|
if (pending.length > 0) await Promise.all(pending);
|
|
2516
2536
|
}
|
|
2517
2537
|
|
|
2538
|
+
// src/measured-failure-brain.ts
|
|
2539
|
+
function coerceCount(v) {
|
|
2540
|
+
if (typeof v === "number") return Number.isFinite(v) ? v : null;
|
|
2541
|
+
if (typeof v === "string") {
|
|
2542
|
+
const n = Number(v);
|
|
2543
|
+
return Number.isFinite(n) ? n : null;
|
|
2544
|
+
}
|
|
2545
|
+
return null;
|
|
2546
|
+
}
|
|
2547
|
+
function isRawFailureRow(x) {
|
|
2548
|
+
if (!x || typeof x !== "object") return false;
|
|
2549
|
+
const r = x;
|
|
2550
|
+
return typeof r.intent_archetype === "string" && typeof r.model === "string" && (typeof r.n === "number" || typeof r.n === "string");
|
|
2551
|
+
}
|
|
2552
|
+
function mapRows(rows) {
|
|
2553
|
+
const out = [];
|
|
2554
|
+
for (const row of rows) {
|
|
2555
|
+
if (!isRawFailureRow(row)) continue;
|
|
2556
|
+
const n = coerceCount(row.n);
|
|
2557
|
+
const nFail = coerceCount(row.n_fail) ?? 0;
|
|
2558
|
+
if (n === null || n <= 0) continue;
|
|
2559
|
+
out.push({
|
|
2560
|
+
archetype: row.intent_archetype,
|
|
2561
|
+
model: row.model,
|
|
2562
|
+
n,
|
|
2563
|
+
nFail
|
|
2564
|
+
});
|
|
2565
|
+
}
|
|
2566
|
+
return out;
|
|
2567
|
+
}
|
|
2568
|
+
var MEASURED_FAILURE_CFG = {
|
|
2569
|
+
/**
|
|
2570
|
+
* Hard minimum attempts before ANY gate may be created. Guards against
|
|
2571
|
+
* pathological tiny samples that the confidence bound alone would let
|
|
2572
|
+
* through in edge cases. At 5-for-5 the bound clears the threshold; at
|
|
2573
|
+
* 3-for-3 it does not, which is the behaviour we want (three failures is
|
|
2574
|
+
* a bad day, five in a row is a pattern).
|
|
2575
|
+
*/
|
|
2576
|
+
minSample: 5,
|
|
2577
|
+
/**
|
|
2578
|
+
* Gate when we are 95% confident the model fails MORE OFTEN THAN IT
|
|
2579
|
+
* SUCCEEDS on this surface. Deliberately unarguable rather than tuned —
|
|
2580
|
+
* a model that probably fails the majority of the time has no business
|
|
2581
|
+
* leading a surface, whatever its declared scores say.
|
|
2582
|
+
*/
|
|
2583
|
+
lowerBoundThreshold: 0.5,
|
|
2584
|
+
/** 95% one-sided-ish confidence (standard two-sided z at α=0.05). */
|
|
2585
|
+
z: 1.96,
|
|
2586
|
+
/** Must match the view's window. Documented here for the advisory text. */
|
|
2587
|
+
windowDays: 28
|
|
2588
|
+
};
|
|
2589
|
+
function wilsonLowerBound(failures, n, z = MEASURED_FAILURE_CFG.z) {
|
|
2590
|
+
if (n <= 0) return 0;
|
|
2591
|
+
const p = failures / n;
|
|
2592
|
+
const z2 = z * z;
|
|
2593
|
+
const denom = 1 + z2 / n;
|
|
2594
|
+
const centre = p + z2 / (2 * n);
|
|
2595
|
+
const margin = z * Math.sqrt(p * (1 - p) / n + z2 / (4 * n * n));
|
|
2596
|
+
const lower2 = (centre - margin) / denom;
|
|
2597
|
+
return lower2 < 0 ? 0 : lower2;
|
|
2598
|
+
}
|
|
2599
|
+
function judgeMeasuredFailure(row, cfg = MEASURED_FAILURE_CFG) {
|
|
2600
|
+
if (!row || row.n < cfg.minSample) return void 0;
|
|
2601
|
+
const lowerBound = wilsonLowerBound(row.nFail, row.n, cfg.z);
|
|
2602
|
+
return {
|
|
2603
|
+
gated: lowerBound > cfg.lowerBoundThreshold,
|
|
2604
|
+
rate: row.nFail / row.n,
|
|
2605
|
+
lowerBound,
|
|
2606
|
+
n: row.n,
|
|
2607
|
+
nFail: row.nFail
|
|
2608
|
+
};
|
|
2609
|
+
}
|
|
2610
|
+
var snapshots5 = /* @__PURE__ */ new Map();
|
|
2611
|
+
var runtime5;
|
|
2612
|
+
var warnedOnce5 = false;
|
|
2613
|
+
var DEFAULT_MEASURED_FAILURE_ENDPOINT = "https://kgauto-dashboard.vercel.app/api/kgauto-v2/measured-failure";
|
|
2614
|
+
function isMeasuredFailureGateEnabledFromEnv(envSource) {
|
|
2615
|
+
const env = envSource ?? (typeof process !== "undefined" && process.env ? process.env : {});
|
|
2616
|
+
const raw = (env.KGAUTO_MEASURED_FAILURE_GATE ?? "").trim().toLowerCase();
|
|
2617
|
+
return !(raw === "0" || raw === "false");
|
|
2618
|
+
}
|
|
2619
|
+
function configureMeasuredFailureBrain(rt) {
|
|
2620
|
+
runtime5 = rt;
|
|
2621
|
+
snapshots5.clear();
|
|
2622
|
+
warnedOnce5 = false;
|
|
2623
|
+
}
|
|
2624
|
+
function isMeasuredFailureBrainActive() {
|
|
2625
|
+
return runtime5 !== void 0;
|
|
2626
|
+
}
|
|
2627
|
+
function getMeasuredFailureVerdict(opts) {
|
|
2628
|
+
const rt = runtime5;
|
|
2629
|
+
if (!rt) return void 0;
|
|
2630
|
+
const { appId, archetype, model } = opts;
|
|
2631
|
+
if (!appId || !archetype || !model) return void 0;
|
|
2632
|
+
let snap = snapshots5.get(appId);
|
|
2633
|
+
if (!snap) {
|
|
2634
|
+
snap = { data: [], expiresAt: 0, refreshing: false };
|
|
2635
|
+
snapshots5.set(appId, snap);
|
|
2636
|
+
}
|
|
2637
|
+
const now = Date.now();
|
|
2638
|
+
if (snap.expiresAt <= now && !snap.refreshing) {
|
|
2639
|
+
snap.refreshing = true;
|
|
2640
|
+
void asyncRefresh5(rt, appId);
|
|
2641
|
+
}
|
|
2642
|
+
const row = snap.data.find(
|
|
2643
|
+
(r) => r.archetype === archetype && r.model === model
|
|
2644
|
+
);
|
|
2645
|
+
return judgeMeasuredFailure(row);
|
|
2646
|
+
}
|
|
2647
|
+
var pendingRefreshes5 = /* @__PURE__ */ new Map();
|
|
2648
|
+
async function asyncRefresh5(rt, appId) {
|
|
2649
|
+
const promise = doRefresh5(rt, appId);
|
|
2650
|
+
pendingRefreshes5.set(appId, promise);
|
|
2651
|
+
try {
|
|
2652
|
+
await promise;
|
|
2653
|
+
} finally {
|
|
2654
|
+
if (pendingRefreshes5.get(appId) === promise) {
|
|
2655
|
+
pendingRefreshes5.delete(appId);
|
|
2656
|
+
}
|
|
2657
|
+
}
|
|
2658
|
+
}
|
|
2659
|
+
async function doRefresh5(rt, appId) {
|
|
2660
|
+
const url = `${rt.endpoint}?app_id=${encodeURIComponent(appId)}`;
|
|
2661
|
+
let snap = snapshots5.get(appId);
|
|
2662
|
+
if (!snap) {
|
|
2663
|
+
snap = { data: [], expiresAt: 0, refreshing: false };
|
|
2664
|
+
snapshots5.set(appId, snap);
|
|
2665
|
+
}
|
|
2666
|
+
try {
|
|
2667
|
+
const res = await rt.fetchImpl(url, { method: "GET" });
|
|
2668
|
+
if (!res.ok) {
|
|
2669
|
+
throw new Error(`measured-failure ${res.status}: ${res.statusText}`);
|
|
2670
|
+
}
|
|
2671
|
+
const body = await res.json();
|
|
2672
|
+
if (runtime5 !== rt) return;
|
|
2673
|
+
snap.data = Array.isArray(body) ? mapRows(body) : [];
|
|
2674
|
+
snap.expiresAt = Date.now() + rt.ttlMs;
|
|
2675
|
+
snap.refreshing = false;
|
|
2676
|
+
} catch (err) {
|
|
2677
|
+
if (runtime5 !== rt) return;
|
|
2678
|
+
snap.refreshing = false;
|
|
2679
|
+
snap.expiresAt = Date.now() + rt.ttlMs;
|
|
2680
|
+
if (!warnedOnce5) {
|
|
2681
|
+
warnedOnce5 = true;
|
|
2682
|
+
(rt.onError ?? defaultOnError5)(err);
|
|
2683
|
+
}
|
|
2684
|
+
}
|
|
2685
|
+
}
|
|
2686
|
+
function defaultOnError5(err) {
|
|
2687
|
+
console.warn(
|
|
2688
|
+
"[kgauto] measured-failure fetch failed (gate inactive until next refresh):",
|
|
2689
|
+
err
|
|
2690
|
+
);
|
|
2691
|
+
}
|
|
2692
|
+
function _testResetMeasuredFailure() {
|
|
2693
|
+
runtime5 = void 0;
|
|
2694
|
+
snapshots5.clear();
|
|
2695
|
+
pendingRefreshes5 = /* @__PURE__ */ new Map();
|
|
2696
|
+
warnedOnce5 = false;
|
|
2697
|
+
}
|
|
2698
|
+
async function _testWaitForMeasuredFailureRefresh() {
|
|
2699
|
+
const pending = Array.from(pendingRefreshes5.values());
|
|
2700
|
+
if (pending.length > 0) await Promise.all(pending);
|
|
2701
|
+
}
|
|
2702
|
+
|
|
2518
2703
|
// src/compile.ts
|
|
2519
2704
|
var counter = 0;
|
|
2520
2705
|
function makeHandle() {
|
|
@@ -2551,12 +2736,31 @@ function compile(ir, opts = {}) {
|
|
|
2551
2736
|
evalRunId: activePromotion.evalRunId,
|
|
2552
2737
|
suppressQualityGate: activePromotion.suppressQualityGate
|
|
2553
2738
|
} : void 0;
|
|
2739
|
+
const measuredFailureGates = /* @__PURE__ */ new Map();
|
|
2740
|
+
for (const entry of workingIR.models ?? []) {
|
|
2741
|
+
const modelId = typeof entry === "string" ? entry : void 0;
|
|
2742
|
+
if (!modelId) continue;
|
|
2743
|
+
const verdict = getMeasuredFailureVerdict({
|
|
2744
|
+
appId: ir.appId,
|
|
2745
|
+
archetype: ir.intent.archetype,
|
|
2746
|
+
model: modelId
|
|
2747
|
+
});
|
|
2748
|
+
if (verdict?.gated) {
|
|
2749
|
+
measuredFailureGates.set(modelId, {
|
|
2750
|
+
rate: verdict.rate,
|
|
2751
|
+
lowerBound: verdict.lowerBound,
|
|
2752
|
+
n: verdict.n,
|
|
2753
|
+
nFail: verdict.nFail
|
|
2754
|
+
});
|
|
2755
|
+
}
|
|
2756
|
+
}
|
|
2554
2757
|
const inputTokens = estimateInputTokens(workingIR);
|
|
2555
2758
|
const scores = passScoreTargets(workingIR, {
|
|
2556
2759
|
estimatedInputTokens: inputTokens,
|
|
2557
2760
|
profilesById: resolver,
|
|
2558
2761
|
policy: opts.policy,
|
|
2559
|
-
promotion
|
|
2762
|
+
promotion,
|
|
2763
|
+
measuredFailureGates
|
|
2560
2764
|
});
|
|
2561
2765
|
accumulatedMutations.push(...scores.mutations);
|
|
2562
2766
|
const target = pickTarget(workingIR, scores.value);
|
|
@@ -2917,12 +3121,24 @@ function configureBrain(config) {
|
|
|
2917
3121
|
} else {
|
|
2918
3122
|
configurePromotionsBrain(void 0);
|
|
2919
3123
|
}
|
|
3124
|
+
const measuredFailureConsent = config.measuredFailureGate ?? isMeasuredFailureGateEnabledFromEnv();
|
|
3125
|
+
if (measuredFailureConsent && bq.measuredFailure !== false) {
|
|
3126
|
+
configureMeasuredFailureBrain({
|
|
3127
|
+
endpoint: bq.measuredFailureEndpoint ?? DEFAULT_MEASURED_FAILURE_ENDPOINT,
|
|
3128
|
+
ttlMs: bq.cacheTtlMs ?? 3e5,
|
|
3129
|
+
fetchImpl: config.fetchImpl ?? fetch,
|
|
3130
|
+
onError: config.onError
|
|
3131
|
+
});
|
|
3132
|
+
} else {
|
|
3133
|
+
configureMeasuredFailureBrain(void 0);
|
|
3134
|
+
}
|
|
2920
3135
|
}
|
|
2921
3136
|
function clearBrain() {
|
|
2922
3137
|
activeConfig = void 0;
|
|
2923
3138
|
configureBrainQuery(void 0);
|
|
2924
3139
|
configureExclusionFindingsBrain(void 0);
|
|
2925
3140
|
configurePromotionsBrain(void 0);
|
|
3141
|
+
configureMeasuredFailureBrain(void 0);
|
|
2926
3142
|
}
|
|
2927
3143
|
var DEAD_LETTER_MAX_ENTRIES = 50;
|
|
2928
3144
|
var DEAD_LETTER_MAX_ATTEMPTS = 3;
|
|
@@ -3039,7 +3255,7 @@ async function flushBrainDeadLetter() {
|
|
|
3039
3255
|
if (entry.attempts >= DEAD_LETTER_MAX_ATTEMPTS) {
|
|
3040
3256
|
const idx = deadLetter.indexOf(entry);
|
|
3041
3257
|
if (idx !== -1) deadLetter.splice(idx, 1);
|
|
3042
|
-
(config.onError ??
|
|
3258
|
+
(config.onError ?? defaultOnError6)(
|
|
3043
3259
|
new Error(
|
|
3044
3260
|
`brain dead-letter: dropping ${entry.route} payload after ${entry.attempts} attempts \u2014 last error: ${entry.lastError}`
|
|
3045
3261
|
)
|
|
@@ -3197,7 +3413,7 @@ async function record(input) {
|
|
|
3197
3413
|
} catch (err) {
|
|
3198
3414
|
noteFailure(err);
|
|
3199
3415
|
pushDeadLetter("outcomes", config.endpoint, payload, err);
|
|
3200
|
-
(config.onError ??
|
|
3416
|
+
(config.onError ?? defaultOnError6)(err);
|
|
3201
3417
|
return;
|
|
3202
3418
|
}
|
|
3203
3419
|
maybeAutoFlush();
|
|
@@ -3225,7 +3441,7 @@ async function record(input) {
|
|
|
3225
3441
|
} catch (err) {
|
|
3226
3442
|
noteFailure(err);
|
|
3227
3443
|
pushDeadLetter("compile_outcome_advisories", config.endpoint, advisoryPayload, err);
|
|
3228
|
-
(config.onError ??
|
|
3444
|
+
(config.onError ?? defaultOnError6)(err);
|
|
3229
3445
|
}
|
|
3230
3446
|
};
|
|
3231
3447
|
if (config.sync) {
|
|
@@ -3234,7 +3450,7 @@ async function record(input) {
|
|
|
3234
3450
|
void send();
|
|
3235
3451
|
}
|
|
3236
3452
|
}
|
|
3237
|
-
function
|
|
3453
|
+
function defaultOnError6(err) {
|
|
3238
3454
|
console.warn("[kgauto] brain record failed:", err);
|
|
3239
3455
|
}
|
|
3240
3456
|
function describeBrainWriteFailure(status, route, body) {
|
|
@@ -3399,7 +3615,7 @@ async function recordOutcome(input) {
|
|
|
3399
3615
|
} catch (err) {
|
|
3400
3616
|
noteFailure(err);
|
|
3401
3617
|
pushDeadLetter("compile_outcome_quality", config.endpoint, payload, err);
|
|
3402
|
-
(config.onError ??
|
|
3618
|
+
(config.onError ?? defaultOnError6)(err);
|
|
3403
3619
|
return { ok: false, reason: "persistence_failed" };
|
|
3404
3620
|
}
|
|
3405
3621
|
};
|
|
@@ -3470,7 +3686,7 @@ async function recordShadowProbe(input) {
|
|
|
3470
3686
|
} catch (err) {
|
|
3471
3687
|
noteFailure(err);
|
|
3472
3688
|
pushDeadLetter("probe_outcomes", config.endpoint, row, err);
|
|
3473
|
-
(config.onError ??
|
|
3689
|
+
(config.onError ?? defaultOnError6)(err);
|
|
3474
3690
|
}
|
|
3475
3691
|
};
|
|
3476
3692
|
if (config.sync) {
|
|
@@ -3525,7 +3741,7 @@ async function recordGoldenIr(input) {
|
|
|
3525
3741
|
} catch (err) {
|
|
3526
3742
|
noteFailure(err);
|
|
3527
3743
|
pushDeadLetter("golden_irs", config.endpoint, row, err);
|
|
3528
|
-
(config.onError ??
|
|
3744
|
+
(config.onError ?? defaultOnError6)(err);
|
|
3529
3745
|
}
|
|
3530
3746
|
};
|
|
3531
3747
|
if (config.sync) {
|
|
@@ -5881,6 +6097,7 @@ export {
|
|
|
5881
6097
|
ARCHETYPE_FLOOR_DEFAULT,
|
|
5882
6098
|
CallError,
|
|
5883
6099
|
DEFAULT_FINDINGS_ENDPOINT,
|
|
6100
|
+
DEFAULT_MEASURED_FAILURE_ENDPOINT,
|
|
5884
6101
|
DEFAULT_PROMOTIONS_ENDPOINT,
|
|
5885
6102
|
DIALECT_VERSION,
|
|
5886
6103
|
FamilyResolutionError,
|
|
@@ -5888,12 +6105,15 @@ export {
|
|
|
5888
6105
|
JUDGE_RUBRICS,
|
|
5889
6106
|
LATENCY_TIER_MS,
|
|
5890
6107
|
LIBRARY_VERSION,
|
|
6108
|
+
MEASURED_FAILURE_CFG,
|
|
5891
6109
|
MEASURED_GROUNDING_MIN_N,
|
|
5892
6110
|
PRODUCER_OWNED_RULE_CODES,
|
|
5893
6111
|
PROVIDER_ENV_KEYS,
|
|
5894
6112
|
RULE_SEQUENTIAL_TOOL_CLIFF,
|
|
5895
6113
|
TRANSLATOR_FLOOR,
|
|
6114
|
+
_testResetMeasuredFailure,
|
|
5896
6115
|
_testResetPromotions,
|
|
6116
|
+
_testWaitForMeasuredFailureRefresh,
|
|
5897
6117
|
_testWaitForPromotionsRefresh,
|
|
5898
6118
|
allProfiles,
|
|
5899
6119
|
applyArchetypeConvention,
|
|
@@ -5914,6 +6134,7 @@ export {
|
|
|
5914
6134
|
compile2 as compile,
|
|
5915
6135
|
compileForAISDKv6,
|
|
5916
6136
|
configureBrain,
|
|
6137
|
+
configureMeasuredFailureBrain,
|
|
5917
6138
|
configurePromotionsBrain,
|
|
5918
6139
|
countTokens,
|
|
5919
6140
|
createBrainForwardRoutes,
|
|
@@ -5930,6 +6151,7 @@ export {
|
|
|
5930
6151
|
getArchetypePerfScore,
|
|
5931
6152
|
getDefaultFallbackChain,
|
|
5932
6153
|
getDefaultFallbackChainWithGrounding,
|
|
6154
|
+
getMeasuredFailureVerdict,
|
|
5933
6155
|
getModelCompatibility,
|
|
5934
6156
|
getPerAxisMetrics,
|
|
5935
6157
|
getProfile,
|
|
@@ -5946,9 +6168,12 @@ export {
|
|
|
5946
6168
|
isBrainQueryActiveFor,
|
|
5947
6169
|
isBrainSync,
|
|
5948
6170
|
isExclusionFindingsBrainActive,
|
|
6171
|
+
isMeasuredFailureBrainActive,
|
|
6172
|
+
isMeasuredFailureGateEnabledFromEnv,
|
|
5949
6173
|
isModelReachable,
|
|
5950
6174
|
isPromotionsBrainActive,
|
|
5951
6175
|
isProviderReachable,
|
|
6176
|
+
judgeMeasuredFailure,
|
|
5952
6177
|
latencyTierOf,
|
|
5953
6178
|
learningKey,
|
|
5954
6179
|
loadAliasesFromBrain,
|
|
@@ -5982,5 +6207,6 @@ export {
|
|
|
5982
6207
|
runGoldenEval,
|
|
5983
6208
|
setTokenizer,
|
|
5984
6209
|
shouldCaptureGolden,
|
|
5985
|
-
tryGetProfile
|
|
6210
|
+
tryGetProfile,
|
|
6211
|
+
wilsonLowerBound
|
|
5986
6212
|
};
|
package/dist/key-health.js
CHANGED
|
@@ -25,7 +25,7 @@ __export(key_health_exports, {
|
|
|
25
25
|
module.exports = __toCommonJS(key_health_exports);
|
|
26
26
|
|
|
27
27
|
// src/version.ts
|
|
28
|
-
var LIBRARY_VERSION = "2.0.0-alpha.
|
|
28
|
+
var LIBRARY_VERSION = "2.0.0-alpha.70";
|
|
29
29
|
|
|
30
30
|
// src/key-health.ts
|
|
31
31
|
var JSON_HEADERS = { "Content-Type": "application/json" };
|
package/dist/key-health.mjs
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@warmdrift/kgauto-compiler",
|
|
3
|
-
"version": "2.0.0-alpha.
|
|
3
|
+
"version": "2.0.0-alpha.70",
|
|
4
4
|
"description": "Prompt compiler with executable provider knowledge for multi-model AI apps: normalized multi-provider transport with fallback chains, compile-time cliff guards, a curated model registry, and a telemetry flight recorder. Swap models without rewriting prompts.",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"module": "./dist/index.mjs",
|