@warmdrift/kgauto-compiler 2.0.0-alpha.97 → 2.0.0-alpha.98
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-IEEROFZW.mjs → chunk-KENTOSJO.mjs} +1 -1
- package/dist/{chunk-AFYBZRQV.mjs → chunk-KZNTCMGM.mjs} +81 -14
- package/dist/index.d.mts +71 -6
- package/dist/index.d.ts +71 -6
- package/dist/index.js +84 -17
- package/dist/index.mjs +4 -4
- package/dist/key-health.js +1 -1
- package/dist/key-health.mjs +1 -1
- package/dist/probe.js +81 -14
- package/dist/probe.mjs +1 -1
- package/package.json +1 -1
|
@@ -1146,12 +1146,19 @@ function mapRows(rows) {
|
|
|
1146
1146
|
if (!isRawFailureRow(row)) continue;
|
|
1147
1147
|
const n = coerceCount(row.n);
|
|
1148
1148
|
const nFail = coerceCount(row.n_fail) ?? 0;
|
|
1149
|
-
|
|
1149
|
+
const nAttempted = coerceCount(row.n_attempted);
|
|
1150
|
+
const nUnavailable = coerceCount(row.n_unavailable) ?? 0;
|
|
1151
|
+
const nUnclassified = coerceCount(row.n_unclassified) ?? 0;
|
|
1152
|
+
if (n === null) continue;
|
|
1153
|
+
if (n <= 0 && nUnavailable <= 0) continue;
|
|
1150
1154
|
out.push({
|
|
1151
1155
|
archetype: row.intent_archetype,
|
|
1152
1156
|
model: row.model,
|
|
1153
1157
|
n,
|
|
1154
|
-
nFail
|
|
1158
|
+
nFail,
|
|
1159
|
+
nAttempted: nAttempted ?? n,
|
|
1160
|
+
nUnavailable,
|
|
1161
|
+
nUnclassified
|
|
1155
1162
|
});
|
|
1156
1163
|
}
|
|
1157
1164
|
return out;
|
|
@@ -1175,7 +1182,24 @@ var MEASURED_FAILURE_CFG = {
|
|
|
1175
1182
|
/** 95% one-sided-ish confidence (standard two-sided z at α=0.05). */
|
|
1176
1183
|
z: 1.96,
|
|
1177
1184
|
/** Must match the view's window. Documented here for the advisory text. */
|
|
1178
|
-
windowDays: 28
|
|
1185
|
+
windowDays: 28,
|
|
1186
|
+
/**
|
|
1187
|
+
* alpha.98 — the AVAILABILITY axis. Same statistic, separate vocabulary,
|
|
1188
|
+
* separate verdict, because a 429 and a parse-failure send an operator in
|
|
1189
|
+
* opposite directions (fund or throttle the account vs. distrust the
|
|
1190
|
+
* model's declared structuredOutput). Merging them into one "failure rate"
|
|
1191
|
+
* is the s75 normalization defect.
|
|
1192
|
+
*
|
|
1193
|
+
* Thresholds match the quality axis deliberately: "we are 95% confident
|
|
1194
|
+
* this model is unreachable more often than not" is the same unarguable
|
|
1195
|
+
* bar, and a de-rank is the same remedy. Kept as a distinct config block
|
|
1196
|
+
* so the two can be tuned apart when evidence says they should be.
|
|
1197
|
+
*/
|
|
1198
|
+
availability: {
|
|
1199
|
+
minSample: 5,
|
|
1200
|
+
lowerBoundThreshold: 0.5,
|
|
1201
|
+
z: 1.96
|
|
1202
|
+
}
|
|
1179
1203
|
};
|
|
1180
1204
|
function wilsonLowerBound(failures, n, z = MEASURED_FAILURE_CFG.z) {
|
|
1181
1205
|
if (n <= 0) return 0;
|
|
@@ -1192,15 +1216,53 @@ function mapMeasuredFailureRows(rows) {
|
|
|
1192
1216
|
}
|
|
1193
1217
|
function judgeMeasuredFailure(row, cfg = MEASURED_FAILURE_CFG) {
|
|
1194
1218
|
if (!row) return void 0;
|
|
1195
|
-
const
|
|
1196
|
-
if (!
|
|
1197
|
-
const
|
|
1219
|
+
const raw = "nFail" in row && typeof row.n === "number" ? row : mapRows([row])[0];
|
|
1220
|
+
if (!raw) return void 0;
|
|
1221
|
+
const normalized = {
|
|
1222
|
+
...raw,
|
|
1223
|
+
nAttempted: typeof raw.nAttempted === "number" ? raw.nAttempted : raw.n,
|
|
1224
|
+
nUnavailable: typeof raw.nUnavailable === "number" ? raw.nUnavailable : 0,
|
|
1225
|
+
nUnclassified: typeof raw.nUnclassified === "number" ? raw.nUnclassified : 0
|
|
1226
|
+
};
|
|
1227
|
+
const quality = judgeAxis(
|
|
1228
|
+
normalized.nFail,
|
|
1229
|
+
normalized.n,
|
|
1230
|
+
cfg.minSample,
|
|
1231
|
+
cfg.lowerBoundThreshold,
|
|
1232
|
+
cfg.z
|
|
1233
|
+
);
|
|
1234
|
+
const availability = judgeAxis(
|
|
1235
|
+
normalized.nUnavailable,
|
|
1236
|
+
normalized.nAttempted,
|
|
1237
|
+
cfg.availability.minSample,
|
|
1238
|
+
cfg.availability.lowerBoundThreshold,
|
|
1239
|
+
cfg.availability.z
|
|
1240
|
+
);
|
|
1241
|
+
if (normalized.n < cfg.minSample && normalized.nAttempted < cfg.availability.minSample) {
|
|
1242
|
+
return void 0;
|
|
1243
|
+
}
|
|
1244
|
+
const axis = quality.gated ? "quality" : availability.gated ? "availability" : void 0;
|
|
1245
|
+
const lead = axis === "availability" ? availability : quality;
|
|
1198
1246
|
return {
|
|
1199
|
-
gated:
|
|
1200
|
-
|
|
1247
|
+
gated: quality.gated || availability.gated,
|
|
1248
|
+
axis,
|
|
1249
|
+
rate: lead.rate,
|
|
1250
|
+
lowerBound: lead.lowerBound,
|
|
1251
|
+
n: lead.n,
|
|
1252
|
+
nFail: lead.nFail,
|
|
1253
|
+
quality,
|
|
1254
|
+
availability,
|
|
1255
|
+
nUnclassified: normalized.nUnclassified
|
|
1256
|
+
};
|
|
1257
|
+
}
|
|
1258
|
+
function judgeAxis(failures, n, minSample, threshold, z) {
|
|
1259
|
+
const lowerBound = wilsonLowerBound(failures, n, z);
|
|
1260
|
+
return {
|
|
1261
|
+
rate: n > 0 ? failures / n : 0,
|
|
1201
1262
|
lowerBound,
|
|
1202
|
-
n
|
|
1203
|
-
nFail:
|
|
1263
|
+
n,
|
|
1264
|
+
nFail: failures,
|
|
1265
|
+
gated: n >= minSample && lowerBound > threshold
|
|
1204
1266
|
};
|
|
1205
1267
|
}
|
|
1206
1268
|
var snapshots2 = /* @__PURE__ */ new Map();
|
|
@@ -2242,9 +2304,13 @@ function detectSingleModelArray(ir, policy) {
|
|
|
2242
2304
|
];
|
|
2243
2305
|
}
|
|
2244
2306
|
function suppressedRecommendationReason(ir, archetype, altProfile) {
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2307
|
+
const measured = getMeasuredFailureVerdict({
|
|
2308
|
+
appId: ir.appId,
|
|
2309
|
+
archetype,
|
|
2310
|
+
model: altProfile.id
|
|
2311
|
+
});
|
|
2312
|
+
if (measured?.quality.gated === true) return "measured-failure-gate";
|
|
2313
|
+
if (measured?.availability.gated === true) return "measured-unavailable-gate";
|
|
2248
2314
|
if (ir.constraints?.structuredOutput && effectiveConventions(altProfile).some(
|
|
2249
2315
|
(c) => c.archetype === archetype && c.structuredOutputHint === "avoid"
|
|
2250
2316
|
)) {
|
|
@@ -3057,7 +3123,8 @@ function compile(ir, opts = {}) {
|
|
|
3057
3123
|
rate: verdict.rate,
|
|
3058
3124
|
lowerBound: verdict.lowerBound,
|
|
3059
3125
|
n: verdict.n,
|
|
3060
|
-
nFail: verdict.nFail
|
|
3126
|
+
nFail: verdict.nFail,
|
|
3127
|
+
axis: verdict.axis ?? "quality"
|
|
3061
3128
|
});
|
|
3062
3129
|
}
|
|
3063
3130
|
}
|
package/dist/index.d.mts
CHANGED
|
@@ -1241,7 +1241,7 @@ declare function runStrategyEvalWithAttribution(opts: Omit<GoldenEvalOptions, 'a
|
|
|
1241
1241
|
* guard in `tests/version.test.ts` fails the suite (and therefore
|
|
1242
1242
|
* `prepublishOnly`) when they diverge — a stale constant cannot reach npm.
|
|
1243
1243
|
*/
|
|
1244
|
-
declare const LIBRARY_VERSION = "2.0.0-alpha.
|
|
1244
|
+
declare const LIBRARY_VERSION = "2.0.0-alpha.98";
|
|
1245
1245
|
|
|
1246
1246
|
/**
|
|
1247
1247
|
* Oracle contract — how an app tells the brain whether a response was good.
|
|
@@ -3735,10 +3735,26 @@ declare function _testWaitForPromotionsRefresh(): Promise<void>;
|
|
|
3735
3735
|
interface SurfaceFailureRow {
|
|
3736
3736
|
archetype: string;
|
|
3737
3737
|
model: string;
|
|
3738
|
-
/**
|
|
3738
|
+
/**
|
|
3739
|
+
* QUALITY-axis denominator: attempts that produced something judgeable
|
|
3740
|
+
* (served + quality walkaways). Excludes calls that never reached an
|
|
3741
|
+
* answer — a 429 is not a trial of the model's quality. See migration 071.
|
|
3742
|
+
*/
|
|
3739
3743
|
n: number;
|
|
3740
3744
|
/** Attempts that failed on the quality axis. */
|
|
3741
3745
|
nFail: number;
|
|
3746
|
+
/**
|
|
3747
|
+
* alpha.98 — AVAILABILITY-axis denominator: every served attempt is a
|
|
3748
|
+
* trial of whether the model can be reached at all.
|
|
3749
|
+
*/
|
|
3750
|
+
nAttempted: number;
|
|
3751
|
+
/** Attempts that never reached an answer — 429, auth, timeout, transport. */
|
|
3752
|
+
nUnavailable: number;
|
|
3753
|
+
/**
|
|
3754
|
+
* Failures in NEITHER vocabulary. Carried so the taxonomy's blind spot is
|
|
3755
|
+
* a number rather than silence; not gated on.
|
|
3756
|
+
*/
|
|
3757
|
+
nUnclassified: number;
|
|
3742
3758
|
}
|
|
3743
3759
|
declare const MEASURED_FAILURE_CFG: {
|
|
3744
3760
|
/**
|
|
@@ -3760,6 +3776,23 @@ declare const MEASURED_FAILURE_CFG: {
|
|
|
3760
3776
|
readonly z: 1.96;
|
|
3761
3777
|
/** Must match the view's window. Documented here for the advisory text. */
|
|
3762
3778
|
readonly windowDays: 28;
|
|
3779
|
+
/**
|
|
3780
|
+
* alpha.98 — the AVAILABILITY axis. Same statistic, separate vocabulary,
|
|
3781
|
+
* separate verdict, because a 429 and a parse-failure send an operator in
|
|
3782
|
+
* opposite directions (fund or throttle the account vs. distrust the
|
|
3783
|
+
* model's declared structuredOutput). Merging them into one "failure rate"
|
|
3784
|
+
* is the s75 normalization defect.
|
|
3785
|
+
*
|
|
3786
|
+
* Thresholds match the quality axis deliberately: "we are 95% confident
|
|
3787
|
+
* this model is unreachable more often than not" is the same unarguable
|
|
3788
|
+
* bar, and a de-rank is the same remedy. Kept as a distinct config block
|
|
3789
|
+
* so the two can be tuned apart when evidence says they should be.
|
|
3790
|
+
*/
|
|
3791
|
+
readonly availability: {
|
|
3792
|
+
readonly minSample: 5;
|
|
3793
|
+
readonly lowerBoundThreshold: 0.5;
|
|
3794
|
+
readonly z: 1.96;
|
|
3795
|
+
};
|
|
3763
3796
|
};
|
|
3764
3797
|
/**
|
|
3765
3798
|
* Wilson score interval, lower bound. Preferred over the normal
|
|
@@ -3771,16 +3804,45 @@ declare const MEASURED_FAILURE_CFG: {
|
|
|
3771
3804
|
* Returns 0 for n <= 0.
|
|
3772
3805
|
*/
|
|
3773
3806
|
declare function wilsonLowerBound(failures: number, n: number, z?: number): number;
|
|
3807
|
+
/** One axis's numbers. Same statistic, different vocabulary. */
|
|
3808
|
+
interface FailureAxisVerdict {
|
|
3809
|
+
/** Observed failure rate in-window on this axis. */
|
|
3810
|
+
rate: number;
|
|
3811
|
+
/** 95% lower confidence bound on that rate — what the gate tests. */
|
|
3812
|
+
lowerBound: number;
|
|
3813
|
+
/** Attempts backing this axis. */
|
|
3814
|
+
n: number;
|
|
3815
|
+
/** Failures on this axis. */
|
|
3816
|
+
nFail: number;
|
|
3817
|
+
/** Whether this axis on its own clears the bar. */
|
|
3818
|
+
gated: boolean;
|
|
3819
|
+
}
|
|
3774
3820
|
interface MeasuredFailureVerdict {
|
|
3775
|
-
/** Whether
|
|
3821
|
+
/** Whether a measured-failure gate fires for this tuple, on EITHER axis. */
|
|
3776
3822
|
gated: boolean;
|
|
3777
|
-
/**
|
|
3823
|
+
/**
|
|
3824
|
+
* alpha.98 — which axis fired. Undefined when `gated` is false. Read this
|
|
3825
|
+
* before rendering `rate`/`n`: the same number means "answered badly" on
|
|
3826
|
+
* one axis and "could not be reached" on the other, and the fixes differ.
|
|
3827
|
+
* When both axes fire, `'quality'` wins the label — the model is reachable
|
|
3828
|
+
* enough to have been judged, so its answers are the actionable half.
|
|
3829
|
+
*/
|
|
3830
|
+
axis?: 'quality' | 'availability';
|
|
3831
|
+
/**
|
|
3832
|
+
* The numbers that JUSTIFY the gate — i.e. drawn from `axis` when gated,
|
|
3833
|
+
* and from the quality axis otherwise. Deliberately not always-quality:
|
|
3834
|
+
* a consumer printing "gated: 0% over 16" for a model that 429'd 42 times
|
|
3835
|
+
* would be reporting a true number as an explanation of the wrong thing.
|
|
3836
|
+
*/
|
|
3778
3837
|
rate: number;
|
|
3779
|
-
/** 95% lower confidence bound on that rate — what the gate tests. */
|
|
3780
3838
|
lowerBound: number;
|
|
3781
|
-
/** Attempts backing the verdict. */
|
|
3782
3839
|
n: number;
|
|
3783
3840
|
nFail: number;
|
|
3841
|
+
/** Both axes, always present, so a caller never has to infer one. */
|
|
3842
|
+
quality: FailureAxisVerdict;
|
|
3843
|
+
availability: FailureAxisVerdict;
|
|
3844
|
+
/** Failures in neither vocabulary. Not gated on; surfaced for visibility. */
|
|
3845
|
+
nUnclassified: number;
|
|
3784
3846
|
}
|
|
3785
3847
|
/**
|
|
3786
3848
|
* alpha.78 — public row normalizer (PB 2026-07-25 trap): the counts
|
|
@@ -3805,6 +3867,9 @@ declare function judgeMeasuredFailure(row: SurfaceFailureRow | {
|
|
|
3805
3867
|
model: string;
|
|
3806
3868
|
n: number | string;
|
|
3807
3869
|
n_fail: number | string;
|
|
3870
|
+
n_attempted?: number | string;
|
|
3871
|
+
n_unavailable?: number | string;
|
|
3872
|
+
n_unclassified?: number | string;
|
|
3808
3873
|
} | undefined, cfg?: typeof MEASURED_FAILURE_CFG): MeasuredFailureVerdict | undefined;
|
|
3809
3874
|
interface MeasuredFailureRuntime {
|
|
3810
3875
|
/** Endpoint base URL. The library appends `?app_id=<id>`. */
|
package/dist/index.d.ts
CHANGED
|
@@ -1241,7 +1241,7 @@ declare function runStrategyEvalWithAttribution(opts: Omit<GoldenEvalOptions, 'a
|
|
|
1241
1241
|
* guard in `tests/version.test.ts` fails the suite (and therefore
|
|
1242
1242
|
* `prepublishOnly`) when they diverge — a stale constant cannot reach npm.
|
|
1243
1243
|
*/
|
|
1244
|
-
declare const LIBRARY_VERSION = "2.0.0-alpha.
|
|
1244
|
+
declare const LIBRARY_VERSION = "2.0.0-alpha.98";
|
|
1245
1245
|
|
|
1246
1246
|
/**
|
|
1247
1247
|
* Oracle contract — how an app tells the brain whether a response was good.
|
|
@@ -3735,10 +3735,26 @@ declare function _testWaitForPromotionsRefresh(): Promise<void>;
|
|
|
3735
3735
|
interface SurfaceFailureRow {
|
|
3736
3736
|
archetype: string;
|
|
3737
3737
|
model: string;
|
|
3738
|
-
/**
|
|
3738
|
+
/**
|
|
3739
|
+
* QUALITY-axis denominator: attempts that produced something judgeable
|
|
3740
|
+
* (served + quality walkaways). Excludes calls that never reached an
|
|
3741
|
+
* answer — a 429 is not a trial of the model's quality. See migration 071.
|
|
3742
|
+
*/
|
|
3739
3743
|
n: number;
|
|
3740
3744
|
/** Attempts that failed on the quality axis. */
|
|
3741
3745
|
nFail: number;
|
|
3746
|
+
/**
|
|
3747
|
+
* alpha.98 — AVAILABILITY-axis denominator: every served attempt is a
|
|
3748
|
+
* trial of whether the model can be reached at all.
|
|
3749
|
+
*/
|
|
3750
|
+
nAttempted: number;
|
|
3751
|
+
/** Attempts that never reached an answer — 429, auth, timeout, transport. */
|
|
3752
|
+
nUnavailable: number;
|
|
3753
|
+
/**
|
|
3754
|
+
* Failures in NEITHER vocabulary. Carried so the taxonomy's blind spot is
|
|
3755
|
+
* a number rather than silence; not gated on.
|
|
3756
|
+
*/
|
|
3757
|
+
nUnclassified: number;
|
|
3742
3758
|
}
|
|
3743
3759
|
declare const MEASURED_FAILURE_CFG: {
|
|
3744
3760
|
/**
|
|
@@ -3760,6 +3776,23 @@ declare const MEASURED_FAILURE_CFG: {
|
|
|
3760
3776
|
readonly z: 1.96;
|
|
3761
3777
|
/** Must match the view's window. Documented here for the advisory text. */
|
|
3762
3778
|
readonly windowDays: 28;
|
|
3779
|
+
/**
|
|
3780
|
+
* alpha.98 — the AVAILABILITY axis. Same statistic, separate vocabulary,
|
|
3781
|
+
* separate verdict, because a 429 and a parse-failure send an operator in
|
|
3782
|
+
* opposite directions (fund or throttle the account vs. distrust the
|
|
3783
|
+
* model's declared structuredOutput). Merging them into one "failure rate"
|
|
3784
|
+
* is the s75 normalization defect.
|
|
3785
|
+
*
|
|
3786
|
+
* Thresholds match the quality axis deliberately: "we are 95% confident
|
|
3787
|
+
* this model is unreachable more often than not" is the same unarguable
|
|
3788
|
+
* bar, and a de-rank is the same remedy. Kept as a distinct config block
|
|
3789
|
+
* so the two can be tuned apart when evidence says they should be.
|
|
3790
|
+
*/
|
|
3791
|
+
readonly availability: {
|
|
3792
|
+
readonly minSample: 5;
|
|
3793
|
+
readonly lowerBoundThreshold: 0.5;
|
|
3794
|
+
readonly z: 1.96;
|
|
3795
|
+
};
|
|
3763
3796
|
};
|
|
3764
3797
|
/**
|
|
3765
3798
|
* Wilson score interval, lower bound. Preferred over the normal
|
|
@@ -3771,16 +3804,45 @@ declare const MEASURED_FAILURE_CFG: {
|
|
|
3771
3804
|
* Returns 0 for n <= 0.
|
|
3772
3805
|
*/
|
|
3773
3806
|
declare function wilsonLowerBound(failures: number, n: number, z?: number): number;
|
|
3807
|
+
/** One axis's numbers. Same statistic, different vocabulary. */
|
|
3808
|
+
interface FailureAxisVerdict {
|
|
3809
|
+
/** Observed failure rate in-window on this axis. */
|
|
3810
|
+
rate: number;
|
|
3811
|
+
/** 95% lower confidence bound on that rate — what the gate tests. */
|
|
3812
|
+
lowerBound: number;
|
|
3813
|
+
/** Attempts backing this axis. */
|
|
3814
|
+
n: number;
|
|
3815
|
+
/** Failures on this axis. */
|
|
3816
|
+
nFail: number;
|
|
3817
|
+
/** Whether this axis on its own clears the bar. */
|
|
3818
|
+
gated: boolean;
|
|
3819
|
+
}
|
|
3774
3820
|
interface MeasuredFailureVerdict {
|
|
3775
|
-
/** Whether
|
|
3821
|
+
/** Whether a measured-failure gate fires for this tuple, on EITHER axis. */
|
|
3776
3822
|
gated: boolean;
|
|
3777
|
-
/**
|
|
3823
|
+
/**
|
|
3824
|
+
* alpha.98 — which axis fired. Undefined when `gated` is false. Read this
|
|
3825
|
+
* before rendering `rate`/`n`: the same number means "answered badly" on
|
|
3826
|
+
* one axis and "could not be reached" on the other, and the fixes differ.
|
|
3827
|
+
* When both axes fire, `'quality'` wins the label — the model is reachable
|
|
3828
|
+
* enough to have been judged, so its answers are the actionable half.
|
|
3829
|
+
*/
|
|
3830
|
+
axis?: 'quality' | 'availability';
|
|
3831
|
+
/**
|
|
3832
|
+
* The numbers that JUSTIFY the gate — i.e. drawn from `axis` when gated,
|
|
3833
|
+
* and from the quality axis otherwise. Deliberately not always-quality:
|
|
3834
|
+
* a consumer printing "gated: 0% over 16" for a model that 429'd 42 times
|
|
3835
|
+
* would be reporting a true number as an explanation of the wrong thing.
|
|
3836
|
+
*/
|
|
3778
3837
|
rate: number;
|
|
3779
|
-
/** 95% lower confidence bound on that rate — what the gate tests. */
|
|
3780
3838
|
lowerBound: number;
|
|
3781
|
-
/** Attempts backing the verdict. */
|
|
3782
3839
|
n: number;
|
|
3783
3840
|
nFail: number;
|
|
3841
|
+
/** Both axes, always present, so a caller never has to infer one. */
|
|
3842
|
+
quality: FailureAxisVerdict;
|
|
3843
|
+
availability: FailureAxisVerdict;
|
|
3844
|
+
/** Failures in neither vocabulary. Not gated on; surfaced for visibility. */
|
|
3845
|
+
nUnclassified: number;
|
|
3784
3846
|
}
|
|
3785
3847
|
/**
|
|
3786
3848
|
* alpha.78 — public row normalizer (PB 2026-07-25 trap): the counts
|
|
@@ -3805,6 +3867,9 @@ declare function judgeMeasuredFailure(row: SurfaceFailureRow | {
|
|
|
3805
3867
|
model: string;
|
|
3806
3868
|
n: number | string;
|
|
3807
3869
|
n_fail: number | string;
|
|
3870
|
+
n_attempted?: number | string;
|
|
3871
|
+
n_unavailable?: number | string;
|
|
3872
|
+
n_unclassified?: number | string;
|
|
3808
3873
|
} | undefined, cfg?: typeof MEASURED_FAILURE_CFG): MeasuredFailureVerdict | undefined;
|
|
3809
3874
|
interface MeasuredFailureRuntime {
|
|
3810
3875
|
/** Endpoint base URL. The library appends `?app_id=<id>`. */
|
package/dist/index.js
CHANGED
|
@@ -4843,12 +4843,19 @@ function mapRows(rows) {
|
|
|
4843
4843
|
if (!isRawFailureRow(row)) continue;
|
|
4844
4844
|
const n = coerceCount(row.n);
|
|
4845
4845
|
const nFail = coerceCount(row.n_fail) ?? 0;
|
|
4846
|
-
|
|
4846
|
+
const nAttempted = coerceCount(row.n_attempted);
|
|
4847
|
+
const nUnavailable = coerceCount(row.n_unavailable) ?? 0;
|
|
4848
|
+
const nUnclassified = coerceCount(row.n_unclassified) ?? 0;
|
|
4849
|
+
if (n === null) continue;
|
|
4850
|
+
if (n <= 0 && nUnavailable <= 0) continue;
|
|
4847
4851
|
out.push({
|
|
4848
4852
|
archetype: row.intent_archetype,
|
|
4849
4853
|
model: row.model,
|
|
4850
4854
|
n,
|
|
4851
|
-
nFail
|
|
4855
|
+
nFail,
|
|
4856
|
+
nAttempted: nAttempted ?? n,
|
|
4857
|
+
nUnavailable,
|
|
4858
|
+
nUnclassified
|
|
4852
4859
|
});
|
|
4853
4860
|
}
|
|
4854
4861
|
return out;
|
|
@@ -4872,7 +4879,24 @@ var MEASURED_FAILURE_CFG = {
|
|
|
4872
4879
|
/** 95% one-sided-ish confidence (standard two-sided z at α=0.05). */
|
|
4873
4880
|
z: 1.96,
|
|
4874
4881
|
/** Must match the view's window. Documented here for the advisory text. */
|
|
4875
|
-
windowDays: 28
|
|
4882
|
+
windowDays: 28,
|
|
4883
|
+
/**
|
|
4884
|
+
* alpha.98 — the AVAILABILITY axis. Same statistic, separate vocabulary,
|
|
4885
|
+
* separate verdict, because a 429 and a parse-failure send an operator in
|
|
4886
|
+
* opposite directions (fund or throttle the account vs. distrust the
|
|
4887
|
+
* model's declared structuredOutput). Merging them into one "failure rate"
|
|
4888
|
+
* is the s75 normalization defect.
|
|
4889
|
+
*
|
|
4890
|
+
* Thresholds match the quality axis deliberately: "we are 95% confident
|
|
4891
|
+
* this model is unreachable more often than not" is the same unarguable
|
|
4892
|
+
* bar, and a de-rank is the same remedy. Kept as a distinct config block
|
|
4893
|
+
* so the two can be tuned apart when evidence says they should be.
|
|
4894
|
+
*/
|
|
4895
|
+
availability: {
|
|
4896
|
+
minSample: 5,
|
|
4897
|
+
lowerBoundThreshold: 0.5,
|
|
4898
|
+
z: 1.96
|
|
4899
|
+
}
|
|
4876
4900
|
};
|
|
4877
4901
|
function wilsonLowerBound(failures, n, z = MEASURED_FAILURE_CFG.z) {
|
|
4878
4902
|
if (n <= 0) return 0;
|
|
@@ -4889,15 +4913,53 @@ function mapMeasuredFailureRows(rows) {
|
|
|
4889
4913
|
}
|
|
4890
4914
|
function judgeMeasuredFailure(row, cfg = MEASURED_FAILURE_CFG) {
|
|
4891
4915
|
if (!row) return void 0;
|
|
4892
|
-
const
|
|
4893
|
-
if (!
|
|
4894
|
-
const
|
|
4916
|
+
const raw = "nFail" in row && typeof row.n === "number" ? row : mapRows([row])[0];
|
|
4917
|
+
if (!raw) return void 0;
|
|
4918
|
+
const normalized = {
|
|
4919
|
+
...raw,
|
|
4920
|
+
nAttempted: typeof raw.nAttempted === "number" ? raw.nAttempted : raw.n,
|
|
4921
|
+
nUnavailable: typeof raw.nUnavailable === "number" ? raw.nUnavailable : 0,
|
|
4922
|
+
nUnclassified: typeof raw.nUnclassified === "number" ? raw.nUnclassified : 0
|
|
4923
|
+
};
|
|
4924
|
+
const quality = judgeAxis(
|
|
4925
|
+
normalized.nFail,
|
|
4926
|
+
normalized.n,
|
|
4927
|
+
cfg.minSample,
|
|
4928
|
+
cfg.lowerBoundThreshold,
|
|
4929
|
+
cfg.z
|
|
4930
|
+
);
|
|
4931
|
+
const availability = judgeAxis(
|
|
4932
|
+
normalized.nUnavailable,
|
|
4933
|
+
normalized.nAttempted,
|
|
4934
|
+
cfg.availability.minSample,
|
|
4935
|
+
cfg.availability.lowerBoundThreshold,
|
|
4936
|
+
cfg.availability.z
|
|
4937
|
+
);
|
|
4938
|
+
if (normalized.n < cfg.minSample && normalized.nAttempted < cfg.availability.minSample) {
|
|
4939
|
+
return void 0;
|
|
4940
|
+
}
|
|
4941
|
+
const axis = quality.gated ? "quality" : availability.gated ? "availability" : void 0;
|
|
4942
|
+
const lead = axis === "availability" ? availability : quality;
|
|
4895
4943
|
return {
|
|
4896
|
-
gated:
|
|
4897
|
-
|
|
4944
|
+
gated: quality.gated || availability.gated,
|
|
4945
|
+
axis,
|
|
4946
|
+
rate: lead.rate,
|
|
4947
|
+
lowerBound: lead.lowerBound,
|
|
4948
|
+
n: lead.n,
|
|
4949
|
+
nFail: lead.nFail,
|
|
4950
|
+
quality,
|
|
4951
|
+
availability,
|
|
4952
|
+
nUnclassified: normalized.nUnclassified
|
|
4953
|
+
};
|
|
4954
|
+
}
|
|
4955
|
+
function judgeAxis(failures, n, minSample, threshold, z) {
|
|
4956
|
+
const lowerBound = wilsonLowerBound(failures, n, z);
|
|
4957
|
+
return {
|
|
4958
|
+
rate: n > 0 ? failures / n : 0,
|
|
4898
4959
|
lowerBound,
|
|
4899
|
-
n
|
|
4900
|
-
nFail:
|
|
4960
|
+
n,
|
|
4961
|
+
nFail: failures,
|
|
4962
|
+
gated: n >= minSample && lowerBound > threshold
|
|
4901
4963
|
};
|
|
4902
4964
|
}
|
|
4903
4965
|
var snapshots2 = /* @__PURE__ */ new Map();
|
|
@@ -5939,9 +6001,13 @@ function detectSingleModelArray(ir, policy) {
|
|
|
5939
6001
|
];
|
|
5940
6002
|
}
|
|
5941
6003
|
function suppressedRecommendationReason(ir, archetype, altProfile) {
|
|
5942
|
-
|
|
5943
|
-
|
|
5944
|
-
|
|
6004
|
+
const measured = getMeasuredFailureVerdict({
|
|
6005
|
+
appId: ir.appId,
|
|
6006
|
+
archetype,
|
|
6007
|
+
model: altProfile.id
|
|
6008
|
+
});
|
|
6009
|
+
if (measured?.quality.gated === true) return "measured-failure-gate";
|
|
6010
|
+
if (measured?.availability.gated === true) return "measured-unavailable-gate";
|
|
5945
6011
|
if (ir.constraints?.structuredOutput && effectiveConventions(altProfile).some(
|
|
5946
6012
|
(c) => c.archetype === archetype && c.structuredOutputHint === "avoid"
|
|
5947
6013
|
)) {
|
|
@@ -6326,7 +6392,8 @@ function compile(ir, opts = {}) {
|
|
|
6326
6392
|
rate: verdict.rate,
|
|
6327
6393
|
lowerBound: verdict.lowerBound,
|
|
6328
6394
|
n: verdict.n,
|
|
6329
|
-
nFail: verdict.nFail
|
|
6395
|
+
nFail: verdict.nFail,
|
|
6396
|
+
axis: verdict.axis ?? "quality"
|
|
6330
6397
|
});
|
|
6331
6398
|
}
|
|
6332
6399
|
}
|
|
@@ -6670,7 +6737,7 @@ function validateFinalFit(ir, profile, tokens) {
|
|
|
6670
6737
|
}
|
|
6671
6738
|
|
|
6672
6739
|
// src/version.ts
|
|
6673
|
-
var LIBRARY_VERSION = "2.0.0-alpha.
|
|
6740
|
+
var LIBRARY_VERSION = "2.0.0-alpha.98";
|
|
6674
6741
|
|
|
6675
6742
|
// src/pricing-brain.ts
|
|
6676
6743
|
function isPricingRow(x) {
|
|
@@ -8816,8 +8883,8 @@ async function call(ir, opts = {}) {
|
|
|
8816
8883
|
appId: ir.appId,
|
|
8817
8884
|
archetype: ir.intent.archetype,
|
|
8818
8885
|
model: targetModel
|
|
8819
|
-
})?.gated === true) {
|
|
8820
|
-
retrySuppressionNote = " [sameModelRetry suppressed: this model carries an active measured-failure gate for this archetype \u2014 retrying it would near-certainly bill a second doomed inference; walking the chain instead]";
|
|
8886
|
+
})?.quality.gated === true) {
|
|
8887
|
+
retrySuppressionNote = " [sameModelRetry suppressed: this model carries an active measured-failure gate (quality axis) for this archetype \u2014 retrying it would near-certainly bill a second doomed inference; walking the chain instead]";
|
|
8821
8888
|
}
|
|
8822
8889
|
if (!validated.ok && isStructuredContractViolation(validated.errorCode) && sameModelRetryEnabled && !retriedSameModel && !retrySuppressionNote) {
|
|
8823
8890
|
retriedSameModel = true;
|
package/dist/index.mjs
CHANGED
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
LIBRARY_VERSION,
|
|
8
8
|
createKeyHealthRoute,
|
|
9
9
|
keyFingerprint
|
|
10
|
-
} from "./chunk-
|
|
10
|
+
} from "./chunk-KENTOSJO.mjs";
|
|
11
11
|
import {
|
|
12
12
|
ARCHETYPE_FAMILY_FITS,
|
|
13
13
|
BLOCKED_MODEL_FAMILY_SIBLING_SERVED_CODE,
|
|
@@ -63,7 +63,7 @@ import {
|
|
|
63
63
|
runAdvisor,
|
|
64
64
|
setTokenizer,
|
|
65
65
|
wilsonLowerBound
|
|
66
|
-
} from "./chunk-
|
|
66
|
+
} from "./chunk-KZNTCMGM.mjs";
|
|
67
67
|
import {
|
|
68
68
|
ABSOLUTE_FLOOR,
|
|
69
69
|
ARCHETYPE_FLOOR_DEFAULT,
|
|
@@ -1980,8 +1980,8 @@ async function call(ir, opts = {}) {
|
|
|
1980
1980
|
appId: ir.appId,
|
|
1981
1981
|
archetype: ir.intent.archetype,
|
|
1982
1982
|
model: targetModel
|
|
1983
|
-
})?.gated === true) {
|
|
1984
|
-
retrySuppressionNote = " [sameModelRetry suppressed: this model carries an active measured-failure gate for this archetype \u2014 retrying it would near-certainly bill a second doomed inference; walking the chain instead]";
|
|
1983
|
+
})?.quality.gated === true) {
|
|
1984
|
+
retrySuppressionNote = " [sameModelRetry suppressed: this model carries an active measured-failure gate (quality axis) for this archetype \u2014 retrying it would near-certainly bill a second doomed inference; walking the chain instead]";
|
|
1985
1985
|
}
|
|
1986
1986
|
if (!validated.ok && isStructuredContractViolation(validated.errorCode) && sameModelRetryEnabled && !retriedSameModel && !retrySuppressionNote) {
|
|
1987
1987
|
retriedSameModel = true;
|
package/dist/key-health.js
CHANGED
|
@@ -28,7 +28,7 @@ __export(key_health_exports, {
|
|
|
28
28
|
module.exports = __toCommonJS(key_health_exports);
|
|
29
29
|
|
|
30
30
|
// src/version.ts
|
|
31
|
-
var LIBRARY_VERSION = "2.0.0-alpha.
|
|
31
|
+
var LIBRARY_VERSION = "2.0.0-alpha.98";
|
|
32
32
|
|
|
33
33
|
// src/key-health.ts
|
|
34
34
|
var JSON_HEADERS = { "Content-Type": "application/json" };
|
package/dist/key-health.mjs
CHANGED
package/dist/probe.js
CHANGED
|
@@ -4284,12 +4284,19 @@ function mapRows(rows) {
|
|
|
4284
4284
|
if (!isRawFailureRow(row)) continue;
|
|
4285
4285
|
const n = coerceCount(row.n);
|
|
4286
4286
|
const nFail = coerceCount(row.n_fail) ?? 0;
|
|
4287
|
-
|
|
4287
|
+
const nAttempted = coerceCount(row.n_attempted);
|
|
4288
|
+
const nUnavailable = coerceCount(row.n_unavailable) ?? 0;
|
|
4289
|
+
const nUnclassified = coerceCount(row.n_unclassified) ?? 0;
|
|
4290
|
+
if (n === null) continue;
|
|
4291
|
+
if (n <= 0 && nUnavailable <= 0) continue;
|
|
4288
4292
|
out.push({
|
|
4289
4293
|
archetype: row.intent_archetype,
|
|
4290
4294
|
model: row.model,
|
|
4291
4295
|
n,
|
|
4292
|
-
nFail
|
|
4296
|
+
nFail,
|
|
4297
|
+
nAttempted: nAttempted ?? n,
|
|
4298
|
+
nUnavailable,
|
|
4299
|
+
nUnclassified
|
|
4293
4300
|
});
|
|
4294
4301
|
}
|
|
4295
4302
|
return out;
|
|
@@ -4313,7 +4320,24 @@ var MEASURED_FAILURE_CFG = {
|
|
|
4313
4320
|
/** 95% one-sided-ish confidence (standard two-sided z at α=0.05). */
|
|
4314
4321
|
z: 1.96,
|
|
4315
4322
|
/** Must match the view's window. Documented here for the advisory text. */
|
|
4316
|
-
windowDays: 28
|
|
4323
|
+
windowDays: 28,
|
|
4324
|
+
/**
|
|
4325
|
+
* alpha.98 — the AVAILABILITY axis. Same statistic, separate vocabulary,
|
|
4326
|
+
* separate verdict, because a 429 and a parse-failure send an operator in
|
|
4327
|
+
* opposite directions (fund or throttle the account vs. distrust the
|
|
4328
|
+
* model's declared structuredOutput). Merging them into one "failure rate"
|
|
4329
|
+
* is the s75 normalization defect.
|
|
4330
|
+
*
|
|
4331
|
+
* Thresholds match the quality axis deliberately: "we are 95% confident
|
|
4332
|
+
* this model is unreachable more often than not" is the same unarguable
|
|
4333
|
+
* bar, and a de-rank is the same remedy. Kept as a distinct config block
|
|
4334
|
+
* so the two can be tuned apart when evidence says they should be.
|
|
4335
|
+
*/
|
|
4336
|
+
availability: {
|
|
4337
|
+
minSample: 5,
|
|
4338
|
+
lowerBoundThreshold: 0.5,
|
|
4339
|
+
z: 1.96
|
|
4340
|
+
}
|
|
4317
4341
|
};
|
|
4318
4342
|
function wilsonLowerBound(failures, n, z = MEASURED_FAILURE_CFG.z) {
|
|
4319
4343
|
if (n <= 0) return 0;
|
|
@@ -4327,15 +4351,53 @@ function wilsonLowerBound(failures, n, z = MEASURED_FAILURE_CFG.z) {
|
|
|
4327
4351
|
}
|
|
4328
4352
|
function judgeMeasuredFailure(row, cfg = MEASURED_FAILURE_CFG) {
|
|
4329
4353
|
if (!row) return void 0;
|
|
4330
|
-
const
|
|
4331
|
-
if (!
|
|
4332
|
-
const
|
|
4354
|
+
const raw = "nFail" in row && typeof row.n === "number" ? row : mapRows([row])[0];
|
|
4355
|
+
if (!raw) return void 0;
|
|
4356
|
+
const normalized = {
|
|
4357
|
+
...raw,
|
|
4358
|
+
nAttempted: typeof raw.nAttempted === "number" ? raw.nAttempted : raw.n,
|
|
4359
|
+
nUnavailable: typeof raw.nUnavailable === "number" ? raw.nUnavailable : 0,
|
|
4360
|
+
nUnclassified: typeof raw.nUnclassified === "number" ? raw.nUnclassified : 0
|
|
4361
|
+
};
|
|
4362
|
+
const quality = judgeAxis(
|
|
4363
|
+
normalized.nFail,
|
|
4364
|
+
normalized.n,
|
|
4365
|
+
cfg.minSample,
|
|
4366
|
+
cfg.lowerBoundThreshold,
|
|
4367
|
+
cfg.z
|
|
4368
|
+
);
|
|
4369
|
+
const availability = judgeAxis(
|
|
4370
|
+
normalized.nUnavailable,
|
|
4371
|
+
normalized.nAttempted,
|
|
4372
|
+
cfg.availability.minSample,
|
|
4373
|
+
cfg.availability.lowerBoundThreshold,
|
|
4374
|
+
cfg.availability.z
|
|
4375
|
+
);
|
|
4376
|
+
if (normalized.n < cfg.minSample && normalized.nAttempted < cfg.availability.minSample) {
|
|
4377
|
+
return void 0;
|
|
4378
|
+
}
|
|
4379
|
+
const axis = quality.gated ? "quality" : availability.gated ? "availability" : void 0;
|
|
4380
|
+
const lead = axis === "availability" ? availability : quality;
|
|
4333
4381
|
return {
|
|
4334
|
-
gated:
|
|
4335
|
-
|
|
4382
|
+
gated: quality.gated || availability.gated,
|
|
4383
|
+
axis,
|
|
4384
|
+
rate: lead.rate,
|
|
4385
|
+
lowerBound: lead.lowerBound,
|
|
4386
|
+
n: lead.n,
|
|
4387
|
+
nFail: lead.nFail,
|
|
4388
|
+
quality,
|
|
4389
|
+
availability,
|
|
4390
|
+
nUnclassified: normalized.nUnclassified
|
|
4391
|
+
};
|
|
4392
|
+
}
|
|
4393
|
+
function judgeAxis(failures, n, minSample, threshold, z) {
|
|
4394
|
+
const lowerBound = wilsonLowerBound(failures, n, z);
|
|
4395
|
+
return {
|
|
4396
|
+
rate: n > 0 ? failures / n : 0,
|
|
4336
4397
|
lowerBound,
|
|
4337
|
-
n
|
|
4338
|
-
nFail:
|
|
4398
|
+
n,
|
|
4399
|
+
nFail: failures,
|
|
4400
|
+
gated: n >= minSample && lowerBound > threshold
|
|
4339
4401
|
};
|
|
4340
4402
|
}
|
|
4341
4403
|
var snapshots2 = /* @__PURE__ */ new Map();
|
|
@@ -5199,9 +5261,13 @@ function detectSingleModelArray(ir, policy) {
|
|
|
5199
5261
|
];
|
|
5200
5262
|
}
|
|
5201
5263
|
function suppressedRecommendationReason(ir, archetype, altProfile) {
|
|
5202
|
-
|
|
5203
|
-
|
|
5204
|
-
|
|
5264
|
+
const measured = getMeasuredFailureVerdict({
|
|
5265
|
+
appId: ir.appId,
|
|
5266
|
+
archetype,
|
|
5267
|
+
model: altProfile.id
|
|
5268
|
+
});
|
|
5269
|
+
if (measured?.quality.gated === true) return "measured-failure-gate";
|
|
5270
|
+
if (measured?.availability.gated === true) return "measured-unavailable-gate";
|
|
5205
5271
|
if (ir.constraints?.structuredOutput && effectiveConventions(altProfile).some(
|
|
5206
5272
|
(c) => c.archetype === archetype && c.structuredOutputHint === "avoid"
|
|
5207
5273
|
)) {
|
|
@@ -5586,7 +5652,8 @@ function compile(ir, opts = {}) {
|
|
|
5586
5652
|
rate: verdict.rate,
|
|
5587
5653
|
lowerBound: verdict.lowerBound,
|
|
5588
5654
|
n: verdict.n,
|
|
5589
|
-
nFail: verdict.nFail
|
|
5655
|
+
nFail: verdict.nFail,
|
|
5656
|
+
axis: verdict.axis ?? "quality"
|
|
5590
5657
|
});
|
|
5591
5658
|
}
|
|
5592
5659
|
}
|
package/dist/probe.mjs
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@warmdrift/kgauto-compiler",
|
|
3
|
-
"version": "2.0.0-alpha.
|
|
3
|
+
"version": "2.0.0-alpha.98",
|
|
4
4
|
"description": "Prompt compiler with executable provider knowledge for multi-model AI apps: normalized multi-provider transport with fallback chains, compile-time cliff guards, a curated model registry, and a telemetry flight recorder. Swap models without rewriting prompts.",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"module": "./dist/index.mjs",
|