@takk/bayesoutputgate 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +92 -0
- package/LICENSE +190 -0
- package/NOTICE +45 -0
- package/README.md +403 -0
- package/SECURITY.md +98 -0
- package/SPEC.md +467 -0
- package/dist/adapter/index.cjs +411 -0
- package/dist/adapter/index.d.cts +29 -0
- package/dist/adapter/index.d.ts +29 -0
- package/dist/adapter/index.js +404 -0
- package/dist/audit/index.cjs +82 -0
- package/dist/audit/index.d.cts +40 -0
- package/dist/audit/index.d.ts +40 -0
- package/dist/audit/index.js +77 -0
- package/dist/bayesfactor/index.cjs +152 -0
- package/dist/bayesfactor/index.d.cts +15 -0
- package/dist/bayesfactor/index.d.ts +15 -0
- package/dist/bayesfactor/index.js +149 -0
- package/dist/beta/index.cjs +180 -0
- package/dist/beta/index.d.cts +45 -0
- package/dist/beta/index.d.ts +45 -0
- package/dist/beta/index.js +178 -0
- package/dist/calibration/index.cjs +339 -0
- package/dist/calibration/index.d.cts +53 -0
- package/dist/calibration/index.d.ts +53 -0
- package/dist/calibration/index.js +333 -0
- package/dist/cli/index.cjs +968 -0
- package/dist/cli/index.d.cts +1 -0
- package/dist/cli/index.d.ts +1 -0
- package/dist/cli/index.js +966 -0
- package/dist/dimensions/index.cjs +106 -0
- package/dist/dimensions/index.d.cts +33 -0
- package/dist/dimensions/index.d.ts +33 -0
- package/dist/dimensions/index.js +104 -0
- package/dist/edge/index.cjs +1141 -0
- package/dist/edge/index.d.cts +12 -0
- package/dist/edge/index.d.ts +12 -0
- package/dist/edge/index.js +1109 -0
- package/dist/gate/index.cjs +803 -0
- package/dist/gate/index.d.cts +77 -0
- package/dist/gate/index.d.ts +77 -0
- package/dist/gate/index.js +799 -0
- package/dist/hypothesis/index.cjs +268 -0
- package/dist/hypothesis/index.d.cts +38 -0
- package/dist/hypothesis/index.d.ts +38 -0
- package/dist/hypothesis/index.js +266 -0
- package/dist/index.cjs +1141 -0
- package/dist/index.d.cts +29 -0
- package/dist/index.d.ts +29 -0
- package/dist/index.js +1109 -0
- package/dist/likelihood/index.cjs +137 -0
- package/dist/likelihood/index.d.cts +23 -0
- package/dist/likelihood/index.d.ts +23 -0
- package/dist/likelihood/index.js +132 -0
- package/dist/node/index.cjs +1282 -0
- package/dist/node/index.d.cts +24 -0
- package/dist/node/index.d.ts +24 -0
- package/dist/node/index.js +1246 -0
- package/dist/policy/index.cjs +88 -0
- package/dist/policy/index.d.cts +11 -0
- package/dist/policy/index.d.ts +11 -0
- package/dist/policy/index.js +85 -0
- package/dist/types-bMjn1j4e.d.cts +159 -0
- package/dist/types-bMjn1j4e.d.ts +159 -0
- package/package.json +142 -0
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// src/errors.ts
|
|
4
|
+
var BayesOutputGateError = class _BayesOutputGateError extends Error {
|
|
5
|
+
code;
|
|
6
|
+
constructor(code, message) {
|
|
7
|
+
super(message);
|
|
8
|
+
this.name = "BayesOutputGateError";
|
|
9
|
+
this.code = code;
|
|
10
|
+
Object.setPrototypeOf(this, _BayesOutputGateError.prototype);
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
function invariant(condition, code, message) {
|
|
14
|
+
if (!condition) {
|
|
15
|
+
throw new BayesOutputGateError(code, message);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
// src/policy/index.ts
|
|
20
|
+
function posteriorHighQuality(logBayesFactor, priorHighQuality) {
|
|
21
|
+
invariant(
|
|
22
|
+
priorHighQuality > 0 && priorHighQuality < 1,
|
|
23
|
+
"INVALID_CONFIG",
|
|
24
|
+
`priorHighQuality must be in (0, 1), got ${priorHighQuality}`
|
|
25
|
+
);
|
|
26
|
+
invariant(
|
|
27
|
+
Number.isFinite(logBayesFactor),
|
|
28
|
+
"NUMERIC",
|
|
29
|
+
`logBayesFactor must be finite, got ${logBayesFactor}`
|
|
30
|
+
);
|
|
31
|
+
const logPosteriorOdds = logBayesFactor + Math.log(priorHighQuality / (1 - priorHighQuality));
|
|
32
|
+
if (logPosteriorOdds >= 0) {
|
|
33
|
+
return 1 / (1 + Math.exp(-logPosteriorOdds));
|
|
34
|
+
}
|
|
35
|
+
const odds = Math.exp(logPosteriorOdds);
|
|
36
|
+
return odds / (1 + odds);
|
|
37
|
+
}
|
|
38
|
+
function decide(logBayesFactor, policy) {
|
|
39
|
+
if (policy.kind === "bayes-factor") {
|
|
40
|
+
invariant(
|
|
41
|
+
policy.passAbove >= 1,
|
|
42
|
+
"INVALID_CONFIG",
|
|
43
|
+
`passAbove must be >= 1, got ${policy.passAbove}`
|
|
44
|
+
);
|
|
45
|
+
invariant(
|
|
46
|
+
policy.failBelow > 0 && policy.failBelow <= 1,
|
|
47
|
+
"INVALID_CONFIG",
|
|
48
|
+
`failBelow must be in (0, 1], got ${policy.failBelow}`
|
|
49
|
+
);
|
|
50
|
+
invariant(
|
|
51
|
+
policy.passAbove > policy.failBelow,
|
|
52
|
+
"INVALID_CONFIG",
|
|
53
|
+
"passAbove must be greater than failBelow"
|
|
54
|
+
);
|
|
55
|
+
const bayesFactor = Math.exp(logBayesFactor);
|
|
56
|
+
const action2 = bayesFactor >= policy.passAbove ? "pass" : bayesFactor <= policy.failBelow ? "fail" : "escalate";
|
|
57
|
+
return { action: action2, rationale: "bayes-factor" };
|
|
58
|
+
}
|
|
59
|
+
invariant(
|
|
60
|
+
policy.lossFalsePass >= 0 && policy.lossFalseFail >= 0 && policy.escalationCost >= 0,
|
|
61
|
+
"INVALID_CONFIG",
|
|
62
|
+
"decision-theoretic losses must be non-negative"
|
|
63
|
+
);
|
|
64
|
+
invariant(
|
|
65
|
+
Number.isFinite(policy.lossFalsePass) && Number.isFinite(policy.lossFalseFail) && Number.isFinite(policy.escalationCost),
|
|
66
|
+
"INVALID_CONFIG",
|
|
67
|
+
"decision-theoretic losses must be finite"
|
|
68
|
+
);
|
|
69
|
+
const posterior = posteriorHighQuality(logBayesFactor, policy.priorHighQuality);
|
|
70
|
+
const expectedLoss = {
|
|
71
|
+
pass: (1 - posterior) * policy.lossFalsePass,
|
|
72
|
+
fail: posterior * policy.lossFalseFail,
|
|
73
|
+
escalate: policy.escalationCost
|
|
74
|
+
};
|
|
75
|
+
let action = "pass";
|
|
76
|
+
let best = expectedLoss.pass;
|
|
77
|
+
if (expectedLoss.fail < best) {
|
|
78
|
+
action = "fail";
|
|
79
|
+
best = expectedLoss.fail;
|
|
80
|
+
}
|
|
81
|
+
if (expectedLoss.escalate < best) {
|
|
82
|
+
action = "escalate";
|
|
83
|
+
}
|
|
84
|
+
return { action, rationale: "expected-loss", posteriorHighQuality: posterior, expectedLoss };
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
exports.decide = decide;
|
|
88
|
+
exports.posteriorHighQuality = posteriorHighQuality;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { P as PolicyConfig, i as PolicyDecision } from '../types-bMjn1j4e.cjs';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Convert a natural-log Bayes Factor and a prior probability of high quality into the posterior
|
|
5
|
+
* probability of high quality, via posterior log-odds equals log-Bayes-Factor plus prior log-odds.
|
|
6
|
+
*/
|
|
7
|
+
declare function posteriorHighQuality(logBayesFactor: number, priorHighQuality: number): number;
|
|
8
|
+
/** Apply a policy to a natural-log Bayes Factor and return the action plus its rationale. */
|
|
9
|
+
declare function decide(logBayesFactor: number, policy: PolicyConfig): PolicyDecision;
|
|
10
|
+
|
|
11
|
+
export { decide, posteriorHighQuality };
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { P as PolicyConfig, i as PolicyDecision } from '../types-bMjn1j4e.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Convert a natural-log Bayes Factor and a prior probability of high quality into the posterior
|
|
5
|
+
* probability of high quality, via posterior log-odds equals log-Bayes-Factor plus prior log-odds.
|
|
6
|
+
*/
|
|
7
|
+
declare function posteriorHighQuality(logBayesFactor: number, priorHighQuality: number): number;
|
|
8
|
+
/** Apply a policy to a natural-log Bayes Factor and return the action plus its rationale. */
|
|
9
|
+
declare function decide(logBayesFactor: number, policy: PolicyConfig): PolicyDecision;
|
|
10
|
+
|
|
11
|
+
export { decide, posteriorHighQuality };
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
// src/errors.ts
|
|
2
|
+
var BayesOutputGateError = class _BayesOutputGateError extends Error {
|
|
3
|
+
code;
|
|
4
|
+
constructor(code, message) {
|
|
5
|
+
super(message);
|
|
6
|
+
this.name = "BayesOutputGateError";
|
|
7
|
+
this.code = code;
|
|
8
|
+
Object.setPrototypeOf(this, _BayesOutputGateError.prototype);
|
|
9
|
+
}
|
|
10
|
+
};
|
|
11
|
+
function invariant(condition, code, message) {
|
|
12
|
+
if (!condition) {
|
|
13
|
+
throw new BayesOutputGateError(code, message);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
// src/policy/index.ts
|
|
18
|
+
function posteriorHighQuality(logBayesFactor, priorHighQuality) {
|
|
19
|
+
invariant(
|
|
20
|
+
priorHighQuality > 0 && priorHighQuality < 1,
|
|
21
|
+
"INVALID_CONFIG",
|
|
22
|
+
`priorHighQuality must be in (0, 1), got ${priorHighQuality}`
|
|
23
|
+
);
|
|
24
|
+
invariant(
|
|
25
|
+
Number.isFinite(logBayesFactor),
|
|
26
|
+
"NUMERIC",
|
|
27
|
+
`logBayesFactor must be finite, got ${logBayesFactor}`
|
|
28
|
+
);
|
|
29
|
+
const logPosteriorOdds = logBayesFactor + Math.log(priorHighQuality / (1 - priorHighQuality));
|
|
30
|
+
if (logPosteriorOdds >= 0) {
|
|
31
|
+
return 1 / (1 + Math.exp(-logPosteriorOdds));
|
|
32
|
+
}
|
|
33
|
+
const odds = Math.exp(logPosteriorOdds);
|
|
34
|
+
return odds / (1 + odds);
|
|
35
|
+
}
|
|
36
|
+
function decide(logBayesFactor, policy) {
|
|
37
|
+
if (policy.kind === "bayes-factor") {
|
|
38
|
+
invariant(
|
|
39
|
+
policy.passAbove >= 1,
|
|
40
|
+
"INVALID_CONFIG",
|
|
41
|
+
`passAbove must be >= 1, got ${policy.passAbove}`
|
|
42
|
+
);
|
|
43
|
+
invariant(
|
|
44
|
+
policy.failBelow > 0 && policy.failBelow <= 1,
|
|
45
|
+
"INVALID_CONFIG",
|
|
46
|
+
`failBelow must be in (0, 1], got ${policy.failBelow}`
|
|
47
|
+
);
|
|
48
|
+
invariant(
|
|
49
|
+
policy.passAbove > policy.failBelow,
|
|
50
|
+
"INVALID_CONFIG",
|
|
51
|
+
"passAbove must be greater than failBelow"
|
|
52
|
+
);
|
|
53
|
+
const bayesFactor = Math.exp(logBayesFactor);
|
|
54
|
+
const action2 = bayesFactor >= policy.passAbove ? "pass" : bayesFactor <= policy.failBelow ? "fail" : "escalate";
|
|
55
|
+
return { action: action2, rationale: "bayes-factor" };
|
|
56
|
+
}
|
|
57
|
+
invariant(
|
|
58
|
+
policy.lossFalsePass >= 0 && policy.lossFalseFail >= 0 && policy.escalationCost >= 0,
|
|
59
|
+
"INVALID_CONFIG",
|
|
60
|
+
"decision-theoretic losses must be non-negative"
|
|
61
|
+
);
|
|
62
|
+
invariant(
|
|
63
|
+
Number.isFinite(policy.lossFalsePass) && Number.isFinite(policy.lossFalseFail) && Number.isFinite(policy.escalationCost),
|
|
64
|
+
"INVALID_CONFIG",
|
|
65
|
+
"decision-theoretic losses must be finite"
|
|
66
|
+
);
|
|
67
|
+
const posterior = posteriorHighQuality(logBayesFactor, policy.priorHighQuality);
|
|
68
|
+
const expectedLoss = {
|
|
69
|
+
pass: (1 - posterior) * policy.lossFalsePass,
|
|
70
|
+
fail: posterior * policy.lossFalseFail,
|
|
71
|
+
escalate: policy.escalationCost
|
|
72
|
+
};
|
|
73
|
+
let action = "pass";
|
|
74
|
+
let best = expectedLoss.pass;
|
|
75
|
+
if (expectedLoss.fail < best) {
|
|
76
|
+
action = "fail";
|
|
77
|
+
best = expectedLoss.fail;
|
|
78
|
+
}
|
|
79
|
+
if (expectedLoss.escalate < best) {
|
|
80
|
+
action = "escalate";
|
|
81
|
+
}
|
|
82
|
+
return { action, rationale: "expected-loss", posteriorHighQuality: posterior, expectedLoss };
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export { decide, posteriorHighQuality };
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/** A quality score for one dimension of an output, always in the closed unit interval [0, 1]. */
|
|
2
|
+
interface QualityScore {
|
|
3
|
+
/** The dimension this score measures, for example "factuality" or "safety". */
|
|
4
|
+
readonly dimension: string;
|
|
5
|
+
/** The score in [0, 1], where higher means better quality. */
|
|
6
|
+
readonly value: number;
|
|
7
|
+
}
|
|
8
|
+
/** A full set of per-dimension scores for a single output. */
|
|
9
|
+
type ScoreVector = readonly QualityScore[];
|
|
10
|
+
/** The two competing hypotheses the gate weighs for every output. */
|
|
11
|
+
type HypothesisKind = "high" | "low";
|
|
12
|
+
/** Parameters of a Beta distribution over a score in [0, 1]. Both must be positive. */
|
|
13
|
+
interface BetaParams {
|
|
14
|
+
readonly a: number;
|
|
15
|
+
readonly b: number;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* The calibrated model for one quality dimension: a Beta over the scores of known-high-quality
|
|
19
|
+
* outputs, a Beta over the scores of known-low-quality outputs, and the weight this dimension
|
|
20
|
+
* carries in the combined Bayes Factor.
|
|
21
|
+
*/
|
|
22
|
+
interface DimensionModel {
|
|
23
|
+
readonly dimension: string;
|
|
24
|
+
readonly high: BetaParams;
|
|
25
|
+
readonly low: BetaParams;
|
|
26
|
+
/** Non-negative multiplicative weight on this dimension's log-Bayes-Factor. Defaults to 1. */
|
|
27
|
+
readonly weight: number;
|
|
28
|
+
}
|
|
29
|
+
/** The label of a historical observation, used to calibrate the per-hypothesis score models. */
|
|
30
|
+
interface LabeledObservation {
|
|
31
|
+
readonly scores: ScoreVector;
|
|
32
|
+
/** The known ground-truth quality of this output. */
|
|
33
|
+
readonly label: HypothesisKind;
|
|
34
|
+
}
|
|
35
|
+
/** Jeffreys-scale strength categories for a Bayes Factor (Kass and Raftery, 1995). */
|
|
36
|
+
type EvidenceStrength = "decisive-low" | "strong-low" | "substantial-low" | "inconclusive" | "substantial-high" | "strong-high" | "decisive-high";
|
|
37
|
+
/** The per-dimension contribution to the combined Bayes Factor. */
|
|
38
|
+
interface DimensionContribution {
|
|
39
|
+
readonly dimension: string;
|
|
40
|
+
readonly score: number;
|
|
41
|
+
readonly logBayesFactor: number;
|
|
42
|
+
readonly weight: number;
|
|
43
|
+
}
|
|
44
|
+
/** The result of weighing one output's scores against the two hypotheses. */
|
|
45
|
+
interface BayesFactorResult {
|
|
46
|
+
/** Natural-log Bayes Factor in favor of high quality, summed across weighted dimensions. */
|
|
47
|
+
readonly logBayesFactor: number;
|
|
48
|
+
/** The Bayes Factor itself, exp(logBayesFactor). May be Infinity for overwhelming evidence. */
|
|
49
|
+
readonly bayesFactor: number;
|
|
50
|
+
/** Which hypothesis the evidence favors. */
|
|
51
|
+
readonly favored: "high" | "low" | "inconclusive";
|
|
52
|
+
/** The Jeffreys-scale interpretation of the strength of the evidence. */
|
|
53
|
+
readonly strength: EvidenceStrength;
|
|
54
|
+
/** Per-dimension breakdown, in input order. */
|
|
55
|
+
readonly contributions: readonly DimensionContribution[];
|
|
56
|
+
}
|
|
57
|
+
/** The gate's verdict for an output. */
|
|
58
|
+
type GateAction = "pass" | "fail" | "escalate";
|
|
59
|
+
/**
|
|
60
|
+
* A decision policy expressed on the Bayes Factor scale. An output passes when the Bayes Factor is
|
|
61
|
+
* at or above `passAbove`, fails when it is at or below `failBelow`, and escalates in between.
|
|
62
|
+
*/
|
|
63
|
+
interface DecisionPolicy {
|
|
64
|
+
/** Bayes Factor at or above which the output passes. Must be >= 1. Defaults to 10 (Jeffreys strong). */
|
|
65
|
+
readonly passAbove: number;
|
|
66
|
+
/** Bayes Factor at or below which the output fails. Must be in (0, 1]. Defaults to 0.1. */
|
|
67
|
+
readonly failBelow: number;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* A pure Bayes Factor decision policy on the Jeffreys scale. An output passes when the Bayes Factor
|
|
71
|
+
* is at or above `passAbove`, fails at or below `failBelow`, and escalates in between. Evidence-only,
|
|
72
|
+
* loss-agnostic. Use {@link DecisionTheoreticPolicy} when the costs of the two error types differ.
|
|
73
|
+
*/
|
|
74
|
+
interface BayesFactorPolicy {
|
|
75
|
+
readonly kind: "bayes-factor";
|
|
76
|
+
/** Bayes Factor at or above which the output passes. Must be >= 1. */
|
|
77
|
+
readonly passAbove: number;
|
|
78
|
+
/** Bayes Factor at or below which the output fails. Must be in (0, 1]. */
|
|
79
|
+
readonly failBelow: number;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* A decision-theoretic policy. The Bayes Factor is converted to a posterior probability of high
|
|
83
|
+
* quality through an explicit prior, then the action that minimizes expected loss is chosen under an
|
|
84
|
+
* asymmetric loss: the cost of passing a low-quality output, the cost of failing a high-quality
|
|
85
|
+
* output, and the fixed cost of escalating to human review. This makes the gate utility-aware rather
|
|
86
|
+
* than a re-skinned threshold.
|
|
87
|
+
*/
|
|
88
|
+
interface DecisionTheoreticPolicy {
|
|
89
|
+
readonly kind: "decision-theoretic";
|
|
90
|
+
/** Prior probability that an output is high quality, before its scores. Must be in (0, 1). */
|
|
91
|
+
readonly priorHighQuality: number;
|
|
92
|
+
/** Loss incurred by passing an output that is actually low quality. Must be >= 0. */
|
|
93
|
+
readonly lossFalsePass: number;
|
|
94
|
+
/** Loss incurred by failing an output that is actually high quality. Must be >= 0. */
|
|
95
|
+
readonly lossFalseFail: number;
|
|
96
|
+
/** Fixed loss of escalating to human review. Must be >= 0. */
|
|
97
|
+
readonly escalationCost: number;
|
|
98
|
+
}
|
|
99
|
+
/** A gate decision policy. */
|
|
100
|
+
type PolicyConfig = BayesFactorPolicy | DecisionTheoreticPolicy;
|
|
101
|
+
/** The expected loss of each possible action, present for a decision-theoretic policy. */
|
|
102
|
+
interface ExpectedLoss {
|
|
103
|
+
readonly pass: number;
|
|
104
|
+
readonly fail: number;
|
|
105
|
+
readonly escalate: number;
|
|
106
|
+
}
|
|
107
|
+
/** The verdict of a policy applied to a Bayes Factor. */
|
|
108
|
+
interface PolicyDecision {
|
|
109
|
+
readonly action: GateAction;
|
|
110
|
+
readonly rationale: "bayes-factor" | "expected-loss";
|
|
111
|
+
/** Posterior probability of high quality, present for a decision-theoretic policy. */
|
|
112
|
+
readonly posteriorHighQuality?: number;
|
|
113
|
+
/** Expected loss per action, present for a decision-theoretic policy. */
|
|
114
|
+
readonly expectedLoss?: ExpectedLoss;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* A report on whether the gate's modeling assumptions hold for the calibration data. The Bayes
|
|
118
|
+
* Factor is only trustworthy when the per-dimension Beta fit is adequate and the dimensions are
|
|
119
|
+
* close enough to conditionally independent; this report makes both checks explicit.
|
|
120
|
+
*/
|
|
121
|
+
interface AssumptionReport {
|
|
122
|
+
/** True when every checked dimension's Beta fit is adequate (Kolmogorov-Smirnov). */
|
|
123
|
+
readonly goodnessOfFitAdequate: boolean;
|
|
124
|
+
/** True when no dimension pair crosses the dependence threshold. */
|
|
125
|
+
readonly independenceAssumptionSafe: boolean;
|
|
126
|
+
/** Dimensions whose high-quality or low-quality Beta fit was inadequate. */
|
|
127
|
+
readonly inadequateDimensions: readonly string[];
|
|
128
|
+
/** Dimension pairs flagged as dependent, each formatted as "a~b". */
|
|
129
|
+
readonly dependentPairs: readonly string[];
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Guards that let the gate refuse to trust the Bayes Factor when its modeling assumptions are
|
|
133
|
+
* violated. A precomputed {@link AssumptionReport} is attached to every decision; when a `require`
|
|
134
|
+
* flag is set and the matching assumption fails, the gate escalates to human review regardless of
|
|
135
|
+
* the Bayes Factor, because a verdict from a misspecified model should not be acted on.
|
|
136
|
+
*/
|
|
137
|
+
interface GateGuards {
|
|
138
|
+
/** A precomputed assumption report, attached to the decision and consulted by the require flags. */
|
|
139
|
+
readonly assumptions?: AssumptionReport;
|
|
140
|
+
/** When true, escalate (do not trust the Bayes Factor) if goodness-of-fit is inadequate. */
|
|
141
|
+
readonly requireGoodnessOfFit?: boolean;
|
|
142
|
+
/** When true, escalate if the conditional-independence assumption is unsafe. */
|
|
143
|
+
readonly requireIndependence?: boolean;
|
|
144
|
+
}
|
|
145
|
+
/** The full gate decision for one output, suitable for logging as compliance evidence. */
|
|
146
|
+
interface GateDecision {
|
|
147
|
+
readonly action: GateAction;
|
|
148
|
+
readonly bayesFactor: number;
|
|
149
|
+
readonly logBayesFactor: number;
|
|
150
|
+
readonly strength: EvidenceStrength;
|
|
151
|
+
readonly rationale: "bayes-factor" | "expected-loss" | "assumption-violated";
|
|
152
|
+
readonly posteriorHighQuality?: number;
|
|
153
|
+
readonly expectedLoss?: ExpectedLoss;
|
|
154
|
+
/** The assumption report, present when the gate was given one through its guards. */
|
|
155
|
+
readonly assumptions?: AssumptionReport;
|
|
156
|
+
readonly contributions: readonly DimensionContribution[];
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export type { AssumptionReport as A, BayesFactorPolicy as B, DecisionPolicy as D, EvidenceStrength as E, GateAction as G, HypothesisKind as H, LabeledObservation as L, PolicyConfig as P, QualityScore as Q, ScoreVector as S, BayesFactorResult as a, BetaParams as b, DecisionTheoreticPolicy as c, DimensionContribution as d, DimensionModel as e, ExpectedLoss as f, GateDecision as g, GateGuards as h, PolicyDecision as i };
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/** A quality score for one dimension of an output, always in the closed unit interval [0, 1]. */
|
|
2
|
+
interface QualityScore {
|
|
3
|
+
/** The dimension this score measures, for example "factuality" or "safety". */
|
|
4
|
+
readonly dimension: string;
|
|
5
|
+
/** The score in [0, 1], where higher means better quality. */
|
|
6
|
+
readonly value: number;
|
|
7
|
+
}
|
|
8
|
+
/** A full set of per-dimension scores for a single output. */
|
|
9
|
+
type ScoreVector = readonly QualityScore[];
|
|
10
|
+
/** The two competing hypotheses the gate weighs for every output. */
|
|
11
|
+
type HypothesisKind = "high" | "low";
|
|
12
|
+
/** Parameters of a Beta distribution over a score in [0, 1]. Both must be positive. */
|
|
13
|
+
interface BetaParams {
|
|
14
|
+
readonly a: number;
|
|
15
|
+
readonly b: number;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* The calibrated model for one quality dimension: a Beta over the scores of known-high-quality
|
|
19
|
+
* outputs, a Beta over the scores of known-low-quality outputs, and the weight this dimension
|
|
20
|
+
* carries in the combined Bayes Factor.
|
|
21
|
+
*/
|
|
22
|
+
interface DimensionModel {
|
|
23
|
+
readonly dimension: string;
|
|
24
|
+
readonly high: BetaParams;
|
|
25
|
+
readonly low: BetaParams;
|
|
26
|
+
/** Non-negative multiplicative weight on this dimension's log-Bayes-Factor. Defaults to 1. */
|
|
27
|
+
readonly weight: number;
|
|
28
|
+
}
|
|
29
|
+
/** The label of a historical observation, used to calibrate the per-hypothesis score models. */
|
|
30
|
+
interface LabeledObservation {
|
|
31
|
+
readonly scores: ScoreVector;
|
|
32
|
+
/** The known ground-truth quality of this output. */
|
|
33
|
+
readonly label: HypothesisKind;
|
|
34
|
+
}
|
|
35
|
+
/** Jeffreys-scale strength categories for a Bayes Factor (Kass and Raftery, 1995). */
|
|
36
|
+
type EvidenceStrength = "decisive-low" | "strong-low" | "substantial-low" | "inconclusive" | "substantial-high" | "strong-high" | "decisive-high";
|
|
37
|
+
/** The per-dimension contribution to the combined Bayes Factor. */
|
|
38
|
+
interface DimensionContribution {
|
|
39
|
+
readonly dimension: string;
|
|
40
|
+
readonly score: number;
|
|
41
|
+
readonly logBayesFactor: number;
|
|
42
|
+
readonly weight: number;
|
|
43
|
+
}
|
|
44
|
+
/** The result of weighing one output's scores against the two hypotheses. */
|
|
45
|
+
interface BayesFactorResult {
|
|
46
|
+
/** Natural-log Bayes Factor in favor of high quality, summed across weighted dimensions. */
|
|
47
|
+
readonly logBayesFactor: number;
|
|
48
|
+
/** The Bayes Factor itself, exp(logBayesFactor). May be Infinity for overwhelming evidence. */
|
|
49
|
+
readonly bayesFactor: number;
|
|
50
|
+
/** Which hypothesis the evidence favors. */
|
|
51
|
+
readonly favored: "high" | "low" | "inconclusive";
|
|
52
|
+
/** The Jeffreys-scale interpretation of the strength of the evidence. */
|
|
53
|
+
readonly strength: EvidenceStrength;
|
|
54
|
+
/** Per-dimension breakdown, in input order. */
|
|
55
|
+
readonly contributions: readonly DimensionContribution[];
|
|
56
|
+
}
|
|
57
|
+
/** The gate's verdict for an output. */
|
|
58
|
+
type GateAction = "pass" | "fail" | "escalate";
|
|
59
|
+
/**
|
|
60
|
+
* A decision policy expressed on the Bayes Factor scale. An output passes when the Bayes Factor is
|
|
61
|
+
* at or above `passAbove`, fails when it is at or below `failBelow`, and escalates in between.
|
|
62
|
+
*/
|
|
63
|
+
interface DecisionPolicy {
|
|
64
|
+
/** Bayes Factor at or above which the output passes. Must be >= 1. Defaults to 10 (Jeffreys strong). */
|
|
65
|
+
readonly passAbove: number;
|
|
66
|
+
/** Bayes Factor at or below which the output fails. Must be in (0, 1]. Defaults to 0.1. */
|
|
67
|
+
readonly failBelow: number;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* A pure Bayes Factor decision policy on the Jeffreys scale. An output passes when the Bayes Factor
|
|
71
|
+
* is at or above `passAbove`, fails at or below `failBelow`, and escalates in between. Evidence-only,
|
|
72
|
+
* loss-agnostic. Use {@link DecisionTheoreticPolicy} when the costs of the two error types differ.
|
|
73
|
+
*/
|
|
74
|
+
interface BayesFactorPolicy {
|
|
75
|
+
readonly kind: "bayes-factor";
|
|
76
|
+
/** Bayes Factor at or above which the output passes. Must be >= 1. */
|
|
77
|
+
readonly passAbove: number;
|
|
78
|
+
/** Bayes Factor at or below which the output fails. Must be in (0, 1]. */
|
|
79
|
+
readonly failBelow: number;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* A decision-theoretic policy. The Bayes Factor is converted to a posterior probability of high
|
|
83
|
+
* quality through an explicit prior, then the action that minimizes expected loss is chosen under an
|
|
84
|
+
* asymmetric loss: the cost of passing a low-quality output, the cost of failing a high-quality
|
|
85
|
+
* output, and the fixed cost of escalating to human review. This makes the gate utility-aware rather
|
|
86
|
+
* than a re-skinned threshold.
|
|
87
|
+
*/
|
|
88
|
+
interface DecisionTheoreticPolicy {
|
|
89
|
+
readonly kind: "decision-theoretic";
|
|
90
|
+
/** Prior probability that an output is high quality, before its scores. Must be in (0, 1). */
|
|
91
|
+
readonly priorHighQuality: number;
|
|
92
|
+
/** Loss incurred by passing an output that is actually low quality. Must be >= 0. */
|
|
93
|
+
readonly lossFalsePass: number;
|
|
94
|
+
/** Loss incurred by failing an output that is actually high quality. Must be >= 0. */
|
|
95
|
+
readonly lossFalseFail: number;
|
|
96
|
+
/** Fixed loss of escalating to human review. Must be >= 0. */
|
|
97
|
+
readonly escalationCost: number;
|
|
98
|
+
}
|
|
99
|
+
/** A gate decision policy. */
|
|
100
|
+
type PolicyConfig = BayesFactorPolicy | DecisionTheoreticPolicy;
|
|
101
|
+
/** The expected loss of each possible action, present for a decision-theoretic policy. */
|
|
102
|
+
interface ExpectedLoss {
|
|
103
|
+
readonly pass: number;
|
|
104
|
+
readonly fail: number;
|
|
105
|
+
readonly escalate: number;
|
|
106
|
+
}
|
|
107
|
+
/** The verdict of a policy applied to a Bayes Factor. */
|
|
108
|
+
interface PolicyDecision {
|
|
109
|
+
readonly action: GateAction;
|
|
110
|
+
readonly rationale: "bayes-factor" | "expected-loss";
|
|
111
|
+
/** Posterior probability of high quality, present for a decision-theoretic policy. */
|
|
112
|
+
readonly posteriorHighQuality?: number;
|
|
113
|
+
/** Expected loss per action, present for a decision-theoretic policy. */
|
|
114
|
+
readonly expectedLoss?: ExpectedLoss;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* A report on whether the gate's modeling assumptions hold for the calibration data. The Bayes
|
|
118
|
+
* Factor is only trustworthy when the per-dimension Beta fit is adequate and the dimensions are
|
|
119
|
+
* close enough to conditionally independent; this report makes both checks explicit.
|
|
120
|
+
*/
|
|
121
|
+
interface AssumptionReport {
|
|
122
|
+
/** True when every checked dimension's Beta fit is adequate (Kolmogorov-Smirnov). */
|
|
123
|
+
readonly goodnessOfFitAdequate: boolean;
|
|
124
|
+
/** True when no dimension pair crosses the dependence threshold. */
|
|
125
|
+
readonly independenceAssumptionSafe: boolean;
|
|
126
|
+
/** Dimensions whose high-quality or low-quality Beta fit was inadequate. */
|
|
127
|
+
readonly inadequateDimensions: readonly string[];
|
|
128
|
+
/** Dimension pairs flagged as dependent, each formatted as "a~b". */
|
|
129
|
+
readonly dependentPairs: readonly string[];
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Guards that let the gate refuse to trust the Bayes Factor when its modeling assumptions are
|
|
133
|
+
* violated. A precomputed {@link AssumptionReport} is attached to every decision; when a `require`
|
|
134
|
+
* flag is set and the matching assumption fails, the gate escalates to human review regardless of
|
|
135
|
+
* the Bayes Factor, because a verdict from a misspecified model should not be acted on.
|
|
136
|
+
*/
|
|
137
|
+
interface GateGuards {
|
|
138
|
+
/** A precomputed assumption report, attached to the decision and consulted by the require flags. */
|
|
139
|
+
readonly assumptions?: AssumptionReport;
|
|
140
|
+
/** When true, escalate (do not trust the Bayes Factor) if goodness-of-fit is inadequate. */
|
|
141
|
+
readonly requireGoodnessOfFit?: boolean;
|
|
142
|
+
/** When true, escalate if the conditional-independence assumption is unsafe. */
|
|
143
|
+
readonly requireIndependence?: boolean;
|
|
144
|
+
}
|
|
145
|
+
/** The full gate decision for one output, suitable for logging as compliance evidence. */
|
|
146
|
+
interface GateDecision {
|
|
147
|
+
readonly action: GateAction;
|
|
148
|
+
readonly bayesFactor: number;
|
|
149
|
+
readonly logBayesFactor: number;
|
|
150
|
+
readonly strength: EvidenceStrength;
|
|
151
|
+
readonly rationale: "bayes-factor" | "expected-loss" | "assumption-violated";
|
|
152
|
+
readonly posteriorHighQuality?: number;
|
|
153
|
+
readonly expectedLoss?: ExpectedLoss;
|
|
154
|
+
/** The assumption report, present when the gate was given one through its guards. */
|
|
155
|
+
readonly assumptions?: AssumptionReport;
|
|
156
|
+
readonly contributions: readonly DimensionContribution[];
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export type { AssumptionReport as A, BayesFactorPolicy as B, DecisionPolicy as D, EvidenceStrength as E, GateAction as G, HypothesisKind as H, LabeledObservation as L, PolicyConfig as P, QualityScore as Q, ScoreVector as S, BayesFactorResult as a, BetaParams as b, DecisionTheoreticPolicy as c, DimensionContribution as d, DimensionModel as e, ExpectedLoss as f, GateDecision as g, GateGuards as h, PolicyDecision as i };
|
package/package.json
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@takk/bayesoutputgate",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "BayesOutputGate: calibrated quality scoring with the Bayes Factor for output validation. Feed the quality scores of an output across one or more dimensions and get a calibrated pass, fail, or escalate decision on the Jeffreys scale instead of an arbitrary threshold. A zero-runtime-dependency, TypeScript-first output-validation gate for Massive Intelligence (IM) infrastructure and non-human entities, with a principled Bayes Factor between a high-quality and a low-quality hypothesis, calibrated likelihoods that update online, multi-dimension scoring (factuality, fluency, safety, relevance), measured calibration (reliability, Brier, ECE), a tamper-evident audit trail, and a node-free core that runs in Node, edge runtimes, and the browser, with optional peers @takk/keymesh and @takk/modelchain.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "Apache-2.0",
|
|
7
|
+
"author": "David C Cavalcante <davcavalcante@proton.me> (https://takk.ag)",
|
|
8
|
+
"homepage": "https://github.com/davccavalcante/bayesoutputgate",
|
|
9
|
+
"repository": {
|
|
10
|
+
"type": "git",
|
|
11
|
+
"url": "git+https://github.com/davccavalcante/bayesoutputgate.git"
|
|
12
|
+
},
|
|
13
|
+
"bugs": {
|
|
14
|
+
"url": "https://github.com/davccavalcante/bayesoutputgate/issues"
|
|
15
|
+
},
|
|
16
|
+
"keywords": [
|
|
17
|
+
"bayes-factor",
|
|
18
|
+
"output-validation",
|
|
19
|
+
"quality-gate",
|
|
20
|
+
"calibration",
|
|
21
|
+
"llm-as-judge",
|
|
22
|
+
"hallucination-detection",
|
|
23
|
+
"guardrails",
|
|
24
|
+
"bayesian",
|
|
25
|
+
"eu-ai-act",
|
|
26
|
+
"compliance",
|
|
27
|
+
"mcp",
|
|
28
|
+
"edge",
|
|
29
|
+
"llm",
|
|
30
|
+
"nodejs",
|
|
31
|
+
"zero-dependency",
|
|
32
|
+
"typescript"
|
|
33
|
+
],
|
|
34
|
+
"engines": {
|
|
35
|
+
"node": ">=20.0.0"
|
|
36
|
+
},
|
|
37
|
+
"packageManager": "pnpm@10.34.1",
|
|
38
|
+
"sideEffects": false,
|
|
39
|
+
"bin": {
|
|
40
|
+
"bayesoutputgate": "./dist/cli/index.js"
|
|
41
|
+
},
|
|
42
|
+
"files": [
|
|
43
|
+
"dist",
|
|
44
|
+
"README.md",
|
|
45
|
+
"CHANGELOG.md",
|
|
46
|
+
"LICENSE",
|
|
47
|
+
"NOTICE",
|
|
48
|
+
"SPEC.md",
|
|
49
|
+
"SECURITY.md"
|
|
50
|
+
],
|
|
51
|
+
"exports": {
|
|
52
|
+
".": {
|
|
53
|
+
"import": { "types": "./dist/index.d.ts", "default": "./dist/index.js" },
|
|
54
|
+
"require": { "types": "./dist/index.d.cts", "default": "./dist/index.cjs" }
|
|
55
|
+
},
|
|
56
|
+
"./beta": {
|
|
57
|
+
"import": { "types": "./dist/beta/index.d.ts", "default": "./dist/beta/index.js" },
|
|
58
|
+
"require": { "types": "./dist/beta/index.d.cts", "default": "./dist/beta/index.cjs" }
|
|
59
|
+
},
|
|
60
|
+
"./hypothesis": {
|
|
61
|
+
"import": { "types": "./dist/hypothesis/index.d.ts", "default": "./dist/hypothesis/index.js" },
|
|
62
|
+
"require": { "types": "./dist/hypothesis/index.d.cts", "default": "./dist/hypothesis/index.cjs" }
|
|
63
|
+
},
|
|
64
|
+
"./likelihood": {
|
|
65
|
+
"import": { "types": "./dist/likelihood/index.d.ts", "default": "./dist/likelihood/index.js" },
|
|
66
|
+
"require": { "types": "./dist/likelihood/index.d.cts", "default": "./dist/likelihood/index.cjs" }
|
|
67
|
+
},
|
|
68
|
+
"./bayesfactor": {
|
|
69
|
+
"import": { "types": "./dist/bayesfactor/index.d.ts", "default": "./dist/bayesfactor/index.js" },
|
|
70
|
+
"require": { "types": "./dist/bayesfactor/index.d.cts", "default": "./dist/bayesfactor/index.cjs" }
|
|
71
|
+
},
|
|
72
|
+
"./dimensions": {
|
|
73
|
+
"import": { "types": "./dist/dimensions/index.d.ts", "default": "./dist/dimensions/index.js" },
|
|
74
|
+
"require": { "types": "./dist/dimensions/index.d.cts", "default": "./dist/dimensions/index.cjs" }
|
|
75
|
+
},
|
|
76
|
+
"./policy": {
|
|
77
|
+
"import": { "types": "./dist/policy/index.d.ts", "default": "./dist/policy/index.js" },
|
|
78
|
+
"require": { "types": "./dist/policy/index.d.cts", "default": "./dist/policy/index.cjs" }
|
|
79
|
+
},
|
|
80
|
+
"./calibration": {
|
|
81
|
+
"import": { "types": "./dist/calibration/index.d.ts", "default": "./dist/calibration/index.js" },
|
|
82
|
+
"require": { "types": "./dist/calibration/index.d.cts", "default": "./dist/calibration/index.cjs" }
|
|
83
|
+
},
|
|
84
|
+
"./gate": {
|
|
85
|
+
"import": { "types": "./dist/gate/index.d.ts", "default": "./dist/gate/index.js" },
|
|
86
|
+
"require": { "types": "./dist/gate/index.d.cts", "default": "./dist/gate/index.cjs" }
|
|
87
|
+
},
|
|
88
|
+
"./audit": {
|
|
89
|
+
"import": { "types": "./dist/audit/index.d.ts", "default": "./dist/audit/index.js" },
|
|
90
|
+
"require": { "types": "./dist/audit/index.d.cts", "default": "./dist/audit/index.cjs" }
|
|
91
|
+
},
|
|
92
|
+
"./adapter": {
|
|
93
|
+
"import": { "types": "./dist/adapter/index.d.ts", "default": "./dist/adapter/index.js" },
|
|
94
|
+
"require": { "types": "./dist/adapter/index.d.cts", "default": "./dist/adapter/index.cjs" }
|
|
95
|
+
},
|
|
96
|
+
"./node": {
|
|
97
|
+
"import": { "types": "./dist/node/index.d.ts", "default": "./dist/node/index.js" },
|
|
98
|
+
"require": { "types": "./dist/node/index.d.cts", "default": "./dist/node/index.cjs" }
|
|
99
|
+
},
|
|
100
|
+
"./edge": {
|
|
101
|
+
"import": { "types": "./dist/edge/index.d.ts", "default": "./dist/edge/index.js" },
|
|
102
|
+
"require": { "types": "./dist/edge/index.d.cts", "default": "./dist/edge/index.cjs" }
|
|
103
|
+
},
|
|
104
|
+
"./package.json": "./package.json"
|
|
105
|
+
},
|
|
106
|
+
"scripts": {
|
|
107
|
+
"build": "tsup",
|
|
108
|
+
"typecheck": "tsc --noEmit",
|
|
109
|
+
"test": "vitest run",
|
|
110
|
+
"test:coverage": "vitest run --coverage",
|
|
111
|
+
"lint": "biome check .",
|
|
112
|
+
"lint:fix": "biome check --write .",
|
|
113
|
+
"format": "biome format --write .",
|
|
114
|
+
"smoke:dist": "node scripts/smoke-dist.mjs",
|
|
115
|
+
"publint": "publint",
|
|
116
|
+
"attw": "attw --pack . --profile node16",
|
|
117
|
+
"size": "size-limit",
|
|
118
|
+
"verify": "pnpm run lint && pnpm run typecheck && pnpm run test && pnpm run build && pnpm run smoke:dist && pnpm run publint",
|
|
119
|
+
"prepublishOnly": "pnpm run build"
|
|
120
|
+
},
|
|
121
|
+
"peerDependencies": {
|
|
122
|
+
"@takk/keymesh": ">=1.0.0",
|
|
123
|
+
"@takk/modelchain": ">=1.0.0"
|
|
124
|
+
},
|
|
125
|
+
"peerDependenciesMeta": {
|
|
126
|
+
"@takk/keymesh": { "optional": true },
|
|
127
|
+
"@takk/modelchain": { "optional": true }
|
|
128
|
+
},
|
|
129
|
+
"devDependencies": {
|
|
130
|
+
"@arethetypeswrong/cli": "^0.18.4",
|
|
131
|
+
"@biomejs/biome": "^2.5.1",
|
|
132
|
+
"@size-limit/preset-small-lib": "^12.1.0",
|
|
133
|
+
"@types/node": "^25.0.0",
|
|
134
|
+
"publint": "^0.3.21",
|
|
135
|
+
"size-limit": "^12.1.0",
|
|
136
|
+
"tsup": "^8.5.1",
|
|
137
|
+
"tsx": "^4.22.4",
|
|
138
|
+
"typescript": "^6.0.3",
|
|
139
|
+
"vitest": "^4.1.9",
|
|
140
|
+
"@vitest/coverage-v8": "^4.1.9"
|
|
141
|
+
}
|
|
142
|
+
}
|