dsh-jev-tools-continued 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.en.md +449 -0
- package/CHANGELOG.i18n.yaml +13 -0
- package/CHANGELOG.md +357 -0
- package/LICENSE +21 -0
- package/README.en.md +210 -0
- package/README.i18n.yaml +13 -0
- package/README.md +209 -2
- package/client/client.js +755 -0
- package/cordis.patch.yml +30 -0
- package/docs/banner.png +0 -0
- package/docs/dev-workflow.md +414 -0
- package/docs/s0-trigger-rate.md +168 -0
- package/icon.svg +16 -0
- package/lib/backends/jev.d.ts +69 -0
- package/lib/backends/jev.d.ts.map +1 -0
- package/lib/backends/jev.js +151 -0
- package/lib/backends/jev.js.map +1 -0
- package/lib/backends/types.d.ts +104 -0
- package/lib/backends/types.d.ts.map +1 -0
- package/lib/backends/types.js +12 -0
- package/lib/backends/types.js.map +1 -0
- package/lib/budget.d.ts +60 -0
- package/lib/budget.d.ts.map +1 -0
- package/lib/budget.js +70 -0
- package/lib/budget.js.map +1 -0
- package/lib/calibrate.d.ts +162 -0
- package/lib/calibrate.d.ts.map +1 -0
- package/lib/calibrate.js +269 -0
- package/lib/calibrate.js.map +1 -0
- package/lib/config.d.ts +261 -0
- package/lib/config.d.ts.map +1 -0
- package/lib/config.js +194 -0
- package/lib/config.js.map +1 -0
- package/lib/context-cache.d.ts +59 -0
- package/lib/context-cache.d.ts.map +1 -0
- package/lib/context-cache.js +116 -0
- package/lib/context-cache.js.map +1 -0
- package/lib/credentials.d.ts +32 -0
- package/lib/credentials.d.ts.map +1 -0
- package/lib/credentials.js +44 -0
- package/lib/credentials.js.map +1 -0
- package/lib/degrade.d.ts +36 -0
- package/lib/degrade.d.ts.map +1 -0
- package/lib/degrade.js +41 -0
- package/lib/degrade.js.map +1 -0
- package/lib/features/prune.d.ts +116 -0
- package/lib/features/prune.d.ts.map +1 -0
- package/lib/features/prune.js +574 -0
- package/lib/features/prune.js.map +1 -0
- package/lib/features/screen.d.ts +112 -0
- package/lib/features/screen.d.ts.map +1 -0
- package/lib/features/screen.js +130 -0
- package/lib/features/screen.js.map +1 -0
- package/lib/features/skill-suggest.d.ts +68 -0
- package/lib/features/skill-suggest.d.ts.map +1 -0
- package/lib/features/skill-suggest.js +283 -0
- package/lib/features/skill-suggest.js.map +1 -0
- package/lib/host.d.ts +280 -0
- package/lib/host.d.ts.map +1 -0
- package/lib/host.js +15 -0
- package/lib/host.js.map +1 -0
- package/lib/http.d.ts +71 -0
- package/lib/http.d.ts.map +1 -0
- package/lib/http.js +162 -0
- package/lib/http.js.map +1 -0
- package/lib/i18n.d.ts +372 -0
- package/lib/i18n.d.ts.map +1 -0
- package/lib/i18n.js +254 -0
- package/lib/i18n.js.map +1 -0
- package/lib/index.d.ts +44 -0
- package/lib/index.d.ts.map +1 -0
- package/lib/index.js +268 -0
- package/lib/index.js.map +1 -0
- package/lib/keys.d.ts +90 -0
- package/lib/keys.d.ts.map +1 -0
- package/lib/keys.js +131 -0
- package/lib/keys.js.map +1 -0
- package/lib/ledger-domain.d.ts +169 -0
- package/lib/ledger-domain.d.ts.map +1 -0
- package/lib/ledger-domain.js +338 -0
- package/lib/ledger-domain.js.map +1 -0
- package/lib/ledger.d.ts +243 -0
- package/lib/ledger.d.ts.map +1 -0
- package/lib/ledger.js +239 -0
- package/lib/ledger.js.map +1 -0
- package/lib/memo.d.ts +32 -0
- package/lib/memo.d.ts.map +1 -0
- package/lib/memo.js +62 -0
- package/lib/memo.js.map +1 -0
- package/lib/notify.d.ts +78 -0
- package/lib/notify.d.ts.map +1 -0
- package/lib/notify.js +122 -0
- package/lib/notify.js.map +1 -0
- package/lib/proxy.d.ts +40 -0
- package/lib/proxy.d.ts.map +1 -0
- package/lib/proxy.js +136 -0
- package/lib/proxy.js.map +1 -0
- package/lib/request.d.ts +99 -0
- package/lib/request.d.ts.map +1 -0
- package/lib/request.js +168 -0
- package/lib/request.js.map +1 -0
- package/lib/source.d.ts +46 -0
- package/lib/source.d.ts.map +1 -0
- package/lib/source.js +35 -0
- package/lib/source.js.map +1 -0
- package/lib/status.d.ts +42 -0
- package/lib/status.d.ts.map +1 -0
- package/lib/status.js +214 -0
- package/lib/status.js.map +1 -0
- package/lib/tokens.d.ts +26 -0
- package/lib/tokens.d.ts.map +1 -0
- package/lib/tokens.js +57 -0
- package/lib/tokens.js.map +1 -0
- package/lib/tools/jev-ask.d.ts +44 -0
- package/lib/tools/jev-ask.d.ts.map +1 -0
- package/lib/tools/jev-ask.js +208 -0
- package/lib/tools/jev-ask.js.map +1 -0
- package/lib/tools/jev-gate.d.ts +79 -0
- package/lib/tools/jev-gate.d.ts.map +1 -0
- package/lib/tools/jev-gate.js +412 -0
- package/lib/tools/jev-gate.js.map +1 -0
- package/package.json +169 -4
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"calibrate.d.ts","sourceRoot":"","sources":["../src/calibrate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,8EAA8E;AAC9E,MAAM,WAAW,MAAM;IACrB,mDAAmD;IACnD,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;IAClB,oEAAoE;IACpE,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAA;CAClB;AAED,0CAA0C;AAC1C,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,iEAAiE;AACjE,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,kCAAkC;IAClC,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAA;IACrC,8CAA8C;IAC9C,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,YAAY,EAAE,CAAA;IACxC,uEAAuE;IACvE,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,CAAA;IACnB,0CAA0C;IAC1C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;CAC1B;AAED,yCAAyC;AACzC,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;IAClB,iFAAiF;IACjF,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAA;IAC5B,uCAAuC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,iBAAiB,CAAA;CAClE;AAYD;;;;;;;;;;GAUG;AACH,wBAAgB,WAAW,CAAE,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,YAAY,EAAE,CAuBvE;AAED;;;;;GAKG;AACH,wBAAgB,QAAQ,CAAE,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAA;CAAE,CAkB9E;AAED;;;;;;;GAOG;AACH,wBAAgB,GAAG,CACjB,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,KAAK,EAAE,MAAM,EACb,GAAG,GAAE,MAAM,MAAiB,GAC3B,iBAAiB,CAOnB;AAqBD;;;;;;;;;;;GAWG;AACH,wBAAgB,KAAK,CACnB,MAAM,EAAE,iBAAiB,GAAG,SAAS,EACrC,CAAC,EAAE,MAAM,EACT,KAAK,EAAE,MAAM,GACZ,UAAU,CAWZ;AAED;;;;;;;;;GASG;AACH,wBAAgB,wBAAwB,CAAE,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAe5E;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,GAAG,CAAE,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAmBvD;AAED,oEAAoE;AACpE,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,0DAA0D;IAC1D,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,iEAAiE;IACjE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,sDAAsD;IACtD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;CAC/B;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,eAAe,CAAE,OAAO,EAAE,SAAS,MAAM,EAAE,EAAE,KAAK,SAAK,GAAG,cAAc,EAAE,CAqBzF;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAE,MAAM,EAAE,SAAS,cAAc,EAAE,GAAG,cAAc,GAAG,SAAS,CAMjG"}
|
package/lib/calibrate.js
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Probability calibration.
|
|
3
|
+
*
|
|
4
|
+
* Jev's raw numbers are not usable as probabilities, and the two independent
|
|
5
|
+
* observations point in opposite directions depending on difficulty: on a hard
|
|
6
|
+
* detection task its scores run *low* (a monitoring experiment found it "quite
|
|
7
|
+
* shy"), while on an easy three-way classification it saturates at 1.000. So
|
|
8
|
+
* neither a fixed threshold nor the raw value travels between tasks.
|
|
9
|
+
*
|
|
10
|
+
* What does travel is a *fitted* mapping on your own labelled data. This module
|
|
11
|
+
* fits one and applies it — and binds the result to the model version that
|
|
12
|
+
* answered, because an alias such as `jev-latest` moves between releases and a
|
|
13
|
+
* mapping fitted on one version is meaningless on the next.
|
|
14
|
+
*
|
|
15
|
+
* v0.1 does not use this: pruning only *ranks*, and ranking needs no
|
|
16
|
+
* calibration. It exists so the thresholds a gate will need are a fit away
|
|
17
|
+
* rather than a guess, and so the runtime that produced the samples is on
|
|
18
|
+
* record.
|
|
19
|
+
*
|
|
20
|
+
* @module dsh-jev-tools/calibrate
|
|
21
|
+
*/
|
|
22
|
+
/** Below this many samples isotonic regression overfits; Platt is the safer fit. */
|
|
23
|
+
const ISOTONIC_MIN_SAMPLES = 30;
|
|
24
|
+
/** Gradient-descent settings for the Platt fit. */
|
|
25
|
+
const PLATT_EPOCHS = 2_000;
|
|
26
|
+
const PLATT_RATE = 0.5;
|
|
27
|
+
/** Clamp to the unit interval. */
|
|
28
|
+
const clamp01 = (value) => Math.min(1, Math.max(0, value));
|
|
29
|
+
/**
|
|
30
|
+
* Fit by pool-adjacent-violators.
|
|
31
|
+
*
|
|
32
|
+
* Produces the best monotone non-decreasing step function under squared error —
|
|
33
|
+
* which is exactly the shape wanted, because a monotone mapping can reorder
|
|
34
|
+
* nothing: it changes the *values* while leaving the ranking that pruning
|
|
35
|
+
* depends on completely intact.
|
|
36
|
+
*
|
|
37
|
+
* @param samples - the labelled observations.
|
|
38
|
+
* @returns ascending knots.
|
|
39
|
+
*/
|
|
40
|
+
export function fitIsotonic(samples) {
|
|
41
|
+
const sorted = [...samples].sort((left, right) => left.p - right.p);
|
|
42
|
+
const blocks = [];
|
|
43
|
+
for (const sample of sorted) {
|
|
44
|
+
blocks.push({ sum: sample.y, weight: 1, max: sample.p });
|
|
45
|
+
// Pool while the previous block's mean exceeds this one's.
|
|
46
|
+
while (blocks.length > 1) {
|
|
47
|
+
const last = blocks[blocks.length - 1];
|
|
48
|
+
const previous = blocks[blocks.length - 2];
|
|
49
|
+
if (previous.sum / previous.weight <= last.sum / last.weight)
|
|
50
|
+
break;
|
|
51
|
+
blocks.pop();
|
|
52
|
+
blocks.pop();
|
|
53
|
+
blocks.push({
|
|
54
|
+
sum: previous.sum + last.sum,
|
|
55
|
+
weight: previous.weight + last.weight,
|
|
56
|
+
max: last.max,
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return blocks.map(block => ({ x: block.max, y: block.sum / block.weight }));
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Fit a logistic mapping by gradient descent.
|
|
64
|
+
*
|
|
65
|
+
* @param samples - the labelled observations.
|
|
66
|
+
* @returns the sigmoid coefficients.
|
|
67
|
+
*/
|
|
68
|
+
export function fitPlatt(samples) {
|
|
69
|
+
let a = 1;
|
|
70
|
+
let b = 0;
|
|
71
|
+
if (samples.length === 0)
|
|
72
|
+
return { a, b };
|
|
73
|
+
for (let epoch = 0; epoch < PLATT_EPOCHS; epoch += 1) {
|
|
74
|
+
let gradA = 0;
|
|
75
|
+
let gradB = 0;
|
|
76
|
+
for (const sample of samples) {
|
|
77
|
+
const predicted = 1 / (1 + Math.exp(-(a * sample.p + b)));
|
|
78
|
+
const error = predicted - sample.y;
|
|
79
|
+
gradA += error * sample.p;
|
|
80
|
+
gradB += error;
|
|
81
|
+
}
|
|
82
|
+
a -= (PLATT_RATE * gradA) / samples.length;
|
|
83
|
+
b -= (PLATT_RATE * gradB) / samples.length;
|
|
84
|
+
}
|
|
85
|
+
return { a, b };
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Fit a mapping from labelled data.
|
|
89
|
+
*
|
|
90
|
+
* @param samples - the labelled observations.
|
|
91
|
+
* @param model - the model version they came from; recorded, never inferred.
|
|
92
|
+
* @param now - clock injection, for deterministic tests.
|
|
93
|
+
* @returns the fitted parameters.
|
|
94
|
+
*/
|
|
95
|
+
export function fit(samples, model, now = Date.now) {
|
|
96
|
+
const base = { model, samples: samples.length, fittedAt: now() };
|
|
97
|
+
if (samples.length >= ISOTONIC_MIN_SAMPLES) {
|
|
98
|
+
return { ...base, method: 'isotonic', knots: fitIsotonic(samples) };
|
|
99
|
+
}
|
|
100
|
+
const { a, b } = fitPlatt(samples);
|
|
101
|
+
return { ...base, method: 'platt', a, b };
|
|
102
|
+
}
|
|
103
|
+
/** Interpolate an isotonic fit at `p`, clamping outside the fitted range. */
|
|
104
|
+
function interpolate(knots, p) {
|
|
105
|
+
if (knots.length === 0)
|
|
106
|
+
return p;
|
|
107
|
+
const first = knots[0];
|
|
108
|
+
const last = knots[knots.length - 1];
|
|
109
|
+
if (p <= first.x)
|
|
110
|
+
return first.y;
|
|
111
|
+
if (p >= last.x)
|
|
112
|
+
return last.y;
|
|
113
|
+
for (let i = 1; i < knots.length; i += 1) {
|
|
114
|
+
const high = knots[i];
|
|
115
|
+
if (p > high.x)
|
|
116
|
+
continue;
|
|
117
|
+
const low = knots[i - 1];
|
|
118
|
+
const span = high.x - low.x;
|
|
119
|
+
if (span <= 0)
|
|
120
|
+
return high.y;
|
|
121
|
+
const ratio = (p - low.x) / span;
|
|
122
|
+
return low.y + ratio * (high.y - low.y);
|
|
123
|
+
}
|
|
124
|
+
return last.y;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Apply a mapping to a raw probability.
|
|
128
|
+
*
|
|
129
|
+
* Refuses to apply a mapping fitted on a different model version: aliases move
|
|
130
|
+
* between releases, and silently reusing a stale mapping would be worse than
|
|
131
|
+
* returning the raw value, because nothing downstream could tell.
|
|
132
|
+
*
|
|
133
|
+
* @param params - the fitted mapping, if any.
|
|
134
|
+
* @param p - the raw probability.
|
|
135
|
+
* @param model - the version that produced `p`.
|
|
136
|
+
* @returns the mapped value and whether the mapping was actually used.
|
|
137
|
+
*/
|
|
138
|
+
export function apply(params, p, model) {
|
|
139
|
+
if (params === undefined)
|
|
140
|
+
return { p, calibrated: false, reason: 'none' };
|
|
141
|
+
if (params.model !== model)
|
|
142
|
+
return { p, calibrated: false, reason: 'version-mismatch' };
|
|
143
|
+
if (params.samples === 0)
|
|
144
|
+
return { p, calibrated: false, reason: 'too-few-samples' };
|
|
145
|
+
if (params.method === 'isotonic') {
|
|
146
|
+
return { p: clamp01(interpolate(params.knots ?? [], p)), calibrated: true };
|
|
147
|
+
}
|
|
148
|
+
const a = params.a ?? 1;
|
|
149
|
+
const b = params.b ?? 0;
|
|
150
|
+
return { p: clamp01(1 / (1 + Math.exp(-(a * p + b)))), calibrated: true };
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Mean absolute gap between predicted and observed, in ten equal-width bins.
|
|
154
|
+
*
|
|
155
|
+
* This is the number a threshold decision should be based on. A well-calibrated
|
|
156
|
+
* model scores near zero; a model that always says 1.000 on a task it gets
|
|
157
|
+
* right 70% of the time scores around 0.3.
|
|
158
|
+
*
|
|
159
|
+
* @param samples - the labelled observations, already mapped if that is the question.
|
|
160
|
+
* @returns expected calibration error in [0, 1].
|
|
161
|
+
*/
|
|
162
|
+
export function expectedCalibrationError(samples) {
|
|
163
|
+
if (samples.length === 0)
|
|
164
|
+
return 0;
|
|
165
|
+
const bins = 10;
|
|
166
|
+
let total = 0;
|
|
167
|
+
for (let bin = 0; bin < bins; bin += 1) {
|
|
168
|
+
const low = bin / bins;
|
|
169
|
+
const high = (bin + 1) / bins;
|
|
170
|
+
const inBin = samples.filter(sample => sample.p >= low && (bin === bins - 1 ? sample.p <= high : sample.p < high));
|
|
171
|
+
if (inBin.length === 0)
|
|
172
|
+
continue;
|
|
173
|
+
const meanPredicted = inBin.reduce((sum, s) => sum + s.p, 0) / inBin.length;
|
|
174
|
+
const observed = inBin.reduce((sum, s) => sum + s.y, 0) / inBin.length;
|
|
175
|
+
total += (inBin.length / samples.length) * Math.abs(meanPredicted - observed);
|
|
176
|
+
}
|
|
177
|
+
return total;
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Area under the ROC curve, by rank.
|
|
181
|
+
*
|
|
182
|
+
* This is the Mann–Whitney U statistic, and it answers the one question a
|
|
183
|
+
* calibration error cannot: **can this model tell the two classes apart at
|
|
184
|
+
* all?** A perfectly calibrated but useless model scores 0.5 here, and a
|
|
185
|
+
* saturated-but-informative one can score high while its ECE is terrible.
|
|
186
|
+
*
|
|
187
|
+
* Ties are given their average rank, so a model that answers a constant gets
|
|
188
|
+
* 0.5 rather than a number that depends on the input order.
|
|
189
|
+
*
|
|
190
|
+
* @param samples - the labelled observations.
|
|
191
|
+
* @returns AUC in [0, 1], or 0.5 when one class is absent — nothing can be
|
|
192
|
+
* ranked, and 0.5 is the value that says so instead of implying a skill.
|
|
193
|
+
*/
|
|
194
|
+
export function auc(samples) {
|
|
195
|
+
const positives = samples.filter(sample => sample.y === 1).length;
|
|
196
|
+
const negatives = samples.length - positives;
|
|
197
|
+
if (positives === 0 || negatives === 0)
|
|
198
|
+
return 0.5;
|
|
199
|
+
const sorted = [...samples].sort((left, right) => left.p - right.p);
|
|
200
|
+
let rankSum = 0;
|
|
201
|
+
let index = 0;
|
|
202
|
+
while (index < sorted.length) {
|
|
203
|
+
// One tie group: every equal score shares the group's average rank.
|
|
204
|
+
let last = index;
|
|
205
|
+
while (last + 1 < sorted.length && sorted[last + 1].p === sorted[index].p)
|
|
206
|
+
last += 1;
|
|
207
|
+
const averageRank = (index + last) / 2 + 1;
|
|
208
|
+
for (let at = index; at <= last; at += 1) {
|
|
209
|
+
if (sorted[at].y === 1)
|
|
210
|
+
rankSum += averageRank;
|
|
211
|
+
}
|
|
212
|
+
index = last + 1;
|
|
213
|
+
}
|
|
214
|
+
return (rankSum - (positives * (positives + 1)) / 2) / (positives * negatives);
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Score every threshold in a sweep.
|
|
218
|
+
*
|
|
219
|
+
* This is the step between "the model is calibrated" and "here is the cut-off
|
|
220
|
+
* to use": pruning only ranks and needs no threshold, but a gate does, and the
|
|
221
|
+
* sweep is what turns the choice into a measurement rather than a preference.
|
|
222
|
+
*
|
|
223
|
+
* @param samples - the labelled observations.
|
|
224
|
+
* @param steps - how many thresholds to try, spread strictly inside (0, 1).
|
|
225
|
+
* @returns one point per threshold, ascending.
|
|
226
|
+
*/
|
|
227
|
+
export function sweepThresholds(samples, steps = 19) {
|
|
228
|
+
const points = [];
|
|
229
|
+
for (let step = 1; step <= steps; step += 1) {
|
|
230
|
+
const threshold = step / (steps + 1);
|
|
231
|
+
let truePositives = 0;
|
|
232
|
+
let falsePositives = 0;
|
|
233
|
+
let falseNegatives = 0;
|
|
234
|
+
for (const sample of samples) {
|
|
235
|
+
const called = sample.p >= threshold;
|
|
236
|
+
if (called && sample.y === 1)
|
|
237
|
+
truePositives += 1;
|
|
238
|
+
else if (called)
|
|
239
|
+
falsePositives += 1;
|
|
240
|
+
else if (sample.y === 1)
|
|
241
|
+
falseNegatives += 1;
|
|
242
|
+
}
|
|
243
|
+
const predicted = truePositives + falsePositives;
|
|
244
|
+
const precision = predicted === 0 ? 0 : truePositives / predicted;
|
|
245
|
+
const actualPositives = truePositives + falseNegatives;
|
|
246
|
+
const recall = actualPositives === 0 ? 0 : truePositives / actualPositives;
|
|
247
|
+
const f1 = precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall);
|
|
248
|
+
points.push({ threshold, precision, recall, f1, predicted, truePositives });
|
|
249
|
+
}
|
|
250
|
+
return points;
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* The sweep's best F1.
|
|
254
|
+
*
|
|
255
|
+
* A tie goes to the **higher** threshold: at the same F1 the stricter cut-off
|
|
256
|
+
* spends less of the reader's attention on the cases that do not need it.
|
|
257
|
+
*
|
|
258
|
+
* @param points - a sweep, ascending in threshold.
|
|
259
|
+
* @returns the chosen point, or `undefined` for an empty sweep.
|
|
260
|
+
*/
|
|
261
|
+
export function bestOperatingPoint(points) {
|
|
262
|
+
let best;
|
|
263
|
+
for (const point of points) {
|
|
264
|
+
if (best === undefined || point.f1 >= best.f1 - 1e-12)
|
|
265
|
+
best = point;
|
|
266
|
+
}
|
|
267
|
+
return best;
|
|
268
|
+
}
|
|
269
|
+
//# sourceMappingURL=calibrate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"calibrate.js","sourceRoot":"","sources":["../src/calibrate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAyCH,oFAAoF;AACpF,MAAM,oBAAoB,GAAG,EAAE,CAAA;AAE/B,mDAAmD;AACnD,MAAM,YAAY,GAAG,KAAK,CAAA;AAC1B,MAAM,UAAU,GAAG,GAAG,CAAA;AAEtB,kCAAkC;AAClC,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAA;AAE1E;;;;;;;;;;GAUG;AACH,MAAM,UAAU,WAAW,CAAE,OAA0B;IACrD,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IAEnE,MAAM,MAAM,GAAY,EAAE,CAAA;IAE1B,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAA;QACxD,2DAA2D;QAC3D,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,CAAA;YACvC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,CAAA;YAC3C,IAAI,QAAQ,CAAC,GAAG,GAAG,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM;gBAAE,MAAK;YACnE,MAAM,CAAC,GAAG,EAAE,CAAA;YACZ,MAAM,CAAC,GAAG,EAAE,CAAA;YACZ,MAAM,CAAC,IAAI,CAAC;gBACV,GAAG,EAAE,QAAQ,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG;gBAC5B,MAAM,EAAE,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM;gBACrC,GAAG,EAAE,IAAI,CAAC,GAAG;aACd,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;AAC7E,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,QAAQ,CAAE,OAA0B;IAClD,IAAI,CAAC,GAAG,CAAC,CAAA;IACT,IAAI,CAAC,GAAG,CAAC,CAAA;IACT,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,CAAA;IAEzC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,YAAY,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACrD,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;YACzD,MAAM,KAAK,GAAG,SAAS,GAAG,MAAM,CAAC,CAAC,CAAA;YAClC,KAAK,IAAI,KAAK,GAAG,MAAM,CAAC,CAAC,CAAA;YACzB,KAAK,IAAI,KAAK,CAAA;QAChB,CAAC;QACD,CAAC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,GAAG,OAAO,CAAC,MAAM,CAAA;QAC1C,CAAC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,GAAG,OAAO,CAAC,MAAM,CAAA;IAC5C,CAAC;IACD,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,CAAA;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,GAAG,CACjB,OAA0B,EAC1B,KAAa,EACb,GAAG,GAAiB,IAAI,CAAC,GAAG;IAE5B,MAAM,IAAI,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,CAAA;IAChE,IAAI,OAAO,CAAC,MAAM,IAAI,oBAAoB,EAAE,CAAC;QAC3C,OAAO,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAA;IACrE,CAAC;IACD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAA;IAClC,OAAO,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,CAAA;AAC3C,CAAC;AAED,6EAA6E;AAC7E,SAAS,WAAW,CAAE,KAA8B,EAAE,CAAS;IAC7D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAA;IAChC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;IACvB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAE,CAAA;IACrC,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC,CAAC,CAAA;IAChC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,CAAA;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACtB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;YAAE,SAAQ;QACxB,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAA;QAC3B,IAAI,IAAI,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,CAAA;QAC5B,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAA;QAChC,OAAO,GAAG,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAA;IACzC,CAAC;IACD,OAAO,IAAI,CAAC,CAAC,CAAA;AACf,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,KAAK,CACnB,MAAqC,EACrC,CAAS,EACT,KAAa;IAEb,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;IACzE,IAAI,MAAM,CAAC,KAAK,KAAK,KAAK;QAAE,OAAO,EAAE,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAA;IACvF,IAAI,MAAM,CAAC,OAAO,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAA;IAEpF,IAAI,MAAM,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QACjC,OAAO,EAAE,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;IAC7E,CAAC;IACD,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAA;IACvB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAA;IACvB,OAAO,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;AAC3E,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,wBAAwB,CAAE,OAA0B;IAClE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAA;IAClC,MAAM,IAAI,GAAG,EAAE,CAAA;IACf,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,CAAA;QACtB,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAA;QAC7B,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CACpC,MAAM,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;QAC7E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAQ;QAChC,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAA;QAC3E,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAA;QACtE,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,GAAG,QAAQ,CAAC,CAAA;IAC/E,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,GAAG,CAAE,OAA0B;IAC7C,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAA;IACjE,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAA;IAC5C,IAAI,SAAS,KAAK,CAAC,IAAI,SAAS,KAAK,CAAC;QAAE,OAAO,GAAG,CAAA;IAElD,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IACnE,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,OAAO,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QAC7B,oEAAoE;QACpE,IAAI,IAAI,GAAG,KAAK,CAAA;QAChB,OAAO,IAAI,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,CAAE,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAE,CAAC,CAAC;YAAE,IAAI,IAAI,CAAC,CAAA;QACtF,MAAM,WAAW,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC1C,KAAK,IAAI,EAAE,GAAG,KAAK,EAAE,EAAE,IAAI,IAAI,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,EAAE,CAAE,CAAC,CAAC,KAAK,CAAC;gBAAE,OAAO,IAAI,WAAW,CAAA;QACjD,CAAC;QACD,KAAK,GAAG,IAAI,GAAG,CAAC,CAAA;IAClB,CAAC;IACD,OAAO,CAAC,OAAO,GAAG,CAAC,SAAS,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,SAAS,GAAG,SAAS,CAAC,CAAA;AAChF,CAAC;AAeD;;;;;;;;;;GAUG;AACH,MAAM,UAAU,eAAe,CAAE,OAA0B,EAAE,KAAK,GAAG,EAAE;IACrE,MAAM,MAAM,GAAqB,EAAE,CAAA;IACnC,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,IAAI,KAAK,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAA;QACpC,IAAI,aAAa,GAAG,CAAC,CAAA;QACrB,IAAI,cAAc,GAAG,CAAC,CAAA;QACtB,IAAI,cAAc,GAAG,CAAC,CAAA;QACtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,IAAI,SAAS,CAAA;YACpC,IAAI,MAAM,IAAI,MAAM,CAAC,CAAC,KAAK,CAAC;gBAAE,aAAa,IAAI,CAAC,CAAA;iBAC3C,IAAI,MAAM;gBAAE,cAAc,IAAI,CAAC,CAAA;iBAC/B,IAAI,MAAM,CAAC,CAAC,KAAK,CAAC;gBAAE,cAAc,IAAI,CAAC,CAAA;QAC9C,CAAC;QACD,MAAM,SAAS,GAAG,aAAa,GAAG,cAAc,CAAA;QAChD,MAAM,SAAS,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,SAAS,CAAA;QACjE,MAAM,eAAe,GAAG,aAAa,GAAG,cAAc,CAAA;QACtD,MAAM,MAAM,GAAG,eAAe,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,eAAe,CAAA;QAC1E,MAAM,EAAE,GAAG,SAAS,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAA;QACzF,MAAM,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC,CAAA;IAC7E,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,kBAAkB,CAAE,MAAiC;IACnE,IAAI,IAAgC,CAAA;IACpC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,GAAG,KAAK;YAAE,IAAI,GAAG,KAAK,CAAA;IACrE,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC"}
|
package/lib/config.d.ts
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Plugin settings schema.
|
|
3
|
+
*
|
|
4
|
+
* Every default here is a *measured* decision, not a guess — see
|
|
5
|
+
* `docs/s0-trigger-rate.md` for the numbers behind `minTokens`, `perTurnLimit`
|
|
6
|
+
* and `minCatalogSize`.
|
|
7
|
+
*
|
|
8
|
+
* The settings *namespace* is not declared here: DSH derives it from the mount
|
|
9
|
+
* row id (`cordis.patch.yml`), which therefore has to equal this package's name —
|
|
10
|
+
* that is the address the bundle's own card reads and writes.
|
|
11
|
+
*
|
|
12
|
+
* @module dsh-jev-tools/config
|
|
13
|
+
*/
|
|
14
|
+
import Schema from '@deepseek-ai/schemastery';
|
|
15
|
+
/** Where a user creates a TypeSafe API key. Shown as a link in the settings card. */
|
|
16
|
+
export declare const TYPESAFE_KEYS_URL = "https://console.typesafe.ai/keys";
|
|
17
|
+
/**
|
|
18
|
+
* Default System One API root, as a bare host.
|
|
19
|
+
*
|
|
20
|
+
* The path (`/v1/systemone`) is appended by the backend, so this must not carry
|
|
21
|
+
* one. It is a *default*, not the only endpoint: `backends/jev.ts` has always
|
|
22
|
+
* accepted an override, so a self-hosted Jev-compatible server — or a deployment
|
|
23
|
+
* that puts a gateway in front — needs the endpoint to be a setting rather than a
|
|
24
|
+
* constant. Declaring the default once, here, is also what lets the settings
|
|
25
|
+
* schema use exactly the value the backend falls back to.
|
|
26
|
+
*/
|
|
27
|
+
export declare const DEFAULT_BASE_URL = "https://api.typesafe.ai";
|
|
28
|
+
/**
|
|
29
|
+
* Default credential reference.
|
|
30
|
+
*
|
|
31
|
+
* Deliberately the same environment variable the official TypeSafe SDK reads,
|
|
32
|
+
* so anyone already using Jev with the official tooling works with no setup.
|
|
33
|
+
*/
|
|
34
|
+
export declare const DEFAULT_API_KEY_ENV = "TYPESAFE_API_KEY";
|
|
35
|
+
/**
|
|
36
|
+
* Tools whose bulk output is worth judging for relevance.
|
|
37
|
+
*
|
|
38
|
+
* Measured: at `minTokens` 2000 these covered 76% of oversized results.
|
|
39
|
+
* `pwsh` is deliberately absent — terminal output carries build logs, error
|
|
40
|
+
* traces and file listings whose "irrelevant" parts are often exactly what a
|
|
41
|
+
* debugging step needs, so a cheap judgment must not trim them.
|
|
42
|
+
*/
|
|
43
|
+
export declare const DEFAULT_TOOL_ALLOWLIST: string[];
|
|
44
|
+
/**
|
|
45
|
+
* Tools whose output is **not** the user's own material.
|
|
46
|
+
*
|
|
47
|
+
* Screening exists because fetched content is untrusted input: a page can carry
|
|
48
|
+
* text aimed at the agent reading it. Local files are deliberately excluded by
|
|
49
|
+
* default — they are the user's own material, they are read far more often, and
|
|
50
|
+
* screening them would spend the per-turn budget that pruning needs. A user who
|
|
51
|
+
* clones untrusted repositories can add `read` themselves.
|
|
52
|
+
*/
|
|
53
|
+
export declare const DEFAULT_SCREEN_ALLOWLIST: string[];
|
|
54
|
+
/** Resolved settings shape, mirroring {@link Config}. */
|
|
55
|
+
export interface JevSettings {
|
|
56
|
+
/** Master switch. Off means both capabilities are inert, exactly like a missing key. */
|
|
57
|
+
enabled: boolean;
|
|
58
|
+
/** Environment-variable names the key is resolved from, re-resolved on every operation. */
|
|
59
|
+
apiKeyEnv: string;
|
|
60
|
+
/** Network exit: empty for direct, `system` for the OS proxy, or a proxy URL. */
|
|
61
|
+
proxy: string;
|
|
62
|
+
/** System One API root, a bare host. The vendor's own path is appended to it. */
|
|
63
|
+
baseUrl: string;
|
|
64
|
+
/** Jev model id, an alias or a pinned version. The response's own `model` is what gets recorded. */
|
|
65
|
+
model: string;
|
|
66
|
+
/** Hard ceiling on judgments per session, across every capability. */
|
|
67
|
+
sessionCallLimit: number;
|
|
68
|
+
prune: {
|
|
69
|
+
enabled: boolean;
|
|
70
|
+
/** Estimated-token floor below which a result is not worth judging. */
|
|
71
|
+
minTokens: number;
|
|
72
|
+
/** Per-turn judgment ceiling. Measured: uncapped worst turn was 27 triggers ≈ 8.1s. */
|
|
73
|
+
perTurnLimit: number;
|
|
74
|
+
toolAllowlist: string[];
|
|
75
|
+
/** Deterministic floor: leading lines always kept. */
|
|
76
|
+
headLines: number;
|
|
77
|
+
/** Deterministic floor: trailing lines always kept. */
|
|
78
|
+
tailLines: number;
|
|
79
|
+
/** Probability at or above which a segment is kept unconditionally. */
|
|
80
|
+
keepHigh: number;
|
|
81
|
+
/** Never reduce a result below this share of its original size. */
|
|
82
|
+
minKeepRatio: number;
|
|
83
|
+
/** Give up (return the original) when the achievable saving is below this share. */
|
|
84
|
+
minSaving: number;
|
|
85
|
+
/** Give up when the current request is shorter than this, in characters. */
|
|
86
|
+
minTaskChars: number;
|
|
87
|
+
/**
|
|
88
|
+
* Judge as usual but change nothing, so a user can see what pruning would
|
|
89
|
+
* have removed before trusting it with their context.
|
|
90
|
+
*/
|
|
91
|
+
shadow: boolean;
|
|
92
|
+
};
|
|
93
|
+
suggest: {
|
|
94
|
+
enabled: boolean;
|
|
95
|
+
/** Only suggest when the catalog is at least this large. Measured: p50 was 29. */
|
|
96
|
+
minCatalogSize: number;
|
|
97
|
+
/** Below this probability, stay silent rather than add noise. */
|
|
98
|
+
minConfidence: number;
|
|
99
|
+
};
|
|
100
|
+
screen: {
|
|
101
|
+
enabled: boolean;
|
|
102
|
+
/** Below this estimated token count there is not enough text to carry an injection. */
|
|
103
|
+
minTokens: number;
|
|
104
|
+
/** Injection probability at or above which an advisory warning is attached. */
|
|
105
|
+
threshold: number;
|
|
106
|
+
toolAllowlist: string[];
|
|
107
|
+
};
|
|
108
|
+
ledger: {
|
|
109
|
+
enabled: boolean;
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Composition-entry schema.
|
|
114
|
+
*
|
|
115
|
+
* Also the declaration a configuration surface dispatches on — and **only fields
|
|
116
|
+
* marked `.volatile()` reach that surface**. `SettingsForms.describe()` derives
|
|
117
|
+
* the form through `volatileForm()`, which returns nothing when a schema has no
|
|
118
|
+
* volatile field, and a write to a non-volatile path is refused. A schema without
|
|
119
|
+
* a single `.volatile()` therefore publishes **no namespace at all**: the bundle's
|
|
120
|
+
* card reads nothing, every toggle stays disabled and both endpoint fields stay
|
|
121
|
+
* blank, with no error printed anywhere.
|
|
122
|
+
*
|
|
123
|
+
* So the fields this bundle's own card draws are volatile: the master switch, the
|
|
124
|
+
* key reference, the endpoint, the model, and the two capability toggles.
|
|
125
|
+
* `test/bundle-contract.test.ts` keeps that set honest. Everything else stays an
|
|
126
|
+
* ordinary composition setting, editable by hand in the profile patch.
|
|
127
|
+
*/
|
|
128
|
+
export declare const Config: Schema<Schemastery.ObjectS<NoInfer<{
|
|
129
|
+
enabled: Schema<boolean, boolean, "volatile-defined">;
|
|
130
|
+
apiKeyEnv: Schema<string, string, "volatile-defined">;
|
|
131
|
+
proxy: Schema<string, string, "volatile-defined">;
|
|
132
|
+
baseUrl: Schema<string, string, "volatile-defined">;
|
|
133
|
+
model: Schema<string, string, "volatile-defined">;
|
|
134
|
+
sessionCallLimit: Schema<number, number, "defined">;
|
|
135
|
+
prune: Schema<Schemastery.ObjectS<NoInfer<{
|
|
136
|
+
enabled: Schema<boolean, boolean, "volatile-defined">;
|
|
137
|
+
minTokens: Schema<number, number, "defined">;
|
|
138
|
+
perTurnLimit: Schema<number, number, "defined">;
|
|
139
|
+
toolAllowlist: Schema<string[], string[], "defined">;
|
|
140
|
+
headLines: Schema<number, number, "defined">;
|
|
141
|
+
tailLines: Schema<number, number, "defined">;
|
|
142
|
+
keepHigh: Schema<number, number, "defined">;
|
|
143
|
+
minKeepRatio: Schema<number, number, "defined">;
|
|
144
|
+
minSaving: Schema<number, number, "defined">;
|
|
145
|
+
minTaskChars: Schema<number, number, "defined">;
|
|
146
|
+
shadow: Schema<boolean, boolean, "defined">;
|
|
147
|
+
}>>, Schemastery.ObjectT<NoInfer<{
|
|
148
|
+
enabled: Schema<boolean, boolean, "volatile-defined">;
|
|
149
|
+
minTokens: Schema<number, number, "defined">;
|
|
150
|
+
perTurnLimit: Schema<number, number, "defined">;
|
|
151
|
+
toolAllowlist: Schema<string[], string[], "defined">;
|
|
152
|
+
headLines: Schema<number, number, "defined">;
|
|
153
|
+
tailLines: Schema<number, number, "defined">;
|
|
154
|
+
keepHigh: Schema<number, number, "defined">;
|
|
155
|
+
minKeepRatio: Schema<number, number, "defined">;
|
|
156
|
+
minSaving: Schema<number, number, "defined">;
|
|
157
|
+
minTaskChars: Schema<number, number, "defined">;
|
|
158
|
+
shadow: Schema<boolean, boolean, "defined">;
|
|
159
|
+
}>>, "plain">;
|
|
160
|
+
suggest: Schema<Schemastery.ObjectS<NoInfer<{
|
|
161
|
+
enabled: Schema<boolean, boolean, "volatile-defined">;
|
|
162
|
+
minCatalogSize: Schema<number, number, "defined">;
|
|
163
|
+
minConfidence: Schema<number, number, "defined">;
|
|
164
|
+
}>>, Schemastery.ObjectT<NoInfer<{
|
|
165
|
+
enabled: Schema<boolean, boolean, "volatile-defined">;
|
|
166
|
+
minCatalogSize: Schema<number, number, "defined">;
|
|
167
|
+
minConfidence: Schema<number, number, "defined">;
|
|
168
|
+
}>>, "plain">;
|
|
169
|
+
screen: Schema<Schemastery.ObjectS<NoInfer<{
|
|
170
|
+
enabled: Schema<boolean, boolean, "defined">;
|
|
171
|
+
minTokens: Schema<number, number, "defined">;
|
|
172
|
+
threshold: Schema<number, number, "defined">;
|
|
173
|
+
toolAllowlist: Schema<string[], string[], "defined">;
|
|
174
|
+
}>>, Schemastery.ObjectT<NoInfer<{
|
|
175
|
+
enabled: Schema<boolean, boolean, "defined">;
|
|
176
|
+
minTokens: Schema<number, number, "defined">;
|
|
177
|
+
threshold: Schema<number, number, "defined">;
|
|
178
|
+
toolAllowlist: Schema<string[], string[], "defined">;
|
|
179
|
+
}>>, "plain">;
|
|
180
|
+
ledger: Schema<Schemastery.ObjectS<NoInfer<{
|
|
181
|
+
enabled: Schema<boolean, boolean, "defined">;
|
|
182
|
+
}>>, Schemastery.ObjectT<NoInfer<{
|
|
183
|
+
enabled: Schema<boolean, boolean, "defined">;
|
|
184
|
+
}>>, "plain">;
|
|
185
|
+
}>>, Schemastery.ObjectT<NoInfer<{
|
|
186
|
+
enabled: Schema<boolean, boolean, "volatile-defined">;
|
|
187
|
+
apiKeyEnv: Schema<string, string, "volatile-defined">;
|
|
188
|
+
proxy: Schema<string, string, "volatile-defined">;
|
|
189
|
+
baseUrl: Schema<string, string, "volatile-defined">;
|
|
190
|
+
model: Schema<string, string, "volatile-defined">;
|
|
191
|
+
sessionCallLimit: Schema<number, number, "defined">;
|
|
192
|
+
prune: Schema<Schemastery.ObjectS<NoInfer<{
|
|
193
|
+
enabled: Schema<boolean, boolean, "volatile-defined">;
|
|
194
|
+
minTokens: Schema<number, number, "defined">;
|
|
195
|
+
perTurnLimit: Schema<number, number, "defined">;
|
|
196
|
+
toolAllowlist: Schema<string[], string[], "defined">;
|
|
197
|
+
headLines: Schema<number, number, "defined">;
|
|
198
|
+
tailLines: Schema<number, number, "defined">;
|
|
199
|
+
keepHigh: Schema<number, number, "defined">;
|
|
200
|
+
minKeepRatio: Schema<number, number, "defined">;
|
|
201
|
+
minSaving: Schema<number, number, "defined">;
|
|
202
|
+
minTaskChars: Schema<number, number, "defined">;
|
|
203
|
+
shadow: Schema<boolean, boolean, "defined">;
|
|
204
|
+
}>>, Schemastery.ObjectT<NoInfer<{
|
|
205
|
+
enabled: Schema<boolean, boolean, "volatile-defined">;
|
|
206
|
+
minTokens: Schema<number, number, "defined">;
|
|
207
|
+
perTurnLimit: Schema<number, number, "defined">;
|
|
208
|
+
toolAllowlist: Schema<string[], string[], "defined">;
|
|
209
|
+
headLines: Schema<number, number, "defined">;
|
|
210
|
+
tailLines: Schema<number, number, "defined">;
|
|
211
|
+
keepHigh: Schema<number, number, "defined">;
|
|
212
|
+
minKeepRatio: Schema<number, number, "defined">;
|
|
213
|
+
minSaving: Schema<number, number, "defined">;
|
|
214
|
+
minTaskChars: Schema<number, number, "defined">;
|
|
215
|
+
shadow: Schema<boolean, boolean, "defined">;
|
|
216
|
+
}>>, "plain">;
|
|
217
|
+
suggest: Schema<Schemastery.ObjectS<NoInfer<{
|
|
218
|
+
enabled: Schema<boolean, boolean, "volatile-defined">;
|
|
219
|
+
minCatalogSize: Schema<number, number, "defined">;
|
|
220
|
+
minConfidence: Schema<number, number, "defined">;
|
|
221
|
+
}>>, Schemastery.ObjectT<NoInfer<{
|
|
222
|
+
enabled: Schema<boolean, boolean, "volatile-defined">;
|
|
223
|
+
minCatalogSize: Schema<number, number, "defined">;
|
|
224
|
+
minConfidence: Schema<number, number, "defined">;
|
|
225
|
+
}>>, "plain">;
|
|
226
|
+
screen: Schema<Schemastery.ObjectS<NoInfer<{
|
|
227
|
+
enabled: Schema<boolean, boolean, "defined">;
|
|
228
|
+
minTokens: Schema<number, number, "defined">;
|
|
229
|
+
threshold: Schema<number, number, "defined">;
|
|
230
|
+
toolAllowlist: Schema<string[], string[], "defined">;
|
|
231
|
+
}>>, Schemastery.ObjectT<NoInfer<{
|
|
232
|
+
enabled: Schema<boolean, boolean, "defined">;
|
|
233
|
+
minTokens: Schema<number, number, "defined">;
|
|
234
|
+
threshold: Schema<number, number, "defined">;
|
|
235
|
+
toolAllowlist: Schema<string[], string[], "defined">;
|
|
236
|
+
}>>, "plain">;
|
|
237
|
+
ledger: Schema<Schemastery.ObjectS<NoInfer<{
|
|
238
|
+
enabled: Schema<boolean, boolean, "defined">;
|
|
239
|
+
}>>, Schemastery.ObjectT<NoInfer<{
|
|
240
|
+
enabled: Schema<boolean, boolean, "defined">;
|
|
241
|
+
}>>, "plain">;
|
|
242
|
+
}>>, "plain">;
|
|
243
|
+
/**
|
|
244
|
+
* Resolve a raw composition entry against the schema.
|
|
245
|
+
*
|
|
246
|
+
* `apply` receives whatever the patch declared, which may be `undefined` when
|
|
247
|
+
* the row carries no `config:`.
|
|
248
|
+
*
|
|
249
|
+
* The **standard-schema** path is used on purpose: it is the one the Loader
|
|
250
|
+
* itself takes (`resolveConfig` in cordis is exactly
|
|
251
|
+
* `Config['~standard'].validate(value)` → `result.value`). Either path hands back
|
|
252
|
+
* a `.volatile()` field as a marker object rather than the value it denotes, so
|
|
253
|
+
* {@link plainConfig} unwraps it on the way in **and** on the way out — which
|
|
254
|
+
* makes this function idempotent, because the Loader hands `apply` the config it
|
|
255
|
+
* already resolved (markers included).
|
|
256
|
+
*
|
|
257
|
+
* @param entry - the composition entry, raw or already resolved by the Loader.
|
|
258
|
+
* @returns the fully resolved settings.
|
|
259
|
+
*/
|
|
260
|
+
export declare function resolveSettings(entry: unknown): JevSettings;
|
|
261
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,MAAM,MAAM,0BAA0B,CAAA;AA6B7C,qFAAqF;AACrF,eAAO,MAAM,iBAAiB,qCAAqC,CAAA;AAEnE;;;;;;;;;GASG;AACH,eAAO,MAAM,gBAAgB,4BAA4B,CAAA;AAEzD;;;;;GAKG;AACH,eAAO,MAAM,mBAAmB,qBAAqB,CAAA;AAErD;;;;;;;GAOG;AACH,eAAO,MAAM,sBAAsB,UAAsD,CAAA;AAEzF;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB,UAA8B,CAAA;AAEnE,yDAAyD;AACzD,MAAM,WAAW,WAAW;IAC1B,wFAAwF;IACxF,OAAO,EAAE,OAAO,CAAA;IAChB,2FAA2F;IAC3F,SAAS,EAAE,MAAM,CAAA;IACjB,iFAAiF;IACjF,KAAK,EAAE,MAAM,CAAA;IACb,iFAAiF;IACjF,OAAO,EAAE,MAAM,CAAA;IACf,oGAAoG;IACpG,KAAK,EAAE,MAAM,CAAA;IACb,sEAAsE;IACtE,gBAAgB,EAAE,MAAM,CAAA;IACxB,KAAK,EAAE;QACL,OAAO,EAAE,OAAO,CAAA;QAChB,uEAAuE;QACvE,SAAS,EAAE,MAAM,CAAA;QACjB,uFAAuF;QACvF,YAAY,EAAE,MAAM,CAAA;QACpB,aAAa,EAAE,MAAM,EAAE,CAAA;QACvB,sDAAsD;QACtD,SAAS,EAAE,MAAM,CAAA;QACjB,uDAAuD;QACvD,SAAS,EAAE,MAAM,CAAA;QACjB,uEAAuE;QACvE,QAAQ,EAAE,MAAM,CAAA;QAChB,mEAAmE;QACnE,YAAY,EAAE,MAAM,CAAA;QACpB,oFAAoF;QACpF,SAAS,EAAE,MAAM,CAAA;QACjB,4EAA4E;QAC5E,YAAY,EAAE,MAAM,CAAA;QACpB;;;WAGG;QACH,MAAM,EAAE,OAAO,CAAA;KAChB,CAAA;IACD,OAAO,EAAE;QACP,OAAO,EAAE,OAAO,CAAA;QAChB,kFAAkF;QAClF,cAAc,EAAE,MAAM,CAAA;QACtB,iEAAiE;QACjE,aAAa,EAAE,MAAM,CAAA;KACtB,CAAA;IACD,MAAM,EAAE;QACN,OAAO,EAAE,OAAO,CAAA;QAChB,uFAAuF;QACvF,SAAS,EAAE,MAAM,CAAA;QACjB,+EAA+E;QAC/E,SAAS,EAAE,MAAM,CAAA;QACjB,aAAa,EAAE,MAAM,EAAE,CAAA;KACxB,CAAA;IACD,MAAM,EAAE;QACN,OAAO,EAAE,OAAO,CAAA;KACjB,CAAA;CACF;AAED;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;aA0EjB,CAAA;AAEF;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,eAAe,CAAE,KAAK,EAAE,OAAO,GAAG,WAAW,CAQ5D"}
|