harnessgap 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +160 -0
- package/dist/adapter/correction.d.ts +14 -0
- package/dist/adapter/correction.js +68 -0
- package/dist/adapter/correction.js.map +1 -0
- package/dist/adapter/parse.d.ts +21 -0
- package/dist/adapter/parse.js +261 -0
- package/dist/adapter/parse.js.map +1 -0
- package/dist/adapter/scrub.d.ts +15 -0
- package/dist/adapter/scrub.js +97 -0
- package/dist/adapter/scrub.js.map +1 -0
- package/dist/adapter/stream.d.ts +14 -0
- package/dist/adapter/stream.js +207 -0
- package/dist/adapter/stream.js.map +1 -0
- package/dist/adapter/taxonomy.d.ts +7 -0
- package/dist/adapter/taxonomy.js +21 -0
- package/dist/adapter/taxonomy.js.map +1 -0
- package/dist/aggregate/leaderboard.d.ts +26 -0
- package/dist/aggregate/leaderboard.js +168 -0
- package/dist/aggregate/leaderboard.js.map +1 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +67 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.d.ts +25 -0
- package/dist/config.js +191 -0
- package/dist/config.js.map +1 -0
- package/dist/detector/areas.d.ts +13 -0
- package/dist/detector/areas.js +95 -0
- package/dist/detector/areas.js.map +1 -0
- package/dist/detector/index.d.ts +14 -0
- package/dist/detector/index.js +31 -0
- package/dist/detector/index.js.map +1 -0
- package/dist/detector/record.d.ts +24 -0
- package/dist/detector/record.js +29 -0
- package/dist/detector/record.js.map +1 -0
- package/dist/detector/scoring.d.ts +21 -0
- package/dist/detector/scoring.js +140 -0
- package/dist/detector/scoring.js.map +1 -0
- package/dist/detector/signals.d.ts +6 -0
- package/dist/detector/signals.js +221 -0
- package/dist/detector/signals.js.map +1 -0
- package/dist/egress.d.ts +8 -0
- package/dist/egress.js +37 -0
- package/dist/egress.js.map +1 -0
- package/dist/git.d.ts +16 -0
- package/dist/git.js +59 -0
- package/dist/git.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -0
- package/dist/output/calibrate.d.ts +35 -0
- package/dist/output/calibrate.js +149 -0
- package/dist/output/calibrate.js.map +1 -0
- package/dist/output/human.d.ts +24 -0
- package/dist/output/human.js +83 -0
- package/dist/output/human.js.map +1 -0
- package/dist/output/json.d.ts +17 -0
- package/dist/output/json.js +22 -0
- package/dist/output/json.js.map +1 -0
- package/dist/pipeline.d.ts +24 -0
- package/dist/pipeline.js +164 -0
- package/dist/pipeline.js.map +1 -0
- package/dist/types.d.ts +122 -0
- package/dist/types.js +4 -0
- package/dist/types.js.map +1 -0
- package/dist/walk.d.ts +12 -0
- package/dist/walk.js +92 -0
- package/dist/walk.js.map +1 -0
- package/package.json +49 -0
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
// Pure detector signal computation. Each SignalValues field is computed by a
|
|
2
|
+
// small named helper; `computeSignals` assembles them. No I/O, no mutation.
|
|
3
|
+
const EXPLORE_TOOLS = new Set(['search', 'read', 'list']);
|
|
4
|
+
/**
|
|
5
|
+
* Compute all detector signals for a normalized event stream. Pure: no I/O,
|
|
6
|
+
* no mutation of inputs. See field contracts in `types.ts` / the design spec.
|
|
7
|
+
*/
|
|
8
|
+
export function computeSignals(events, cfg) {
|
|
9
|
+
return {
|
|
10
|
+
explore_ratio: computeExploreRatio(events),
|
|
11
|
+
reread: computeReread(events, cfg),
|
|
12
|
+
failure_streak: computeFailureStreak(events),
|
|
13
|
+
corrections: computeCorrections(events, cfg),
|
|
14
|
+
abandonment: computeAbandonment(events, cfg),
|
|
15
|
+
oscillation: computeOscillation(events, cfg),
|
|
16
|
+
wall_clock_per_line_ms: computeWallClockPerLine(events),
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* (search + read + list tool_calls) / totalEditedLines. `null` when no edit
|
|
21
|
+
* lines were produced (does not contribute to scoring).
|
|
22
|
+
*/
|
|
23
|
+
function computeExploreRatio(events) {
|
|
24
|
+
let exploreCount = 0;
|
|
25
|
+
let totalEditedLines = 0;
|
|
26
|
+
for (const e of events) {
|
|
27
|
+
if (e.kind !== 'tool_call' || e.tool === null)
|
|
28
|
+
continue;
|
|
29
|
+
if (EXPLORE_TOOLS.has(e.tool)) {
|
|
30
|
+
exploreCount += 1;
|
|
31
|
+
}
|
|
32
|
+
else if (e.tool === 'edit') {
|
|
33
|
+
totalEditedLines += e.input_digest.lines_changed ?? 0;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
if (totalEditedLines === 0)
|
|
37
|
+
return null;
|
|
38
|
+
return exploreCount / totalEditedLines;
|
|
39
|
+
}
|
|
40
|
+
/** Distinct file paths read >= `reread_threshold` times across read tool_calls. */
|
|
41
|
+
function computeReread(events, cfg) {
|
|
42
|
+
const threshold = cfg.detector.reread_threshold;
|
|
43
|
+
const counts = new Map();
|
|
44
|
+
for (const e of events) {
|
|
45
|
+
if (e.kind !== 'tool_call' || e.tool !== 'read')
|
|
46
|
+
continue;
|
|
47
|
+
for (const f of e.input_digest.files) {
|
|
48
|
+
counts.set(f, (counts.get(f) ?? 0) + 1);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
let reread = 0;
|
|
52
|
+
for (const c of counts.values()) {
|
|
53
|
+
if (c >= threshold)
|
|
54
|
+
reread += 1;
|
|
55
|
+
}
|
|
56
|
+
return reread;
|
|
57
|
+
}
|
|
58
|
+
/** Longest run of consecutive `exec` tool_calls with `ok === false`. 0 if none. */
|
|
59
|
+
function computeFailureStreak(events) {
|
|
60
|
+
let max = 0;
|
|
61
|
+
let cur = 0;
|
|
62
|
+
for (const e of events) {
|
|
63
|
+
if (e.kind === 'tool_call' && e.tool === 'exec' && e.ok === false) {
|
|
64
|
+
cur += 1;
|
|
65
|
+
if (cur > max)
|
|
66
|
+
max = cur;
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
cur = 0;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return max;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Count `user_msg` corrections tied to the most recent preceding `tool_call`.
|
|
76
|
+
* A correction counts if:
|
|
77
|
+
* - it lands within `correction_window_ms` after the last tool_call, OR
|
|
78
|
+
* - it is late but no `assistant_msg` intervened between that tool_call and
|
|
79
|
+
* the correction (still the user's direct response to that tool_call).
|
|
80
|
+
* A new `tool_call` resets the window; an `assistant_msg` marks intervention.
|
|
81
|
+
*/
|
|
82
|
+
function computeCorrections(events, cfg) {
|
|
83
|
+
const windowMs = cfg.detector.correction_window_ms;
|
|
84
|
+
let count = 0;
|
|
85
|
+
let lastToolCallTime = null;
|
|
86
|
+
let sawAssistantMsgSinceToolCall = false;
|
|
87
|
+
for (const e of events) {
|
|
88
|
+
if (e.kind === 'tool_call') {
|
|
89
|
+
lastToolCallTime = Date.parse(e.t);
|
|
90
|
+
sawAssistantMsgSinceToolCall = false;
|
|
91
|
+
}
|
|
92
|
+
else if (e.kind === 'assistant_msg') {
|
|
93
|
+
sawAssistantMsgSinceToolCall = true;
|
|
94
|
+
}
|
|
95
|
+
else if (e.kind === 'user_msg' && e.correction?.matched === true) {
|
|
96
|
+
if (lastToolCallTime !== null) {
|
|
97
|
+
const dt = Date.parse(e.t) - lastToolCallTime;
|
|
98
|
+
if (dt <= windowMs) {
|
|
99
|
+
count += 1;
|
|
100
|
+
}
|
|
101
|
+
else if (!sawAssistantMsgSinceToolCall) {
|
|
102
|
+
count += 1;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return count;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* True when the last `tail_fraction` of events is explore-heavy (explore-ratio
|
|
111
|
+
* ≥ `explore_ratio_min`) with zero edit events. Suppressed (forced false) when
|
|
112
|
+
* `suppress_abandonment_when_no_exec` is set and the whole session is a
|
|
113
|
+
* research signature (zero edits AND zero test/build exec calls).
|
|
114
|
+
*/
|
|
115
|
+
function computeAbandonment(events, cfg) {
|
|
116
|
+
if (events.length === 0)
|
|
117
|
+
return false;
|
|
118
|
+
const tailSize = Math.floor(events.length * cfg.areas.tail_fraction);
|
|
119
|
+
if (tailSize === 0)
|
|
120
|
+
return false;
|
|
121
|
+
const tail = events.slice(events.length - tailSize);
|
|
122
|
+
let tailExploreCount = 0;
|
|
123
|
+
let tailEditedLines = 0;
|
|
124
|
+
let tailEditCount = 0;
|
|
125
|
+
for (const e of tail) {
|
|
126
|
+
if (e.kind !== 'tool_call' || e.tool === null)
|
|
127
|
+
continue;
|
|
128
|
+
if (EXPLORE_TOOLS.has(e.tool)) {
|
|
129
|
+
tailExploreCount += 1;
|
|
130
|
+
}
|
|
131
|
+
else if (e.tool === 'edit') {
|
|
132
|
+
tailEditCount += 1;
|
|
133
|
+
tailEditedLines += e.input_digest.lines_changed ?? 0;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
const tailExploreRatio = tailExploreCount / Math.max(tailEditedLines, 1);
|
|
137
|
+
let abandonment = tailExploreRatio >= cfg.areas.explore_ratio_min && tailEditCount === 0;
|
|
138
|
+
if (abandonment && cfg.areas.suppress_abandonment_when_no_exec) {
|
|
139
|
+
let wholeSessionEdits = 0;
|
|
140
|
+
let wholeSessionTestExec = false;
|
|
141
|
+
for (const e of events) {
|
|
142
|
+
if (e.kind !== 'tool_call' || e.tool === null)
|
|
143
|
+
continue;
|
|
144
|
+
if (e.tool === 'edit') {
|
|
145
|
+
wholeSessionEdits += 1;
|
|
146
|
+
}
|
|
147
|
+
else if (e.tool === 'exec' &&
|
|
148
|
+
isTestBuildExec(e.input_digest.cmd, cfg.areas.test_cmd_patterns)) {
|
|
149
|
+
wholeSessionTestExec = true;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
if (wholeSessionEdits === 0 && !wholeSessionTestExec) {
|
|
153
|
+
abandonment = false;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return abandonment;
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Count completed `edit → test/build-exec(ok=false) → edit-same-file` cycles
|
|
160
|
+
* per file. Tracks two sets per file: `pending` (edit seen, awaiting a failed
|
|
161
|
+
* test) and `failedTestPending` (edit + failed test seen, awaiting a re-edit).
|
|
162
|
+
* On a failed test/build exec, all pending files move to failedTestPending.
|
|
163
|
+
* On an edit touching F: if F is in failedTestPending a cycle completes; F then
|
|
164
|
+
* re-enters pending (a new potential cycle). Exact path-string match.
|
|
165
|
+
*/
|
|
166
|
+
function computeOscillation(events, cfg) {
|
|
167
|
+
const patterns = cfg.areas.test_cmd_patterns;
|
|
168
|
+
const pending = new Set();
|
|
169
|
+
const failedTestPending = new Set();
|
|
170
|
+
let cycles = 0;
|
|
171
|
+
for (const e of events) {
|
|
172
|
+
if (e.kind !== 'tool_call' || e.tool === null)
|
|
173
|
+
continue;
|
|
174
|
+
if (e.tool === 'edit') {
|
|
175
|
+
for (const f of e.input_digest.files) {
|
|
176
|
+
if (failedTestPending.has(f)) {
|
|
177
|
+
cycles += 1;
|
|
178
|
+
failedTestPending.delete(f);
|
|
179
|
+
}
|
|
180
|
+
pending.add(f);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
else if (e.tool === 'exec' &&
|
|
184
|
+
e.ok === false &&
|
|
185
|
+
isTestBuildExec(e.input_digest.cmd, patterns)) {
|
|
186
|
+
for (const f of pending) {
|
|
187
|
+
failedTestPending.add(f);
|
|
188
|
+
}
|
|
189
|
+
pending.clear();
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
return cycles;
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* `(last event t − first event t)` in ms / totalEditedLines. `null` when no
|
|
196
|
+
* edit lines were produced. Derived from event timestamps only; this can
|
|
197
|
+
* diverge from `envelope.duration_ms`, which the adapter derives from raw
|
|
198
|
+
* record timestamps (first..last record). After the tool_use/result merge a
|
|
199
|
+
* merged tool_call's `t` is the result's timestamp, so the two spans are not
|
|
200
|
+
* guaranteed equal.
|
|
201
|
+
*/
|
|
202
|
+
function computeWallClockPerLine(events) {
|
|
203
|
+
let totalEditedLines = 0;
|
|
204
|
+
for (const e of events) {
|
|
205
|
+
if (e.kind === 'tool_call' && e.tool === 'edit') {
|
|
206
|
+
totalEditedLines += e.input_digest.lines_changed ?? 0;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
if (totalEditedLines === 0)
|
|
210
|
+
return null;
|
|
211
|
+
const durationMs = Date.parse(events[events.length - 1].t) - Date.parse(events[0].t);
|
|
212
|
+
return durationMs / totalEditedLines;
|
|
213
|
+
}
|
|
214
|
+
/** Case-insensitive substring match of any pattern against a scrubbed cmd. */
|
|
215
|
+
function isTestBuildExec(cmd, patterns) {
|
|
216
|
+
if (cmd === null)
|
|
217
|
+
return false;
|
|
218
|
+
const lower = cmd.toLowerCase();
|
|
219
|
+
return patterns.some((p) => lower.includes(p.toLowerCase()));
|
|
220
|
+
}
|
|
221
|
+
//# sourceMappingURL=signals.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"signals.js","sourceRoot":"","sources":["../../src/detector/signals.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,4EAA4E;AAI5E,MAAM,aAAa,GAA0B,IAAI,GAAG,CAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAE3F;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,MAAyB,EAAE,GAAW;IACnE,OAAO;QACL,aAAa,EAAE,mBAAmB,CAAC,MAAM,CAAC;QAC1C,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC;QAClC,cAAc,EAAE,oBAAoB,CAAC,MAAM,CAAC;QAC5C,WAAW,EAAE,kBAAkB,CAAC,MAAM,EAAE,GAAG,CAAC;QAC5C,WAAW,EAAE,kBAAkB,CAAC,MAAM,EAAE,GAAG,CAAC;QAC5C,WAAW,EAAE,kBAAkB,CAAC,MAAM,EAAE,GAAG,CAAC;QAC5C,sBAAsB,EAAE,uBAAuB,CAAC,MAAM,CAAC;KACxD,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAAC,MAAyB;IACpD,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI;YAAE,SAAS;QACxD,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9B,YAAY,IAAI,CAAC,CAAC;QACpB,CAAC;aAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7B,gBAAgB,IAAI,CAAC,CAAC,YAAY,CAAC,aAAa,IAAI,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IACD,IAAI,gBAAgB,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACxC,OAAO,YAAY,GAAG,gBAAgB,CAAC;AACzC,CAAC;AAED,mFAAmF;AACnF,SAAS,aAAa,CAAC,MAAyB,EAAE,GAAW;IAC3D,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IAChD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;YAAE,SAAS;QAC1D,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;YACrC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IACD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QAChC,IAAI,CAAC,IAAI,SAAS;YAAE,MAAM,IAAI,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,mFAAmF;AACnF,SAAS,oBAAoB,CAAC,MAAyB;IACrD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC;YAClE,GAAG,IAAI,CAAC,CAAC;YACT,IAAI,GAAG,GAAG,GAAG;gBAAE,GAAG,GAAG,GAAG,CAAC;QAC3B,CAAC;aAAM,CAAC;YACN,GAAG,GAAG,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,kBAAkB,CAAC,MAAyB,EAAE,GAAW;IAChE,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,oBAAoB,CAAC;IACnD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,gBAAgB,GAAkB,IAAI,CAAC;IAC3C,IAAI,4BAA4B,GAAG,KAAK,CAAC;IAEzC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC3B,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,4BAA4B,GAAG,KAAK,CAAC;QACvC,CAAC;aAAM,IAAI,CAAC,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;YACtC,4BAA4B,GAAG,IAAI,CAAC;QACtC,CAAC;aAAM,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,UAAU,EAAE,OAAO,KAAK,IAAI,EAAE,CAAC;YACnE,IAAI,gBAAgB,KAAK,IAAI,EAAE,CAAC;gBAC9B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,gBAAgB,CAAC;gBAC9C,IAAI,EAAE,IAAI,QAAQ,EAAE,CAAC;oBACnB,KAAK,IAAI,CAAC,CAAC;gBACb,CAAC;qBAAM,IAAI,CAAC,4BAA4B,EAAE,CAAC;oBACzC,KAAK,IAAI,CAAC,CAAC;gBACb,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,SAAS,kBAAkB,CAAC,MAAyB,EAAE,GAAW;IAChE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACrE,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACjC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC;IAEpD,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI;YAAE,SAAS;QACxD,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9B,gBAAgB,IAAI,CAAC,CAAC;QACxB,CAAC;aAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7B,aAAa,IAAI,CAAC,CAAC;YACnB,eAAe,IAAI,CAAC,CAAC,YAAY,CAAC,aAAa,IAAI,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IACD,MAAM,gBAAgB,GAAG,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;IACzE,IAAI,WAAW,GAAG,gBAAgB,IAAI,GAAG,CAAC,KAAK,CAAC,iBAAiB,IAAI,aAAa,KAAK,CAAC,CAAC;IAEzF,IAAI,WAAW,IAAI,GAAG,CAAC,KAAK,CAAC,iCAAiC,EAAE,CAAC;QAC/D,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAC1B,IAAI,oBAAoB,GAAG,KAAK,CAAC;QACjC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI;gBAAE,SAAS;YACxD,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACtB,iBAAiB,IAAI,CAAC,CAAC;YACzB,CAAC;iBAAM,IACL,CAAC,CAAC,IAAI,KAAK,MAAM;gBACjB,eAAe,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAChE,CAAC;gBACD,oBAAoB,GAAG,IAAI,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,IAAI,iBAAiB,KAAK,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACrD,WAAW,GAAG,KAAK,CAAC;QACtB,CAAC;IACH,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,kBAAkB,CAAC,MAAyB,EAAE,GAAW;IAChE,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,iBAAiB,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC5C,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI;YAAE,SAAS;QACxD,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACtB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;gBACrC,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC7B,MAAM,IAAI,CAAC,CAAC;oBACZ,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAC9B,CAAC;gBACD,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;aAAM,IACL,CAAC,CAAC,IAAI,KAAK,MAAM;YACjB,CAAC,CAAC,EAAE,KAAK,KAAK;YACd,eAAe,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,EAAE,QAAQ,CAAC,EAC7C,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACxB,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC3B,CAAC;YACD,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,uBAAuB,CAAC,MAAyB;IACxD,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAChD,gBAAgB,IAAI,CAAC,CAAC,YAAY,CAAC,aAAa,IAAI,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IACD,IAAI,gBAAgB,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACxC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,OAAO,UAAU,GAAG,gBAAgB,CAAC;AACvC,CAAC;AAED,8EAA8E;AAC9E,SAAS,eAAe,CAAC,GAAkB,EAAE,QAAkB;IAC7D,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC/B,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;IAChC,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AAC/D,CAAC"}
|
package/dist/egress.d.ts
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export declare const FORBIDDEN_IMPORT: RegExp;
|
|
2
|
+
export declare const FORBIDDEN_FETCH_CALL: RegExp;
|
|
3
|
+
/** True if `content` contains a forbidden network-module import. */
|
|
4
|
+
export declare function hasForbiddenImport(content: string): boolean;
|
|
5
|
+
/** True if `content` invokes the global fetch. */
|
|
6
|
+
export declare function hasFetchCall(content: string): boolean;
|
|
7
|
+
/** True if `content` has any forbidden network egress (import or fetch call). */
|
|
8
|
+
export declare function hasForbiddenEgress(content: string): boolean;
|
package/dist/egress.js
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// §11 egress guard: forbidden network-egress detection. Single source of truth
|
|
2
|
+
// shared by the no-network audit (test/egress.test.ts) and its unit tests.
|
|
3
|
+
//
|
|
4
|
+
// The CLI must be stateless and offline (§11): no src file may import a network
|
|
5
|
+
// module or invoke the global fetch. This module encodes what counts as
|
|
6
|
+
// "forbidden" so the scan and the unit tests cannot drift apart.
|
|
7
|
+
// Matches four import shapes whose specifier is a forbidden network module
|
|
8
|
+
// (http, https, net, node:net, node:http, node:https, undici, fetch — the
|
|
9
|
+
// `node:` prefix is optional and `https?` collapses http/https):
|
|
10
|
+
// 1. static import with a binding list (possibly multi-line): import … from 'spec'
|
|
11
|
+
// — `[\s\S]*?` matches across newlines so multi-line named imports are caught.
|
|
12
|
+
// 2. side-effect import: bare import 'spec' (no `from`).
|
|
13
|
+
// 3. dynamic import: import('spec').
|
|
14
|
+
// 4. CommonJS require: require('spec').
|
|
15
|
+
// Errs toward flagging (matches inside comments/strings too) — acceptable for a
|
|
16
|
+
// security control; the scan asserts zero offenders in src/.
|
|
17
|
+
export const FORBIDDEN_IMPORT = /(?:import\s+[\s\S]*?\s+from\s+|import\s*|import\s*\(\s*|require\s*\(\s*)['"](?:node:)?(?:https?|net|undici|fetch)['"]/;
|
|
18
|
+
// Matches a call to the global fetch — a network API available as a global in
|
|
19
|
+
// Node 18+ (this project targets Node >=22.12), so an import is not required to
|
|
20
|
+
// use it. Catches a fetch invocation with any arguments. Case-sensitive so it
|
|
21
|
+
// does not match `WebFetch` or `hasFetchCall`; a locally-defined fetch function
|
|
22
|
+
// would still match, which is acceptable in a no-network codebase where no such
|
|
23
|
+
// local binding exists. Errs toward flagging (matches inside comments/strings).
|
|
24
|
+
export const FORBIDDEN_FETCH_CALL = /\bfetch\s*\(/;
|
|
25
|
+
/** True if `content` contains a forbidden network-module import. */
|
|
26
|
+
export function hasForbiddenImport(content) {
|
|
27
|
+
return FORBIDDEN_IMPORT.test(content);
|
|
28
|
+
}
|
|
29
|
+
/** True if `content` invokes the global fetch. */
|
|
30
|
+
export function hasFetchCall(content) {
|
|
31
|
+
return FORBIDDEN_FETCH_CALL.test(content);
|
|
32
|
+
}
|
|
33
|
+
/** True if `content` has any forbidden network egress (import or fetch call). */
|
|
34
|
+
export function hasForbiddenEgress(content) {
|
|
35
|
+
return hasForbiddenImport(content) || hasFetchCall(content);
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=egress.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"egress.js","sourceRoot":"","sources":["../src/egress.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,2EAA2E;AAC3E,EAAE;AACF,gFAAgF;AAChF,wEAAwE;AACxE,iEAAiE;AAEjE,2EAA2E;AAC3E,0EAA0E;AAC1E,iEAAiE;AACjE,qFAAqF;AACrF,oFAAoF;AACpF,2DAA2D;AAC3D,uCAAuC;AACvC,0CAA0C;AAC1C,gFAAgF;AAChF,6DAA6D;AAC7D,MAAM,CAAC,MAAM,gBAAgB,GAC3B,uHAAuH,CAAC;AAE1H,8EAA8E;AAC9E,gFAAgF;AAChF,8EAA8E;AAC9E,gFAAgF;AAChF,gFAAgF;AAChF,gFAAgF;AAChF,MAAM,CAAC,MAAM,oBAAoB,GAAG,cAAc,CAAC;AAEnD,oEAAoE;AACpE,MAAM,UAAU,kBAAkB,CAAC,OAAe;IAChD,OAAO,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACxC,CAAC;AAED,kDAAkD;AAClD,MAAM,UAAU,YAAY,CAAC,OAAe;IAC1C,OAAO,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC5C,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,kBAAkB,CAAC,OAAe;IAChD,OAAO,kBAAkB,CAAC,OAAO,CAAC,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;AAC9D,CAAC"}
|
package/dist/git.d.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sandboxed git toplevel resolver.
|
|
3
|
+
*
|
|
4
|
+
* Resolves the git toplevel for `cwd` via a locked-down
|
|
5
|
+
* `git rev-parse --show-toplevel`. Returns `null` (never throws) when `cwd` is
|
|
6
|
+
* not a repo, git is unavailable, or `cwd` is missing.
|
|
7
|
+
*
|
|
8
|
+
* Security: `cwd` originates from transcripts (untrusted). The sandbox — env
|
|
9
|
+
* vars (`GIT_CONFIG_NOSYSTEM`, `GIT_CONFIG_GLOBAL`), `-c` overrides
|
|
10
|
+
* (`core.fsmonitor`, `core.pager`, `core.hooksPath`), `execFile` (no shell),
|
|
11
|
+
* and rev-parse only — prevents git from invoking external programs via
|
|
12
|
+
* repo-local config. Do not relax these.
|
|
13
|
+
*
|
|
14
|
+
* Memoized by `cwd` when a `cache` Map is passed.
|
|
15
|
+
*/
|
|
16
|
+
export declare function resolveToplevel(cwd: string, cache?: Map<string, string | null>): Promise<string | null>;
|
package/dist/git.js
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import * as child_process from 'node:child_process';
|
|
2
|
+
/**
|
|
3
|
+
* Sandboxed git toplevel resolver.
|
|
4
|
+
*
|
|
5
|
+
* Resolves the git toplevel for `cwd` via a locked-down
|
|
6
|
+
* `git rev-parse --show-toplevel`. Returns `null` (never throws) when `cwd` is
|
|
7
|
+
* not a repo, git is unavailable, or `cwd` is missing.
|
|
8
|
+
*
|
|
9
|
+
* Security: `cwd` originates from transcripts (untrusted). The sandbox — env
|
|
10
|
+
* vars (`GIT_CONFIG_NOSYSTEM`, `GIT_CONFIG_GLOBAL`), `-c` overrides
|
|
11
|
+
* (`core.fsmonitor`, `core.pager`, `core.hooksPath`), `execFile` (no shell),
|
|
12
|
+
* and rev-parse only — prevents git from invoking external programs via
|
|
13
|
+
* repo-local config. Do not relax these.
|
|
14
|
+
*
|
|
15
|
+
* Memoized by `cwd` when a `cache` Map is passed.
|
|
16
|
+
*/
|
|
17
|
+
export function resolveToplevel(cwd, cache) {
|
|
18
|
+
if (cache?.has(cwd)) {
|
|
19
|
+
return Promise.resolve(cache.get(cwd) ?? null);
|
|
20
|
+
}
|
|
21
|
+
return new Promise((resolve) => {
|
|
22
|
+
try {
|
|
23
|
+
child_process.execFile('git', [
|
|
24
|
+
'-C',
|
|
25
|
+
cwd,
|
|
26
|
+
'-c',
|
|
27
|
+
'core.fsmonitor=',
|
|
28
|
+
'-c',
|
|
29
|
+
'core.pager=cat',
|
|
30
|
+
'-c',
|
|
31
|
+
'core.hooksPath=',
|
|
32
|
+
'rev-parse',
|
|
33
|
+
'--show-toplevel',
|
|
34
|
+
], {
|
|
35
|
+
env: {
|
|
36
|
+
...process.env,
|
|
37
|
+
GIT_CONFIG_NOSYSTEM: '1',
|
|
38
|
+
GIT_CONFIG_GLOBAL: '/dev/null',
|
|
39
|
+
},
|
|
40
|
+
windowsHide: true,
|
|
41
|
+
}, (err, stdout) => {
|
|
42
|
+
if (err) {
|
|
43
|
+
// non-zero exit, git missing (ENOENT), cwd missing, etc.
|
|
44
|
+
cache?.set(cwd, null);
|
|
45
|
+
return resolve(null);
|
|
46
|
+
}
|
|
47
|
+
const toplevel = String(stdout).trim();
|
|
48
|
+
cache?.set(cwd, toplevel);
|
|
49
|
+
resolve(toplevel);
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
// synchronous throw (invalid args, etc.) — never propagate
|
|
54
|
+
cache?.set(cwd, null);
|
|
55
|
+
resolve(null);
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=git.js.map
|
package/dist/git.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"git.js","sourceRoot":"","sources":["../src/git.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,aAAa,MAAM,oBAAoB,CAAC;AAEpD;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,eAAe,CAC7B,GAAW,EACX,KAAkC;IAElC,IAAI,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACpB,OAAO,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC;IACjD,CAAC;IAED,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,IAAI,CAAC;YACH,aAAa,CAAC,QAAQ,CACpB,KAAK,EACL;gBACE,IAAI;gBACJ,GAAG;gBACH,IAAI;gBACJ,iBAAiB;gBACjB,IAAI;gBACJ,gBAAgB;gBAChB,IAAI;gBACJ,iBAAiB;gBACjB,WAAW;gBACX,iBAAiB;aAClB,EACD;gBACE,GAAG,EAAE;oBACH,GAAG,OAAO,CAAC,GAAG;oBACd,mBAAmB,EAAE,GAAG;oBACxB,iBAAiB,EAAE,WAAW;iBAC/B;gBACD,WAAW,EAAE,IAAI;aAClB,EACD,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;gBACd,IAAI,GAAG,EAAE,CAAC;oBACR,yDAAyD;oBACzD,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;oBACtB,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;gBACvB,CAAC;gBACD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;gBACvC,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;gBAC1B,OAAO,CAAC,QAAQ,CAAC,CAAC;YACpB,CAAC,CACF,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,2DAA2D;YAC3D,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function ping(): string;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,IAAI;IAClB,OAAO,YAAY,CAAC;AACtB,CAAC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { Config, ScoringMode, SignalName, SignalValues } from '../types.js';
|
|
2
|
+
/** Per-signal aggregate stats in the calibrate object. */
|
|
3
|
+
interface CalibrateSignalStat {
|
|
4
|
+
min: number;
|
|
5
|
+
p50: number;
|
|
6
|
+
p90: number;
|
|
7
|
+
max: number;
|
|
8
|
+
active_threshold: number;
|
|
9
|
+
}
|
|
10
|
+
/** Inputs to `buildCalibrateObject`. */
|
|
11
|
+
interface CalibrateInput {
|
|
12
|
+
mode: ScoringMode;
|
|
13
|
+
session_count: number;
|
|
14
|
+
flag_pct: number;
|
|
15
|
+
signals: SignalValues[];
|
|
16
|
+
bootstrap_thresholds: Config['detector']['bootstrap_thresholds'];
|
|
17
|
+
}
|
|
18
|
+
/** The calibrate aggregate-stats object (no per-session values). */
|
|
19
|
+
interface CalibrateObject {
|
|
20
|
+
mode: ScoringMode;
|
|
21
|
+
session_count: number;
|
|
22
|
+
flag_pct: number;
|
|
23
|
+
signals: Record<SignalName, CalibrateSignalStat>;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Build the calibrate aggregate-stats object. Pure: no I/O, no per-session
|
|
27
|
+
* values in the output (only min/p50/p90/max/active_threshold per signal).
|
|
28
|
+
*/
|
|
29
|
+
export declare function buildCalibrateObject(input: CalibrateInput): CalibrateObject;
|
|
30
|
+
/**
|
|
31
|
+
* Format the calibrate object as a human-readable table. Aggregate numbers +
|
|
32
|
+
* signal names only — no per-session examples, no commands, no prose.
|
|
33
|
+
*/
|
|
34
|
+
export declare function formatCalibrateTable(obj: ReturnType<typeof buildCalibrateObject>): string;
|
|
35
|
+
export {};
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
// Pure calibrate builders: turn per-session `SignalValues[]` into aggregate
|
|
2
|
+
// statistics (min/p50/p90/max/active_threshold per signal) and a human table.
|
|
3
|
+
// No I/O, no per-session values in the output, no commands, no prose.
|
|
4
|
+
//
|
|
5
|
+
// `active_threshold` semantics:
|
|
6
|
+
// - bootstrap mode → `bootstrap_thresholds[name]` (the configured absolute
|
|
7
|
+
// threshold; for `wall_clock_per_line` this is `wall_clock_per_line_ms`).
|
|
8
|
+
// - percentile mode → the `flag_pct`-percentile value of that signal across
|
|
9
|
+
// sessions (sorted-index linear interpolation).
|
|
10
|
+
// - `abandonment` (boolean): treated as 0/1 throughout. Its `active_threshold`
|
|
11
|
+
// is `bootstrap_thresholds.abandonment` (true) represented as 1 for both
|
|
12
|
+
// modes, so the entire abandonment entry stays uniformly 0/1.
|
|
13
|
+
// SignalName → field on `SignalValues`. Only `wall_clock_per_line` differs
|
|
14
|
+
// (its field is `wall_clock_per_line_ms`).
|
|
15
|
+
const SIGNAL_FIELDS = {
|
|
16
|
+
explore_ratio: 'explore_ratio',
|
|
17
|
+
reread: 'reread',
|
|
18
|
+
failure_streak: 'failure_streak',
|
|
19
|
+
corrections: 'corrections',
|
|
20
|
+
abandonment: 'abandonment',
|
|
21
|
+
oscillation: 'oscillation',
|
|
22
|
+
wall_clock_per_line: 'wall_clock_per_line_ms',
|
|
23
|
+
};
|
|
24
|
+
// SignalName → key on `bootstrap_thresholds`. Only `wall_clock_per_line` differs
|
|
25
|
+
// (its threshold key is `wall_clock_per_line_ms`).
|
|
26
|
+
const THRESHOLD_KEYS = {
|
|
27
|
+
explore_ratio: 'explore_ratio',
|
|
28
|
+
reread: 'reread',
|
|
29
|
+
failure_streak: 'failure_streak',
|
|
30
|
+
corrections: 'corrections',
|
|
31
|
+
abandonment: 'abandonment',
|
|
32
|
+
oscillation: 'oscillation',
|
|
33
|
+
wall_clock_per_line: 'wall_clock_per_line_ms',
|
|
34
|
+
};
|
|
35
|
+
// Canonical signal order for deterministic output.
|
|
36
|
+
const SIGNAL_ORDER = [
|
|
37
|
+
'explore_ratio',
|
|
38
|
+
'reread',
|
|
39
|
+
'failure_streak',
|
|
40
|
+
'corrections',
|
|
41
|
+
'abandonment',
|
|
42
|
+
'oscillation',
|
|
43
|
+
'wall_clock_per_line',
|
|
44
|
+
];
|
|
45
|
+
/**
|
|
46
|
+
* Build the calibrate aggregate-stats object. Pure: no I/O, no per-session
|
|
47
|
+
* values in the output (only min/p50/p90/max/active_threshold per signal).
|
|
48
|
+
*/
|
|
49
|
+
export function buildCalibrateObject(input) {
|
|
50
|
+
const { mode, session_count, flag_pct, signals, bootstrap_thresholds } = input;
|
|
51
|
+
const out = {};
|
|
52
|
+
for (const name of SIGNAL_ORDER) {
|
|
53
|
+
out[name] = computeStat(name, signals, mode, flag_pct, bootstrap_thresholds);
|
|
54
|
+
}
|
|
55
|
+
return { mode, session_count, flag_pct, signals: out };
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Compute min/p50/p90/max/active_threshold for one signal across all sessions.
|
|
59
|
+
*
|
|
60
|
+
* - `abandonment`: 0/1 stats (min = any false, max = any true, p50 = strict
|
|
61
|
+
* majority true, p90 = max, active_threshold = configured boolean → 1).
|
|
62
|
+
* - numeric/nullable: filter nulls; min/max are extremes; p50/p90/active use
|
|
63
|
+
* sorted-index linear interpolation. Empty (all-null) → all zeros.
|
|
64
|
+
*/
|
|
65
|
+
function computeStat(name, signals, mode, flag_pct, bootstrap_thresholds) {
|
|
66
|
+
if (name === 'abandonment') {
|
|
67
|
+
return computeAbandonmentStat(signals, bootstrap_thresholds);
|
|
68
|
+
}
|
|
69
|
+
const field = SIGNAL_FIELDS[name];
|
|
70
|
+
const raw = signals
|
|
71
|
+
.map((s) => s[field])
|
|
72
|
+
.filter((v) => v !== null);
|
|
73
|
+
const bootThreshold = bootstrap_thresholds[THRESHOLD_KEYS[name]];
|
|
74
|
+
if (raw.length === 0) {
|
|
75
|
+
// No non-null values: stats are 0; active_threshold is the configured boot
|
|
76
|
+
// threshold in bootstrap mode, else 0 (no percentile of an empty set).
|
|
77
|
+
return {
|
|
78
|
+
min: 0,
|
|
79
|
+
p50: 0,
|
|
80
|
+
p90: 0,
|
|
81
|
+
max: 0,
|
|
82
|
+
active_threshold: mode === 'bootstrap' ? round(bootThreshold) : 0,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
const sorted = [...raw].sort((a, b) => a - b);
|
|
86
|
+
const min = sorted[0];
|
|
87
|
+
const max = sorted[sorted.length - 1];
|
|
88
|
+
const p50 = percentile(sorted, 50);
|
|
89
|
+
const p90 = percentile(sorted, 90);
|
|
90
|
+
const active_threshold = mode === 'bootstrap' ? bootThreshold : percentile(sorted, flag_pct);
|
|
91
|
+
return {
|
|
92
|
+
min: round(min),
|
|
93
|
+
p50: round(p50),
|
|
94
|
+
p90: round(p90),
|
|
95
|
+
max: round(max),
|
|
96
|
+
active_threshold: round(active_threshold),
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
/** Abandonment stats: booleans as 0/1, all outputs constrained to {0,1}. */
|
|
100
|
+
function computeAbandonmentStat(signals, bootstrap_thresholds) {
|
|
101
|
+
if (signals.length === 0) {
|
|
102
|
+
return { min: 0, p50: 0, p90: 0, max: 0, active_threshold: 0 };
|
|
103
|
+
}
|
|
104
|
+
const vals = signals.map((s) => (s.abandonment ? 1 : 0));
|
|
105
|
+
const trues = vals.reduce((acc, v) => acc + v, 0);
|
|
106
|
+
const min = vals.includes(0) ? 0 : 1;
|
|
107
|
+
const max = trues > 0 ? 1 : 0;
|
|
108
|
+
// strict majority true (tie → 0), matching the leaderboard median-boolean rule
|
|
109
|
+
const p50 = trues * 2 > vals.length ? 1 : 0;
|
|
110
|
+
const p90 = max; // 90th percentile of 0/1: 1 if any session abandoned
|
|
111
|
+
const active_threshold = bootstrap_thresholds.abandonment ? 1 : 0;
|
|
112
|
+
return { min, p50, p90, max, active_threshold };
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Percentile via sorted-index linear interpolation (R-7 / numpy default).
|
|
116
|
+
* `sorted` must be ascending and non-empty. `p` is in [0, 100].
|
|
117
|
+
*/
|
|
118
|
+
function percentile(sorted, p) {
|
|
119
|
+
const n = sorted.length;
|
|
120
|
+
if (n === 0)
|
|
121
|
+
return 0;
|
|
122
|
+
if (n === 1)
|
|
123
|
+
return sorted[0];
|
|
124
|
+
const rank = (p / 100) * (n - 1);
|
|
125
|
+
const lower = Math.floor(rank);
|
|
126
|
+
const upper = Math.ceil(rank);
|
|
127
|
+
if (lower === upper)
|
|
128
|
+
return sorted[lower];
|
|
129
|
+
return sorted[lower] + (rank - lower) * (sorted[upper] - sorted[lower]);
|
|
130
|
+
}
|
|
131
|
+
/** Round to 2 decimal places to keep aggregate stats readable. */
|
|
132
|
+
function round(x) {
|
|
133
|
+
return Math.round(x * 100) / 100;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Format the calibrate object as a human-readable table. Aggregate numbers +
|
|
137
|
+
* signal names only — no per-session examples, no commands, no prose.
|
|
138
|
+
*/
|
|
139
|
+
export function formatCalibrateTable(obj) {
|
|
140
|
+
const lines = [];
|
|
141
|
+
lines.push(`harnessgap calibrate — mode: ${obj.mode} · ${obj.session_count} sessions · flag_pct: ${obj.flag_pct}`);
|
|
142
|
+
lines.push(`${'SIGNAL'.padEnd(22)} | ${'MIN'.padStart(10)} | ${'P50'.padStart(10)} | ${'P90'.padStart(10)} | ${'MAX'.padStart(10)} | ${'THRESHOLD'.padStart(10)}`);
|
|
143
|
+
for (const name of SIGNAL_ORDER) {
|
|
144
|
+
const s = obj.signals[name];
|
|
145
|
+
lines.push(`${name.padEnd(22)} | ${String(s.min).padStart(10)} | ${String(s.p50).padStart(10)} | ${String(s.p90).padStart(10)} | ${String(s.max).padStart(10)} | ${String(s.active_threshold).padStart(10)}`);
|
|
146
|
+
}
|
|
147
|
+
return lines.join('\n');
|
|
148
|
+
}
|
|
149
|
+
//# sourceMappingURL=calibrate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"calibrate.js","sourceRoot":"","sources":["../../src/output/calibrate.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,8EAA8E;AAC9E,sEAAsE;AACtE,EAAE;AACF,gCAAgC;AAChC,2EAA2E;AAC3E,4EAA4E;AAC5E,4EAA4E;AAC5E,kDAAkD;AAClD,+EAA+E;AAC/E,2EAA2E;AAC3E,gEAAgE;AAmChE,2EAA2E;AAC3E,2CAA2C;AAC3C,MAAM,aAAa,GAA2C;IAC5D,aAAa,EAAE,eAAe;IAC9B,MAAM,EAAE,QAAQ;IAChB,cAAc,EAAE,gBAAgB;IAChC,WAAW,EAAE,aAAa;IAC1B,WAAW,EAAE,aAAa;IAC1B,WAAW,EAAE,aAAa;IAC1B,mBAAmB,EAAE,wBAAwB;CAC9C,CAAC;AAEF,iFAAiF;AACjF,mDAAmD;AACnD,MAAM,cAAc,GAGhB;IACF,aAAa,EAAE,eAAe;IAC9B,MAAM,EAAE,QAAQ;IAChB,cAAc,EAAE,gBAAgB;IAChC,WAAW,EAAE,aAAa;IAC1B,WAAW,EAAE,aAAa;IAC1B,WAAW,EAAE,aAAa;IAC1B,mBAAmB,EAAE,wBAAwB;CAC9C,CAAC;AAEF,mDAAmD;AACnD,MAAM,YAAY,GAA0B;IAC1C,eAAe;IACf,QAAQ;IACR,gBAAgB;IAChB,aAAa;IACb,aAAa;IACb,aAAa;IACb,qBAAqB;CACtB,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAqB;IACxD,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,OAAO,EAAE,oBAAoB,EAAE,GAAG,KAAK,CAAC;IAC/E,MAAM,GAAG,GAAG,EAA6C,CAAC;IAC1D,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,GAAG,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,oBAAoB,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;AACzD,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,WAAW,CAClB,IAAgB,EAChB,OAAuB,EACvB,IAAiB,EACjB,QAAgB,EAChB,oBAAgE;IAEhE,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;QAC3B,OAAO,sBAAsB,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;IAC/D,CAAC;IACD,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,GAAG,GAAG,OAAO;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAkB,CAAC;SACrC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IAE1C,MAAM,aAAa,GAAG,oBAAoB,CAAC,cAAc,CAAC,IAAI,CAAC,CAAW,CAAC;IAE3E,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrB,2EAA2E;QAC3E,uEAAuE;QACvE,OAAO;YACL,GAAG,EAAE,CAAC;YACN,GAAG,EAAE,CAAC;YACN,GAAG,EAAE,CAAC;YACN,GAAG,EAAE,CAAC;YACN,gBAAgB,EAAE,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;SAClE,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9C,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACtB,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACnC,MAAM,gBAAgB,GACpB,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEtE,OAAO;QACL,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC;QACf,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC;QACf,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC;QACf,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC;QACf,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,CAAC;KAC1C,CAAC;AACJ,CAAC;AAED,4EAA4E;AAC5E,SAAS,sBAAsB,CAC7B,OAAuB,EACvB,oBAAgE;IAEhE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;IACjE,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAS,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrC,MAAM,GAAG,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,+EAA+E;IAC/E,MAAM,GAAG,GAAG,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,qDAAqD;IACtE,MAAM,gBAAgB,GAAG,oBAAoB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,gBAAgB,EAAE,CAAC;AAClD,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,MAAgB,EAAE,CAAS;IAC7C,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IACxB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9B,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,KAAK,KAAK,KAAK;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1C,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAC1E,CAAC;AAED,kEAAkE;AAClE,SAAS,KAAK,CAAC,CAAS;IACtB,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;AACnC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,GAA4C;IAC/E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CACR,gCAAgC,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC,aAAa,yBAAyB,GAAG,CAAC,QAAQ,EAAE,CACvG,CAAC;IACF,KAAK,CAAC,IAAI,CACR,GAAG,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CACvJ,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5B,KAAK,CAAC,IAAI,CACR,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAClM,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { AreaRow, ScoringMode, Warnings } from '../types.js';
|
|
2
|
+
/** Inputs to `formatHuman`. */
|
|
3
|
+
interface HumanInput {
|
|
4
|
+
repo: string;
|
|
5
|
+
mode: ScoringMode;
|
|
6
|
+
sessionCount: number;
|
|
7
|
+
areas: AreaRow[];
|
|
8
|
+
summary: {
|
|
9
|
+
flagged: number;
|
|
10
|
+
unflagged: number;
|
|
11
|
+
unlocalized: number;
|
|
12
|
+
};
|
|
13
|
+
warnings: Warnings;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Format the scan leaderboard as a human-readable string. Pure.
|
|
17
|
+
*
|
|
18
|
+
* Layout: header line, column-aligned table (or a "no flagged areas" line when
|
|
19
|
+
* empty), summary line, and a warnings line (only non-zero categories). The
|
|
20
|
+
* bootstrap count in the summary line is `sessionCount` when mode is bootstrap,
|
|
21
|
+
* else 0.
|
|
22
|
+
*/
|
|
23
|
+
export declare function formatHuman(input: HumanInput): string;
|
|
24
|
+
export {};
|