jev-test-filter 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +590 -0
- package/dist/cargo.d.ts +30 -0
- package/dist/cargo.js +130 -0
- package/dist/cli.d.ts +49 -0
- package/dist/cli.js +265 -0
- package/dist/diff.d.ts +52 -0
- package/dist/diff.js +168 -0
- package/dist/discover.d.ts +2 -0
- package/dist/discover.js +32 -0
- package/dist/extract-go.d.ts +4 -0
- package/dist/extract-go.js +71 -0
- package/dist/extract.d.ts +15 -0
- package/dist/extract.js +146 -0
- package/dist/filter.d.ts +46 -0
- package/dist/filter.js +81 -0
- package/dist/framework.d.ts +21 -0
- package/dist/framework.js +116 -0
- package/dist/gate.d.ts +26 -0
- package/dist/gate.js +63 -0
- package/dist/jev.d.ts +216 -0
- package/dist/jev.js +390 -0
- package/dist/languages.d.ts +1 -0
- package/dist/languages.js +18 -0
- package/dist/questions.d.ts +42 -0
- package/dist/questions.js +57 -0
- package/dist/run.d.ts +74 -0
- package/dist/run.js +172 -0
- package/dist/state.d.ts +19 -0
- package/dist/state.js +55 -0
- package/dist/types.d.ts +93 -0
- package/dist/types.js +18 -0
- package/package.json +69 -0
package/dist/jev.d.ts
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Zero-dependency Jev client.
|
|
3
|
+
*
|
|
4
|
+
* Jev (TypeSafe's System One model) takes one `state` plus many independent
|
|
5
|
+
* `questions` and answers them all in parallel in a single round trip. That
|
|
6
|
+
* shape is the whole reason this tool can afford to ask about every test in
|
|
7
|
+
* a repository: the state -- the diff -- is sent once, and each extra question
|
|
8
|
+
* costs only its own text.
|
|
9
|
+
*
|
|
10
|
+
* Two server ceilings matter, and neither is a question count:
|
|
11
|
+
*
|
|
12
|
+
* - the whole request must be under 64Ki input tokens
|
|
13
|
+
* - the `state` alone must be under 32Ki input tokens (an independent budget,
|
|
14
|
+
* and the one that fills up first)
|
|
15
|
+
*
|
|
16
|
+
* There is no documented cap on the NUMBER of questions; over a thousand in one
|
|
17
|
+
* request is fine. `askSplitting` therefore does not try to predict the ceiling
|
|
18
|
+
* precisely -- it reacts to the server's own `max_tokens_exceeded` by halving
|
|
19
|
+
* the question set, which keeps the estimator below free to be
|
|
20
|
+
* approximate.
|
|
21
|
+
*/
|
|
22
|
+
import type { ScoreQuestion } from "./questions.ts";
|
|
23
|
+
export type Question = ScoreQuestion;
|
|
24
|
+
export interface SystemOneResponse {
|
|
25
|
+
model?: string;
|
|
26
|
+
answers?: Record<string, unknown>;
|
|
27
|
+
usage?: {
|
|
28
|
+
input_tokens?: number;
|
|
29
|
+
output_tokens?: number;
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
export interface Spend {
|
|
33
|
+
calls: number;
|
|
34
|
+
inputTokens: number;
|
|
35
|
+
outputTokens: number;
|
|
36
|
+
ms: number;
|
|
37
|
+
retried: number;
|
|
38
|
+
rateLimited: number;
|
|
39
|
+
tokensPerSecond: number;
|
|
40
|
+
splits: number;
|
|
41
|
+
usd: number;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* A rough input-token count for the pacer's mirror of the server's bucket.
|
|
45
|
+
*
|
|
46
|
+
* Deliberately approximate: the pacer corrects itself to the server's own
|
|
47
|
+
* count when the answer comes back, and `askSplitting` reacts to the server's
|
|
48
|
+
* `max_tokens_exceeded` rather than predicting it, so an estimate that is
|
|
49
|
+
* wrong by a third costs nothing.
|
|
50
|
+
*/
|
|
51
|
+
export declare function estimateTokens(body: unknown): number;
|
|
52
|
+
export declare const DEFAULT_BASE_URL = "https://api.typesafe.ai";
|
|
53
|
+
export declare const DEFAULT_MODEL = "jev-latest";
|
|
54
|
+
/**
|
|
55
|
+
* Where the key and the endpoint come from, in order.
|
|
56
|
+
*
|
|
57
|
+
* `TYPESAFE_*` first because that is the prevailing spelling; `TYPESAFEAI_*`
|
|
58
|
+
* stays as a fallback so an existing environment keeps working without an edit.
|
|
59
|
+
* Both are read rather than one aliased to the other, so a shell that has only
|
|
60
|
+
* the older name set is not a configuration error.
|
|
61
|
+
*/
|
|
62
|
+
export declare const API_KEY_VARS: readonly ["TYPESAFE_API_KEY", "TYPESAFEAI_API_KEY"];
|
|
63
|
+
export declare const BASE_URL_VARS: readonly ["TYPESAFE_BASE_URL", "TYPESAFEAI_BASE_URL"];
|
|
64
|
+
/** First of these variables that is set and non-empty. */
|
|
65
|
+
export declare function fromEnv(names: readonly string[], env?: NodeJS.ProcessEnv): string | null;
|
|
66
|
+
/** Published input price, USD per million input tokens. Output is not billed. */
|
|
67
|
+
export declare const USD_PER_MTOK = 0.042;
|
|
68
|
+
/** What the caller can actually do about a failure. */
|
|
69
|
+
export type JevErrorKind = "too_big" | "auth" | "transient" | "other";
|
|
70
|
+
export declare class JevError extends Error {
|
|
71
|
+
status: number;
|
|
72
|
+
kind: JevErrorKind;
|
|
73
|
+
constructor(message: string, { status, kind }?: {
|
|
74
|
+
status?: number;
|
|
75
|
+
kind?: JevErrorKind;
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
export interface JevOptions {
|
|
79
|
+
apiKey?: string | null;
|
|
80
|
+
baseUrl?: string | null;
|
|
81
|
+
model?: string | null;
|
|
82
|
+
retries?: number;
|
|
83
|
+
timeoutMs?: number;
|
|
84
|
+
/** Called with each request body before it is sent, for leak assertions. */
|
|
85
|
+
onRequest?: ((body: string) => void) | null;
|
|
86
|
+
/** The client's mirror of the server's token bucket; the default is the measured one. */
|
|
87
|
+
pacer?: Pacer | null;
|
|
88
|
+
/** How many times a rate-limited request goes again before it is given up on. */
|
|
89
|
+
rateLimitRetries?: number;
|
|
90
|
+
/** The base wait after a 429, before jitter and growth. */
|
|
91
|
+
rateLimitWaitMs?: number;
|
|
92
|
+
/** The transport, for tests. */
|
|
93
|
+
fetch?: typeof fetch;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* The most requests in flight at once. Latency is 260 ms plus 5.7 ms per
|
|
97
|
+
* thousand tokens, so 78 requests at 4 abreast take 9.6 s and the same 78
|
|
98
|
+
* fired together take 1.6; what stops "all at once" is the token rate below,
|
|
99
|
+
* not the connection count -- 40 small requests at once have never drawn a
|
|
100
|
+
* 429. Measured on the full run over this repository (79 requests, 2.15M
|
|
101
|
+
* tokens) with the pacer: 16 abreast 4.8 s, 32 3.6-3.7 s, 64 3.9 s. Past 32
|
|
102
|
+
* the server's own latency grows with what it is holding and the wall time
|
|
103
|
+
* stops falling.
|
|
104
|
+
*/
|
|
105
|
+
export declare const DEFAULT_CONCURRENCY = 32;
|
|
106
|
+
/**
|
|
107
|
+
* The server's rate limit, as measured, and mirrored here so that the client
|
|
108
|
+
* paces itself instead of being told.
|
|
109
|
+
*
|
|
110
|
+
* The server answers a bare 429 -- no retry-after, no ratelimit headers --
|
|
111
|
+
* and what it limits is input tokens, not requests: 40 small requests at
|
|
112
|
+
* once go through, 20 of the largest at once go through, and the same 20
|
|
113
|
+
* again a second later lose 4, then 14, then 16. The fit is a token bucket
|
|
114
|
+
* of about 1.6M tokens refilling at 200-250k per second. A full run over this
|
|
115
|
+
* repository is 2.1M tokens, so it sits at the edge: fired together it loses
|
|
116
|
+
* between 8 and 36 of 78 requests depending on how full the bucket was.
|
|
117
|
+
*
|
|
118
|
+
* The client keeps its own bucket, charged with each request's estimate as it
|
|
119
|
+
* is sent and corrected to the server's count when the answer comes back. A
|
|
120
|
+
* request waits until the bucket can pay for it. Started a little under the
|
|
121
|
+
* measured size, it draws no 429 on a run that starts with a full bucket; a
|
|
122
|
+
* 429 -- the mirror was wrong, or another process shares the key -- empties
|
|
123
|
+
* the mirror and lowers its rate by a quarter, and a success while requests
|
|
124
|
+
* have had to wait raises the rate by two percent, so a key with a higher
|
|
125
|
+
* limit finds it. The rate is bounded on both sides; the burst is not
|
|
126
|
+
* adapted, since a burst too small costs a run of this size two seconds and
|
|
127
|
+
* one too large costs it lost verdicts.
|
|
128
|
+
*/
|
|
129
|
+
export declare const DEFAULT_TOKENS_PER_SECOND = 200000;
|
|
130
|
+
export declare const DEFAULT_TOKEN_BURST = 1200000;
|
|
131
|
+
export declare class Pacer {
|
|
132
|
+
rate: number;
|
|
133
|
+
readonly burst: number;
|
|
134
|
+
private level;
|
|
135
|
+
private at;
|
|
136
|
+
/** Whether a request has had to wait: only then is the rate probed upward. */
|
|
137
|
+
private waited;
|
|
138
|
+
/** When the rate last grew. */
|
|
139
|
+
private grew;
|
|
140
|
+
constructor(rate?: number, burst?: number, now?: number);
|
|
141
|
+
private refill;
|
|
142
|
+
/** How many tokens the bucket holds now. */
|
|
143
|
+
available(now?: number): number;
|
|
144
|
+
/** Milliseconds until `tokens` can be paid for; 0 if now. */
|
|
145
|
+
delay(tokens: number, now?: number): number;
|
|
146
|
+
/** Charge the bucket, waiting until it can pay. */
|
|
147
|
+
take(tokens: number): Promise<void>;
|
|
148
|
+
/** The server counted differently from the estimate: charge the difference. */
|
|
149
|
+
settle(estimated: number, actual: number): void;
|
|
150
|
+
/** A 429: the mirror was optimistic. Empty it and slow down. */
|
|
151
|
+
throttled(now?: number): void;
|
|
152
|
+
/**
|
|
153
|
+
* A 200 while requests have been waiting: the limit may be higher than
|
|
154
|
+
* mirrored. Probed by time, not by request -- two percent per success
|
|
155
|
+
* compounded over 79 requests to a rate half again the server's and drew
|
|
156
|
+
* the 429s it was meant to avoid.
|
|
157
|
+
*/
|
|
158
|
+
succeeded(now?: number): void;
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* What the runner needs from a client: the one call it makes, and the three
|
|
162
|
+
* things it reads back for the report and the cache. `Jev` is the real one;
|
|
163
|
+
* a test hands in an object with these four members and no network.
|
|
164
|
+
*/
|
|
165
|
+
export interface AskClient {
|
|
166
|
+
model: string;
|
|
167
|
+
servedModel: string | null;
|
|
168
|
+
readonly spent: Spend;
|
|
169
|
+
askSplitting(state: unknown, questions: Record<string, Question>): Promise<SystemOneResponse>;
|
|
170
|
+
}
|
|
171
|
+
export declare class Jev implements AskClient {
|
|
172
|
+
apiKey: string;
|
|
173
|
+
baseUrl: string;
|
|
174
|
+
model: string;
|
|
175
|
+
retries: number;
|
|
176
|
+
timeoutMs: number;
|
|
177
|
+
onRequest: ((body: string) => void) | null;
|
|
178
|
+
pacer: Pacer;
|
|
179
|
+
rateLimitRetries: number;
|
|
180
|
+
rateLimitWaitMs: number;
|
|
181
|
+
private transport;
|
|
182
|
+
calls: number;
|
|
183
|
+
inputTokens: number;
|
|
184
|
+
outputTokens: number;
|
|
185
|
+
totalMs: number;
|
|
186
|
+
retried: number;
|
|
187
|
+
rateLimited: number;
|
|
188
|
+
splits: number;
|
|
189
|
+
servedModel: string | null;
|
|
190
|
+
constructor({ apiKey, baseUrl, model, retries, timeoutMs, onRequest, pacer, rateLimitRetries, rateLimitWaitMs, fetch: transport, }?: JevOptions);
|
|
191
|
+
/** The paced token rate as it stands, per second. */
|
|
192
|
+
get tokensPerSecond(): number;
|
|
193
|
+
/**
|
|
194
|
+
* What a failed status means for the caller.
|
|
195
|
+
*
|
|
196
|
+
* `too_big` is the one 400 a caller can fix by sending less; naming it is
|
|
197
|
+
* what lets askSplitting recover instead of dropping the batch. `auth` is
|
|
198
|
+
* anything the account has to fix -- a bad key (401, 403) or no credit
|
|
199
|
+
* (402) -- and it is the same answer for every batch and every pass, so the
|
|
200
|
+
* runner stops on the first one rather than collecting it 69 times.
|
|
201
|
+
*/
|
|
202
|
+
static classify(status: number, body: string): JevErrorKind;
|
|
203
|
+
get spent(): Spend;
|
|
204
|
+
get usd(): number;
|
|
205
|
+
/** One request: one state, N questions, N answers. */
|
|
206
|
+
ask(state: unknown, questions: Record<string, Question>): Promise<SystemOneResponse>;
|
|
207
|
+
/**
|
|
208
|
+
* Ask about one state, halving the question set if the server says the
|
|
209
|
+
* request is too big. The state is unchanged by a split, so a file whose
|
|
210
|
+
* SOURCE alone exceeds the 32Ki state budget cannot be rescued here -- the
|
|
211
|
+
* batch planner has to have shrunk the state itself.
|
|
212
|
+
*/
|
|
213
|
+
askSplitting(state: unknown, questions: Record<string, Question>): Promise<SystemOneResponse>;
|
|
214
|
+
}
|
|
215
|
+
/** Bounded-concurrency map that preserves input order. */
|
|
216
|
+
export declare function mapLimit<T, R>(items: T[], limit: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]>;
|
package/dist/jev.js
ADDED
|
@@ -0,0 +1,390 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A rough input-token count for the pacer's mirror of the server's bucket.
|
|
3
|
+
*
|
|
4
|
+
* Deliberately approximate: the pacer corrects itself to the server's own
|
|
5
|
+
* count when the answer comes back, and `askSplitting` reacts to the server's
|
|
6
|
+
* `max_tokens_exceeded` rather than predicting it, so an estimate that is
|
|
7
|
+
* wrong by a third costs nothing.
|
|
8
|
+
*/
|
|
9
|
+
export function estimateTokens(body) {
|
|
10
|
+
return Math.ceil(JSON.stringify(body).length / 3.6);
|
|
11
|
+
}
|
|
12
|
+
export const DEFAULT_BASE_URL = "https://api.typesafe.ai";
|
|
13
|
+
export const DEFAULT_MODEL = "jev-latest";
|
|
14
|
+
/**
|
|
15
|
+
* Where the key and the endpoint come from, in order.
|
|
16
|
+
*
|
|
17
|
+
* `TYPESAFE_*` first because that is the prevailing spelling; `TYPESAFEAI_*`
|
|
18
|
+
* stays as a fallback so an existing environment keeps working without an edit.
|
|
19
|
+
* Both are read rather than one aliased to the other, so a shell that has only
|
|
20
|
+
* the older name set is not a configuration error.
|
|
21
|
+
*/
|
|
22
|
+
export const API_KEY_VARS = ["TYPESAFE_API_KEY", "TYPESAFEAI_API_KEY"];
|
|
23
|
+
export const BASE_URL_VARS = ["TYPESAFE_BASE_URL", "TYPESAFEAI_BASE_URL"];
|
|
24
|
+
/** First of these variables that is set and non-empty. */
|
|
25
|
+
export function fromEnv(names, env = process.env) {
|
|
26
|
+
for (const n of names) {
|
|
27
|
+
const v = env[n];
|
|
28
|
+
if (typeof v === "string" && v.trim() !== "")
|
|
29
|
+
return v.trim();
|
|
30
|
+
}
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
/** Published input price, USD per million input tokens. Output is not billed. */
|
|
34
|
+
export const USD_PER_MTOK = 0.042;
|
|
35
|
+
export class JevError extends Error {
|
|
36
|
+
status;
|
|
37
|
+
kind;
|
|
38
|
+
constructor(message, { status = 0, kind = "other" } = {}) {
|
|
39
|
+
super(message);
|
|
40
|
+
this.name = "JevError";
|
|
41
|
+
this.status = status;
|
|
42
|
+
// "too_big" -> send fewer questions (the only recoverable 400)
|
|
43
|
+
// "auth" -> fix the key; retrying will not help
|
|
44
|
+
// "transient" -> retry
|
|
45
|
+
// "other" -> give up on this batch
|
|
46
|
+
this.kind = kind;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* The most requests in flight at once. Latency is 260 ms plus 5.7 ms per
|
|
51
|
+
* thousand tokens, so 78 requests at 4 abreast take 9.6 s and the same 78
|
|
52
|
+
* fired together take 1.6; what stops "all at once" is the token rate below,
|
|
53
|
+
* not the connection count -- 40 small requests at once have never drawn a
|
|
54
|
+
* 429. Measured on the full run over this repository (79 requests, 2.15M
|
|
55
|
+
* tokens) with the pacer: 16 abreast 4.8 s, 32 3.6-3.7 s, 64 3.9 s. Past 32
|
|
56
|
+
* the server's own latency grows with what it is holding and the wall time
|
|
57
|
+
* stops falling.
|
|
58
|
+
*/
|
|
59
|
+
export const DEFAULT_CONCURRENCY = 32;
|
|
60
|
+
const DEFAULT_RATE_LIMIT_RETRIES = 8;
|
|
61
|
+
const DEFAULT_RATE_LIMIT_WAIT_MS = 300;
|
|
62
|
+
/**
|
|
63
|
+
* The server's rate limit, as measured, and mirrored here so that the client
|
|
64
|
+
* paces itself instead of being told.
|
|
65
|
+
*
|
|
66
|
+
* The server answers a bare 429 -- no retry-after, no ratelimit headers --
|
|
67
|
+
* and what it limits is input tokens, not requests: 40 small requests at
|
|
68
|
+
* once go through, 20 of the largest at once go through, and the same 20
|
|
69
|
+
* again a second later lose 4, then 14, then 16. The fit is a token bucket
|
|
70
|
+
* of about 1.6M tokens refilling at 200-250k per second. A full run over this
|
|
71
|
+
* repository is 2.1M tokens, so it sits at the edge: fired together it loses
|
|
72
|
+
* between 8 and 36 of 78 requests depending on how full the bucket was.
|
|
73
|
+
*
|
|
74
|
+
* The client keeps its own bucket, charged with each request's estimate as it
|
|
75
|
+
* is sent and corrected to the server's count when the answer comes back. A
|
|
76
|
+
* request waits until the bucket can pay for it. Started a little under the
|
|
77
|
+
* measured size, it draws no 429 on a run that starts with a full bucket; a
|
|
78
|
+
* 429 -- the mirror was wrong, or another process shares the key -- empties
|
|
79
|
+
* the mirror and lowers its rate by a quarter, and a success while requests
|
|
80
|
+
* have had to wait raises the rate by two percent, so a key with a higher
|
|
81
|
+
* limit finds it. The rate is bounded on both sides; the burst is not
|
|
82
|
+
* adapted, since a burst too small costs a run of this size two seconds and
|
|
83
|
+
* one too large costs it lost verdicts.
|
|
84
|
+
*/
|
|
85
|
+
export const DEFAULT_TOKENS_PER_SECOND = 200_000;
|
|
86
|
+
export const DEFAULT_TOKEN_BURST = 1_200_000;
|
|
87
|
+
const MIN_TOKENS_PER_SECOND = 20_000;
|
|
88
|
+
const MAX_TOKENS_PER_SECOND = 2_000_000;
|
|
89
|
+
/** A positive number from the environment, or nothing. */
|
|
90
|
+
function envNumber(name) {
|
|
91
|
+
const n = Number(process.env[name]);
|
|
92
|
+
return Number.isFinite(n) && n > 0 ? n : null;
|
|
93
|
+
}
|
|
94
|
+
export class Pacer {
|
|
95
|
+
rate;
|
|
96
|
+
burst;
|
|
97
|
+
level;
|
|
98
|
+
at;
|
|
99
|
+
/** Whether a request has had to wait: only then is the rate probed upward. */
|
|
100
|
+
waited = false;
|
|
101
|
+
/** When the rate last grew. */
|
|
102
|
+
grew = 0;
|
|
103
|
+
constructor(rate = envNumber("JEV_TEST_FILTER_TOKENS_PER_SECOND") ?? DEFAULT_TOKENS_PER_SECOND, burst = envNumber("JEV_TEST_FILTER_TOKEN_BURST") ?? DEFAULT_TOKEN_BURST, now = Date.now()) {
|
|
104
|
+
this.rate = rate;
|
|
105
|
+
this.burst = burst;
|
|
106
|
+
this.level = burst;
|
|
107
|
+
this.at = now;
|
|
108
|
+
}
|
|
109
|
+
refill(now) {
|
|
110
|
+
// A clock that goes backwards -- an NTP step, a suspended laptop waking --
|
|
111
|
+
// must not DRAIN the mirror. Unclamped, a one-second backwards jump takes
|
|
112
|
+
// a second's worth of refill out of the bucket, and the client then waits
|
|
113
|
+
// for a limit the server is not imposing. Treat it as no elapsed time.
|
|
114
|
+
const elapsed = Math.max(0, now - this.at);
|
|
115
|
+
this.level = Math.min(this.burst, this.level + (elapsed / 1000) * this.rate);
|
|
116
|
+
this.at = now;
|
|
117
|
+
}
|
|
118
|
+
/** How many tokens the bucket holds now. */
|
|
119
|
+
available(now = Date.now()) {
|
|
120
|
+
this.refill(now);
|
|
121
|
+
return this.level;
|
|
122
|
+
}
|
|
123
|
+
/** Milliseconds until `tokens` can be paid for; 0 if now. */
|
|
124
|
+
delay(tokens, now = Date.now()) {
|
|
125
|
+
this.refill(now);
|
|
126
|
+
const need = Math.min(tokens, this.burst) - this.level;
|
|
127
|
+
return need <= 0 ? 0 : Math.ceil((need / this.rate) * 1000);
|
|
128
|
+
}
|
|
129
|
+
/** Charge the bucket, waiting until it can pay. */
|
|
130
|
+
async take(tokens) {
|
|
131
|
+
for (;;) {
|
|
132
|
+
const wait = this.delay(tokens);
|
|
133
|
+
if (wait === 0) {
|
|
134
|
+
this.level -= tokens;
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
this.waited = true;
|
|
138
|
+
await new Promise((r) => setTimeout(r, wait));
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
/** The server counted differently from the estimate: charge the difference. */
|
|
142
|
+
settle(estimated, actual) {
|
|
143
|
+
this.level -= actual - estimated;
|
|
144
|
+
}
|
|
145
|
+
/** A 429: the mirror was optimistic. Empty it and slow down. */
|
|
146
|
+
throttled(now = Date.now()) {
|
|
147
|
+
this.refill(now);
|
|
148
|
+
this.level = 0;
|
|
149
|
+
this.rate = Math.max(MIN_TOKENS_PER_SECOND, this.rate * 0.75);
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* A 200 while requests have been waiting: the limit may be higher than
|
|
153
|
+
* mirrored. Probed by time, not by request -- two percent per success
|
|
154
|
+
* compounded over 79 requests to a rate half again the server's and drew
|
|
155
|
+
* the 429s it was meant to avoid.
|
|
156
|
+
*/
|
|
157
|
+
succeeded(now = Date.now()) {
|
|
158
|
+
if (!this.waited || now - this.grew < 500)
|
|
159
|
+
return;
|
|
160
|
+
this.grew = now;
|
|
161
|
+
this.rate = Math.min(MAX_TOKENS_PER_SECOND, this.rate * 1.02);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
export class Jev {
|
|
165
|
+
apiKey;
|
|
166
|
+
baseUrl;
|
|
167
|
+
model;
|
|
168
|
+
retries;
|
|
169
|
+
timeoutMs;
|
|
170
|
+
onRequest;
|
|
171
|
+
pacer;
|
|
172
|
+
rateLimitRetries;
|
|
173
|
+
rateLimitWaitMs;
|
|
174
|
+
transport;
|
|
175
|
+
calls;
|
|
176
|
+
inputTokens;
|
|
177
|
+
outputTokens;
|
|
178
|
+
totalMs;
|
|
179
|
+
retried;
|
|
180
|
+
rateLimited;
|
|
181
|
+
splits;
|
|
182
|
+
servedModel;
|
|
183
|
+
constructor({ apiKey, baseUrl, model, retries = 4, timeoutMs = 60_000, onRequest = null, pacer = null, rateLimitRetries = DEFAULT_RATE_LIMIT_RETRIES, rateLimitWaitMs = DEFAULT_RATE_LIMIT_WAIT_MS, fetch: transport = globalThis.fetch, } = {}) {
|
|
184
|
+
this.apiKey = apiKey ?? fromEnv(API_KEY_VARS) ?? "";
|
|
185
|
+
this.baseUrl = (baseUrl ?? fromEnv(BASE_URL_VARS) ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
|
|
186
|
+
this.model = model ?? process.env.JEV_TEST_FILTER_MODEL ?? DEFAULT_MODEL;
|
|
187
|
+
this.retries = retries;
|
|
188
|
+
this.timeoutMs = timeoutMs;
|
|
189
|
+
/** Called with each request body before it is sent, for leak assertions. */
|
|
190
|
+
this.onRequest = onRequest;
|
|
191
|
+
this.pacer = pacer ?? new Pacer();
|
|
192
|
+
this.rateLimitRetries = rateLimitRetries;
|
|
193
|
+
this.rateLimitWaitMs = rateLimitWaitMs;
|
|
194
|
+
this.transport = transport;
|
|
195
|
+
this.calls = 0;
|
|
196
|
+
this.inputTokens = 0;
|
|
197
|
+
this.outputTokens = 0;
|
|
198
|
+
this.totalMs = 0;
|
|
199
|
+
this.retried = 0;
|
|
200
|
+
this.rateLimited = 0;
|
|
201
|
+
this.splits = 0;
|
|
202
|
+
this.servedModel = null;
|
|
203
|
+
}
|
|
204
|
+
/** The paced token rate as it stands, per second. */
|
|
205
|
+
get tokensPerSecond() {
|
|
206
|
+
return this.pacer.rate;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* What a failed status means for the caller.
|
|
210
|
+
*
|
|
211
|
+
* `too_big` is the one 400 a caller can fix by sending less; naming it is
|
|
212
|
+
* what lets askSplitting recover instead of dropping the batch. `auth` is
|
|
213
|
+
* anything the account has to fix -- a bad key (401, 403) or no credit
|
|
214
|
+
* (402) -- and it is the same answer for every batch and every pass, so the
|
|
215
|
+
* runner stops on the first one rather than collecting it 69 times.
|
|
216
|
+
*/
|
|
217
|
+
static classify(status, body) {
|
|
218
|
+
if (status === 400 && body.includes("max_tokens_exceeded"))
|
|
219
|
+
return "too_big";
|
|
220
|
+
if (status === 401 || status === 402 || status === 403)
|
|
221
|
+
return "auth";
|
|
222
|
+
return "other";
|
|
223
|
+
}
|
|
224
|
+
get spent() {
|
|
225
|
+
return {
|
|
226
|
+
calls: this.calls,
|
|
227
|
+
inputTokens: this.inputTokens,
|
|
228
|
+
outputTokens: this.outputTokens,
|
|
229
|
+
ms: this.totalMs,
|
|
230
|
+
retried: this.retried,
|
|
231
|
+
rateLimited: this.rateLimited,
|
|
232
|
+
tokensPerSecond: Math.round(this.pacer.rate),
|
|
233
|
+
splits: this.splits,
|
|
234
|
+
usd: this.usd,
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
get usd() {
|
|
238
|
+
return (this.inputTokens / 1_000_000) * USD_PER_MTOK;
|
|
239
|
+
}
|
|
240
|
+
/** One request: one state, N questions, N answers. */
|
|
241
|
+
async ask(state, questions) {
|
|
242
|
+
if (!this.apiKey) {
|
|
243
|
+
throw new JevError(`no API key; set ${API_KEY_VARS[0]}`, { kind: "auth" });
|
|
244
|
+
}
|
|
245
|
+
const names = Object.keys(questions);
|
|
246
|
+
if (names.length === 0)
|
|
247
|
+
return { answers: {}, usage: { input_tokens: 0 } };
|
|
248
|
+
const body = JSON.stringify({ model: this.model, state, questions });
|
|
249
|
+
if (this.onRequest)
|
|
250
|
+
this.onRequest(body);
|
|
251
|
+
let last = new JevError("no attempt made");
|
|
252
|
+
const estimated = estimateTokens({ model: this.model, state, questions });
|
|
253
|
+
// Two budgets: `retries` for failures of the request (network, 5xx),
|
|
254
|
+
// `rateLimitRetries` for the server saying "not now". A 429 is answered
|
|
255
|
+
// in 20 ms upstream and means nothing about the request, so it is
|
|
256
|
+
// counted, waited out at a lower rate, and sent again.
|
|
257
|
+
let attempt = 0;
|
|
258
|
+
let limited = 0;
|
|
259
|
+
for (;;) {
|
|
260
|
+
await this.pacer.take(estimated);
|
|
261
|
+
const started = Date.now();
|
|
262
|
+
const ac = new AbortController();
|
|
263
|
+
const timer = setTimeout(() => ac.abort(), this.timeoutMs);
|
|
264
|
+
let res;
|
|
265
|
+
try {
|
|
266
|
+
res = await this.transport(`${this.baseUrl}/v1/systemone`, {
|
|
267
|
+
method: "POST",
|
|
268
|
+
headers: {
|
|
269
|
+
authorization: `Bearer ${this.apiKey}`,
|
|
270
|
+
"content-type": "application/json",
|
|
271
|
+
},
|
|
272
|
+
body,
|
|
273
|
+
signal: ac.signal,
|
|
274
|
+
});
|
|
275
|
+
}
|
|
276
|
+
catch (err) {
|
|
277
|
+
clearTimeout(timer);
|
|
278
|
+
last = new JevError(`network: ${String(err).slice(0, 200)}`, { kind: "transient" });
|
|
279
|
+
if (attempt === this.retries)
|
|
280
|
+
break;
|
|
281
|
+
attempt += 1;
|
|
282
|
+
this.retried += 1;
|
|
283
|
+
await backoff(attempt - 1);
|
|
284
|
+
continue;
|
|
285
|
+
}
|
|
286
|
+
clearTimeout(timer);
|
|
287
|
+
const text = await res.text();
|
|
288
|
+
if (res.ok) {
|
|
289
|
+
const parsed = JSON.parse(text);
|
|
290
|
+
this.pacer.settle(estimated, parsed.usage?.input_tokens ?? estimated);
|
|
291
|
+
this.pacer.succeeded();
|
|
292
|
+
this.calls += 1;
|
|
293
|
+
this.totalMs += Date.now() - started;
|
|
294
|
+
this.inputTokens += parsed.usage?.input_tokens ?? 0;
|
|
295
|
+
this.outputTokens += parsed.usage?.output_tokens ?? 0;
|
|
296
|
+
if (parsed.model)
|
|
297
|
+
this.servedModel = parsed.model;
|
|
298
|
+
return parsed;
|
|
299
|
+
}
|
|
300
|
+
last = new JevError(`HTTP ${res.status}: ${text.slice(0, 240)}`, {
|
|
301
|
+
status: res.status,
|
|
302
|
+
kind: Jev.classify(res.status, text),
|
|
303
|
+
});
|
|
304
|
+
if (res.status === 429) {
|
|
305
|
+
this.pacer.throttled();
|
|
306
|
+
this.rateLimited += 1;
|
|
307
|
+
if (limited === this.rateLimitRetries)
|
|
308
|
+
break;
|
|
309
|
+
limited += 1;
|
|
310
|
+
await rateLimitWait(limited, this.rateLimitWaitMs, res.headers.get("retry-after"));
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
313
|
+
const transient = res.status >= 500;
|
|
314
|
+
if (!transient || attempt === this.retries)
|
|
315
|
+
break;
|
|
316
|
+
last.kind = "transient";
|
|
317
|
+
attempt += 1;
|
|
318
|
+
this.retried += 1;
|
|
319
|
+
await backoff(attempt - 1, res.headers.get("retry-after"));
|
|
320
|
+
}
|
|
321
|
+
throw last;
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* Ask about one state, halving the question set if the server says the
|
|
325
|
+
* request is too big. The state is unchanged by a split, so a file whose
|
|
326
|
+
* SOURCE alone exceeds the 32Ki state budget cannot be rescued here -- the
|
|
327
|
+
* batch planner has to have shrunk the state itself.
|
|
328
|
+
*/
|
|
329
|
+
async askSplitting(state, questions) {
|
|
330
|
+
const names = Object.keys(questions);
|
|
331
|
+
try {
|
|
332
|
+
return await this.ask(state, questions);
|
|
333
|
+
}
|
|
334
|
+
catch (err) {
|
|
335
|
+
if (!(err instanceof JevError) || err.kind !== "too_big" || names.length < 2)
|
|
336
|
+
throw err;
|
|
337
|
+
this.splits += 1;
|
|
338
|
+
const half = Math.ceil(names.length / 2);
|
|
339
|
+
const merged = {
|
|
340
|
+
answers: {},
|
|
341
|
+
usage: { input_tokens: 0, output_tokens: 0 },
|
|
342
|
+
};
|
|
343
|
+
for (const part of [names.slice(0, half), names.slice(half)]) {
|
|
344
|
+
// `part` is a slice of this object's own keys, so the lookup cannot miss.
|
|
345
|
+
const subset = Object.fromEntries(part.map((n) => [n, questions[n]]));
|
|
346
|
+
const res = await this.askSplitting(state, subset);
|
|
347
|
+
Object.assign(merged.answers, res.answers);
|
|
348
|
+
merged.usage.input_tokens += res.usage?.input_tokens ?? 0;
|
|
349
|
+
merged.usage.output_tokens += res.usage?.output_tokens ?? 0;
|
|
350
|
+
}
|
|
351
|
+
return merged;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
/**
|
|
356
|
+
* The wait after a 429. The server sends no retry-after (honoured if it ever
|
|
357
|
+
* does), and the pacer has already emptied its mirror, so this is short: the base,
|
|
358
|
+
* growing by half per repeat, jittered so a burst of refused requests does
|
|
359
|
+
* not come back as a burst, capped at 5 s.
|
|
360
|
+
*/
|
|
361
|
+
function rateLimitWait(nth, baseMs, retryAfter) {
|
|
362
|
+
const hinted = retryAfter ? Number.parseFloat(retryAfter) * 1000 : Number.NaN;
|
|
363
|
+
const wait = Number.isFinite(hinted)
|
|
364
|
+
? hinted
|
|
365
|
+
: Math.min(5_000, baseMs * 1.5 ** (nth - 1)) * (0.5 + Math.random());
|
|
366
|
+
return new Promise((r) => setTimeout(r, wait));
|
|
367
|
+
}
|
|
368
|
+
function backoff(attempt, retryAfter) {
|
|
369
|
+
const hinted = retryAfter ? Number.parseFloat(retryAfter) * 1000 : Number.NaN;
|
|
370
|
+
const wait = Number.isFinite(hinted)
|
|
371
|
+
? hinted
|
|
372
|
+
: Math.min(20_000, 500 * 2 ** attempt) * (0.5 + Math.random());
|
|
373
|
+
return new Promise((r) => setTimeout(r, wait));
|
|
374
|
+
}
|
|
375
|
+
/** Bounded-concurrency map that preserves input order. */
|
|
376
|
+
export async function mapLimit(items, limit, fn) {
|
|
377
|
+
const out = new Array(items.length);
|
|
378
|
+
let next = 0;
|
|
379
|
+
const width = Math.max(1, Math.min(limit, items.length));
|
|
380
|
+
await Promise.all(Array.from({ length: width }, async () => {
|
|
381
|
+
for (;;) {
|
|
382
|
+
const i = next;
|
|
383
|
+
next += 1;
|
|
384
|
+
if (i >= items.length)
|
|
385
|
+
return;
|
|
386
|
+
out[i] = await fn(items[i], i);
|
|
387
|
+
}
|
|
388
|
+
}));
|
|
389
|
+
return out;
|
|
390
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function registerLanguages(): void;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The grammars ast-grep does not bundle.
|
|
3
|
+
*
|
|
4
|
+
* `@ast-grep/napi` ships the web languages only; Rust and Go arrive as
|
|
5
|
+
* separate packages and have to be registered before the first `parse` that
|
|
6
|
+
* names them. Registration is process-wide and rejects a second call for the
|
|
7
|
+
* same language, so it happens once behind a flag rather than per file.
|
|
8
|
+
*/
|
|
9
|
+
import { registerDynamicLanguage } from "@ast-grep/napi";
|
|
10
|
+
import rust from "@ast-grep/lang-rust";
|
|
11
|
+
import go from "@ast-grep/lang-go";
|
|
12
|
+
let registered = false;
|
|
13
|
+
export function registerLanguages() {
|
|
14
|
+
if (registered)
|
|
15
|
+
return;
|
|
16
|
+
registerDynamicLanguage({ rust, go });
|
|
17
|
+
registered = true;
|
|
18
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One question per test.
|
|
3
|
+
*
|
|
4
|
+
* `score` rather than `choice` because the answer is an ordered conclusion:
|
|
5
|
+
* asking "how much can this change break this test" as an unordered choice
|
|
6
|
+
* discards the ordering, splits the probability mass between adjacent levels,
|
|
7
|
+
* and returns a low confidence that is indistinguishable from real
|
|
8
|
+
* uncertainty.
|
|
9
|
+
*
|
|
10
|
+
* No threshold ever appears in a question. The cutoff is a decision the gate
|
|
11
|
+
* makes from the answer; writing it into the question would mean every
|
|
12
|
+
* recalibration rewrote the question, and no run could be compared with an
|
|
13
|
+
* earlier one.
|
|
14
|
+
*/
|
|
15
|
+
import type { TestCase } from "./types.ts";
|
|
16
|
+
export interface ScoreQuestion {
|
|
17
|
+
type: "score";
|
|
18
|
+
instructions: Record<string, unknown>;
|
|
19
|
+
criteria: readonly string[];
|
|
20
|
+
}
|
|
21
|
+
export declare const SCORE_LEVELS: readonly ["Unrelated to this change: running this test cannot produce a different result than before.", "This test exercises code the change touched, but nothing in the change can alter its outcome.", "This test could be affected by the change: it might fail.", "This test directly exercises behaviour the change altered or broke: it is likely to fail."];
|
|
22
|
+
export declare const SCORE_LEVEL_NAMES: readonly ["unrelated", "unaffected", "at-risk", "likely-failing"];
|
|
23
|
+
/** Stable question name, so answers can be matched back by the test's index. */
|
|
24
|
+
export declare function questionId(i: number): string;
|
|
25
|
+
/**
|
|
26
|
+
* The name the test is asked about is the runner's own spelling of it, so a
|
|
27
|
+
* reader of `--json` can paste it into `-t` and see the same test. Playwright
|
|
28
|
+
* is selected by location rather than by name, so its chain is joined the way
|
|
29
|
+
* its reporter prints it.
|
|
30
|
+
*/
|
|
31
|
+
export declare function displayName(t: TestCase): string;
|
|
32
|
+
export declare function buildQuestion(t: TestCase, id: string): ScoreQuestion;
|
|
33
|
+
/**
|
|
34
|
+
* One answer, or null when it cannot be used.
|
|
35
|
+
*
|
|
36
|
+
* Null rather than zero: a malformed answer must select the test, and a zero
|
|
37
|
+
* would silently deselect it.
|
|
38
|
+
*/
|
|
39
|
+
export declare function readAnswer(raw: unknown): {
|
|
40
|
+
value: number;
|
|
41
|
+
confidence: number | null;
|
|
42
|
+
} | null;
|