opencode-zen-freex 0.2.8 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +2 -9
- package/MAINTAINING.md +44 -3
- package/NOTICE +18 -0
- package/README.md +6 -4
- package/core-tools.mjs +85 -7
- package/index.mjs +14 -4
- package/limits.mjs +221 -57
- package/models.mjs +27 -3
- package/package.json +14 -6
- package/provider.mjs +111 -22
- package/stream.mjs +119 -21
- package/tool-response.mjs +56 -11
- package/capture.mjs +0 -231
- package/limits-audit.mjs +0 -340
- package/tools-smoke.mjs +0 -234
package/capture.mjs
DELETED
|
@@ -1,231 +0,0 @@
|
|
|
1
|
-
// Recaptures fingerprint.json from a real OpenCode.
|
|
2
|
-
//
|
|
3
|
-
// The free tier reads the wording of the tools it knows by name out of the
|
|
4
|
-
// request body and refuses the request when it differs from its own copy. That
|
|
5
|
-
// copy moves with OpenCode, so the one held here goes stale — and a stale copy
|
|
6
|
-
// looks exactly like a working one until somebody is refused. This is how a new
|
|
7
|
-
// one is taken, so that refreshing it is a command and not an archaeology
|
|
8
|
-
// exercise.
|
|
9
|
-
//
|
|
10
|
-
// bun run capture # writes fingerprint.json
|
|
11
|
-
// bun run capture -- --dry # reports what changed, writes nothing
|
|
12
|
-
//
|
|
13
|
-
// Needs opencode on PATH (https://opencode.ai) and bun. Nothing goes to Zen: the
|
|
14
|
-
// OpenCode under test is pointed at a server running here, and the request it
|
|
15
|
-
// makes is the thing being read.
|
|
16
|
-
|
|
17
|
-
import { spawn } from "node:child_process";
|
|
18
|
-
import { readFileSync } from "node:fs";
|
|
19
|
-
import { mkdtempSync } from "node:fs";
|
|
20
|
-
import { rmSync } from "node:fs";
|
|
21
|
-
import { tmpdir } from "node:os";
|
|
22
|
-
import { join } from "node:path";
|
|
23
|
-
import { PROFILES } from "./core-tools.mjs";
|
|
24
|
-
|
|
25
|
-
const PORT = Number(process.env.CAPTURE_PORT ?? 8791);
|
|
26
|
-
const DRY = process.argv.includes("--dry");
|
|
27
|
-
const HERE = import.meta.dir;
|
|
28
|
-
const FINGERPRINT = join(HERE, "fingerprint.json");
|
|
29
|
-
|
|
30
|
-
// The names Zen reads, by generation: 2.x sends `shell` where 1.x sends `bash`
|
|
31
|
-
// and carries none of the others. Everything else a host declares is ignored.
|
|
32
|
-
const READ = {
|
|
33
|
-
"1.x": ["bash", "glob", "grep", "read", "write", "edit"],
|
|
34
|
-
"2.x": ["shell", "read"],
|
|
35
|
-
};
|
|
36
|
-
|
|
37
|
-
// The capture host's own scratch directory and OS banner belong to the machine
|
|
38
|
-
// the capture ran on, not to the tool. These are put back to something neutral
|
|
39
|
-
// so the capture is not tied to one machine.
|
|
40
|
-
const NEUTRALISE = [
|
|
41
|
-
[/Be aware: OS: [^\n]*/g, "Be aware: OS: linux, Shell: bash"],
|
|
42
|
-
[/Use `[^`]*` for temporary work outside the workspace\./g, "Use `/tmp` for temporary work outside the workspace."],
|
|
43
|
-
[/Use `[A-Za-z]:\\[^`]*`/g, "Use `/tmp`"],
|
|
44
|
-
];
|
|
45
|
-
// Zen compares roughly the first 700 characters and ignores the rest; 1200
|
|
46
|
-
// leaves margin, and the tail is where a note to the model goes.
|
|
47
|
-
const KEEP = 1200;
|
|
48
|
-
|
|
49
|
-
const capturedPath = join(tmpdir(), `zen-capture-${process.pid}.json`);
|
|
50
|
-
// A run that was killed leaves its file behind and a PID comes round again, so
|
|
51
|
-
// this one is removed rather than read back as if it were this run's capture.
|
|
52
|
-
rmSync(capturedPath, { force: true });
|
|
53
|
-
// OpenCode makes more than one request (a short one for the title, then the
|
|
54
|
-
// real one). The one worth reading is the longest, since it is the one that
|
|
55
|
-
// carries the full toolset. It is kept here rather than written from the handler
|
|
56
|
-
// as it arrives: two requests in flight at once would both be writing this one
|
|
57
|
-
// file, and the writes interleave into it.
|
|
58
|
-
let longest = { agent: null, body: "" };
|
|
59
|
-
const server = Bun.serve({
|
|
60
|
-
port: PORT,
|
|
61
|
-
hostname: "127.0.0.1",
|
|
62
|
-
async fetch(request) {
|
|
63
|
-
const url = new URL(request.url);
|
|
64
|
-
if (url.pathname.endsWith("/models"))
|
|
65
|
-
return Response.json({ object: "list", data: [{ id: "big-pickle", object: "model", owned_by: "opencode" }] });
|
|
66
|
-
const body = await request.text();
|
|
67
|
-
if (body.length > longest.body.length) longest = { agent: request.headers.get("user-agent"), body };
|
|
68
|
-
return new Response(
|
|
69
|
-
[
|
|
70
|
-
'data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"big-pickle","choices":[{"index":0,"delta":{"role":"assistant","content":"OK"},"finish_reason":null}]}',
|
|
71
|
-
'data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"big-pickle","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}',
|
|
72
|
-
"data: [DONE]",
|
|
73
|
-
"",
|
|
74
|
-
].join("\n\n"),
|
|
75
|
-
{ headers: { "content-type": "text/event-stream" } },
|
|
76
|
-
);
|
|
77
|
-
},
|
|
78
|
-
});
|
|
79
|
-
|
|
80
|
-
const home = mkdtempSync(join(tmpdir(), "zen-capture-home-"));
|
|
81
|
-
// Where OpenCode looks for its config is XDG_CONFIG_HOME/opencode, and the
|
|
82
|
-
// three XDG directories are pointed at `home` below.
|
|
83
|
-
const config = join(home, "config", "opencode", "opencode.json");
|
|
84
|
-
await Bun.write(
|
|
85
|
-
config,
|
|
86
|
-
JSON.stringify({ provider: { opencode: { options: { baseURL: `http://127.0.0.1:${PORT}/v1` } } } }),
|
|
87
|
-
);
|
|
88
|
-
|
|
89
|
-
console.log(`listening on 127.0.0.1:${PORT}, running opencode against it`);
|
|
90
|
-
let ran = true;
|
|
91
|
-
const code = await new Promise((done) => {
|
|
92
|
-
const child = spawn("opencode", ["run", "-m", "opencode/big-pickle", "Say OK."], {
|
|
93
|
-
env: {
|
|
94
|
-
...process.env,
|
|
95
|
-
XDG_DATA_HOME: join(home, "data"),
|
|
96
|
-
XDG_CONFIG_HOME: join(home, "config"),
|
|
97
|
-
XDG_CACHE_HOME: join(home, "cache"),
|
|
98
|
-
},
|
|
99
|
-
stdio: ["ignore", "ignore", "inherit"],
|
|
100
|
-
});
|
|
101
|
-
child.on("error", () => {
|
|
102
|
-
console.error("could not run opencode; install it from https://opencode.ai and put it on PATH");
|
|
103
|
-
ran = false;
|
|
104
|
-
done(1);
|
|
105
|
-
});
|
|
106
|
-
child.on("exit", (value) => done(value ?? 0));
|
|
107
|
-
});
|
|
108
|
-
server.stop(true);
|
|
109
|
-
if (!ran) process.exit(1);
|
|
110
|
-
// Written here, once nothing else can be writing it.
|
|
111
|
-
if (longest.body.length) await Bun.write(capturedPath, JSON.stringify(longest));
|
|
112
|
-
|
|
113
|
-
let request;
|
|
114
|
-
try {
|
|
115
|
-
request = JSON.parse(readFileSync(capturedPath, "utf8"));
|
|
116
|
-
} catch {
|
|
117
|
-
console.error("no request was captured; opencode did not reach the local server");
|
|
118
|
-
process.exit(1);
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
let body;
|
|
122
|
-
try {
|
|
123
|
-
body = JSON.parse(request.body);
|
|
124
|
-
} catch {
|
|
125
|
-
console.error("the captured request body is not JSON; nothing to read");
|
|
126
|
-
process.exit(1);
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
// Which generation this client is: the names it sends say so. It is read
|
|
130
|
-
// before the tools are, since the two generations do not send the same names
|
|
131
|
-
// and 2.x has to be recognised by `shell` rather than judged against 1.x's.
|
|
132
|
-
const names = new Set((body.tools ?? []).map((raw) => (raw.function ?? raw)?.name));
|
|
133
|
-
const generation = names.has("shell") ? "2.x" : "1.x";
|
|
134
|
-
const read = READ[generation];
|
|
135
|
-
|
|
136
|
-
const tools = {};
|
|
137
|
-
for (const raw of body.tools ?? []) {
|
|
138
|
-
const tool = raw.function ?? raw;
|
|
139
|
-
if (!tool?.name || !read.includes(tool.name) || !tool.description) continue;
|
|
140
|
-
tools[tool.name] = {
|
|
141
|
-
name: tool.name,
|
|
142
|
-
description: NEUTRALISE.reduce((text, [find, to]) => text.replace(find, to), tool.description).slice(0, KEEP),
|
|
143
|
-
parameters: tool.parameters,
|
|
144
|
-
};
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
const missing = read.filter((name) => !tools[name]);
|
|
148
|
-
if (missing.length) {
|
|
149
|
-
console.error(
|
|
150
|
-
`the captured request has no ${missing.join(", ")}; this OpenCode no longer sends them, so it cannot be captured this way`,
|
|
151
|
-
);
|
|
152
|
-
process.exit(1);
|
|
153
|
-
}
|
|
154
|
-
for (const [name, tool] of Object.entries(tools))
|
|
155
|
-
if (tool.description.length < 100) {
|
|
156
|
-
console.error(`${name}'s description looks truncated (${tool.description.length} chars); not writing`);
|
|
157
|
-
process.exit(1);
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
const required = generation === "2.x" ? ["shell", "read"] : ["bash", "glob", "grep", "read"];
|
|
161
|
-
const absent = required.filter((name) => !tools[name]);
|
|
162
|
-
if (absent.length) {
|
|
163
|
-
console.error(`a ${generation} request must carry ${absent.join(", ")}, and the capture does not`);
|
|
164
|
-
process.exit(1);
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
// Which protocol was captured, since each of 1.x's protocols bundles its own
|
|
168
|
-
// provider-utils and a capture must not overwrite the ones it did not observe.
|
|
169
|
-
// Anthropic is not one this can read: its tools carry `input_schema` where the
|
|
170
|
-
// other two carry `parameters`, so read as the chat one it would write every
|
|
171
|
-
// tool without its parameters and take the chat slot's provider-utils from an
|
|
172
|
-
// anthropic client. Refused rather than written.
|
|
173
|
-
if ((body.tools ?? []).some((raw) => (raw.function ?? raw)?.input_schema !== undefined)) {
|
|
174
|
-
console.error("the captured request is in the anthropic shape, which this script cannot read; nothing written");
|
|
175
|
-
process.exit(1);
|
|
176
|
-
}
|
|
177
|
-
const protocol = body.input !== undefined && body.messages === undefined ? "responses" : "chat";
|
|
178
|
-
|
|
179
|
-
console.log(`\nagent: ${request.agent}`);
|
|
180
|
-
console.log(`tools: ${Object.keys(tools).join(", ")} (${generation}, ${protocol})`);
|
|
181
|
-
if (code !== 0) console.log(`(opencode exited ${code}; its request was still captured)`);
|
|
182
|
-
|
|
183
|
-
const agent = parseAgent(request.agent);
|
|
184
|
-
if (agent.providerUtils) agent.providerUtils = { ...PROFILES[generation]?.agent?.providerUtils, [protocol]: agent.providerUtils[protocol] };
|
|
185
|
-
|
|
186
|
-
const before = PROFILES[generation];
|
|
187
|
-
const agentChanged = JSON.stringify(before?.agent) !== JSON.stringify(agent);
|
|
188
|
-
let changed = 0;
|
|
189
|
-
for (const name of Object.keys(tools)) {
|
|
190
|
-
if (JSON.stringify(before?.tools?.[name]) !== JSON.stringify(tools[name])) changed++;
|
|
191
|
-
console.log(` ${name.padEnd(6)} ${JSON.stringify(before?.tools?.[name]) === JSON.stringify(tools[name]) ? "unchanged" : "CHANGED"} ${tools[name].description.length} chars`);
|
|
192
|
-
}
|
|
193
|
-
if (agent.providerUtils)
|
|
194
|
-
console.log(` agent ${agentChanged ? "CHANGED" : "unchanged"} ${protocol} provider-utils kept separate from the others`);
|
|
195
|
-
|
|
196
|
-
if (DRY) {
|
|
197
|
-
const what = [changed ? `${changed} tool(s)` : null, agentChanged ? "the agent" : null].filter(Boolean);
|
|
198
|
-
console.log(`\n${what.length ? `${what.join(" and ")} would change.` : "Nothing would change."} Nothing written (--dry).`);
|
|
199
|
-
process.exit(what.length ? 1 : 0);
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
if (!changed && !agentChanged) {
|
|
203
|
-
console.log("\nnothing changed.");
|
|
204
|
-
process.exit(0);
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
const next = JSON.parse(readFileSync(FINGERPRINT, "utf8"));
|
|
208
|
-
next.profiles[generation] = { agent, required, tools };
|
|
209
|
-
await Bun.write(FINGERPRINT, JSON.stringify(next, null, 2) + "\n");
|
|
210
|
-
console.log(`\nwrote ${FINGERPRINT} for ${generation}. Now run \`bun run check:gate\` and commit if it passes.`);
|
|
211
|
-
|
|
212
|
-
// The User-Agent is stored the way agentFor() reads it back, which is not
|
|
213
|
-
// necessarily the same split the raw header used.
|
|
214
|
-
function parseAgent(agent) {
|
|
215
|
-
const twoPart = /^opencode\/(\d[^ ]*) ai-sdk\/provider-utils\/(\S+) (\S+)$/.exec(agent ?? "");
|
|
216
|
-
if (twoPart) {
|
|
217
|
-
const [, opencode, providerUtils, runtime] = twoPart;
|
|
218
|
-
return {
|
|
219
|
-
opencode,
|
|
220
|
-
providerUtils: { chat: providerUtils, responses: providerUtils, anthropic: providerUtils },
|
|
221
|
-
runtime,
|
|
222
|
-
};
|
|
223
|
-
}
|
|
224
|
-
const threePart = /^opencode\/([a-z]+)\/(\d[^/]*)\/([a-z-]+)$/.exec(agent ?? "");
|
|
225
|
-
if (threePart) {
|
|
226
|
-
const [, channel, opencode, client] = threePart;
|
|
227
|
-
return { channel, opencode, client };
|
|
228
|
-
}
|
|
229
|
-
console.error(`\ncould not read \`${agent}\` as an OpenCode User-Agent; write PROFILES[${generation}].agent by hand`);
|
|
230
|
-
process.exit(1);
|
|
231
|
-
}
|
package/limits-audit.mjs
DELETED
|
@@ -1,340 +0,0 @@
|
|
|
1
|
-
// Asks the route which free models it still offers, groups them by the upstream
|
|
2
|
-
// model serving them, and says which look under-declared.
|
|
3
|
-
//
|
|
4
|
-
// bun run limits:audit
|
|
5
|
-
//
|
|
6
|
-
// Why this exists. Zen's catalogue is where this plugin declares limits from,
|
|
7
|
-
// and two of the ids it publishes were declared far below what the model does:
|
|
8
|
-
// `deepseek-v4.1-flash-free` at 200000/32000 against 1048576/384000, and
|
|
9
|
-
// `mimo-v2.6-flash-free` at 200000/32000 against 1048576/131072. Both were found
|
|
10
|
-
// by hand, one id at a time. What makes them findable without reading every id
|
|
11
|
-
// by eye is that the route answers `x-opencode-upstream-model-id` on every
|
|
12
|
-
// request: two ids naming one upstream model are one capability, so a
|
|
13
|
-
// declaration that differs inside such a group is the question worth asking.
|
|
14
|
-
//
|
|
15
|
-
// What this does not do: measure a window, or write anything. A window costs a
|
|
16
|
-
// prompt of about a million tokens — minutes and megabytes — and belongs to the
|
|
17
|
-
// one id a signal has already pointed at; the numbers live in limits.mjs and are
|
|
18
|
-
// put there by a measurement, never by a script. What this does ask is cheap:
|
|
19
|
-
// one request of `max_tokens: 1000000` per offered id that speaks the chat
|
|
20
|
-
// shape, stopped as soon as it starts writing. An upstream that validates the
|
|
21
|
-
// ceiling refuses and names its own; one that does not just begins to answer and
|
|
22
|
-
// is cut off after the first chunk. Either way the cost is a handful of tokens,
|
|
23
|
-
// and a 410 or a 400 for a model that is gone is visible in the same request.
|
|
24
|
-
// An id the snapshot places on another protocol is listed and not asked.
|
|
25
|
-
//
|
|
26
|
-
// The offered set is the listing's, not the catalogue's: Zen's /v1/models
|
|
27
|
-
// carries 10 of the 40 free ids the catalogue still describes, and a plugin
|
|
28
|
-
// only ever offers what the listing has. Auditing the other 30 reports a page
|
|
29
|
-
// of `Model is unavailable` about models nobody can pick.
|
|
30
|
-
//
|
|
31
|
-
// What is reported, per offered id: the upstream model and the ids sharing it,
|
|
32
|
-
// the ceiling a refusal named, the window a refusal named, and the status. The
|
|
33
|
-
// candidates at the end are the reason to run it.
|
|
34
|
-
//
|
|
35
|
-
// Exit code is 0 when at least one offered id answered, 1 when none did: being
|
|
36
|
-
// unable to ask is not a finding about limits.
|
|
37
|
-
|
|
38
|
-
import { fileURLToPath } from "node:url";
|
|
39
|
-
import { PROFILES } from "./core-tools.mjs";
|
|
40
|
-
import { LIMITS, limitOverrides, namedContextWindow, namedOutputCap } from "./limits.mjs";
|
|
41
|
-
import { SDK, bootstrapProtocols, isFree } from "./models.mjs";
|
|
42
|
-
import { CATALOG, DEFAULT_BASE } from "./provider.mjs";
|
|
43
|
-
import { chatRequest } from "./tools-smoke.mjs";
|
|
44
|
-
|
|
45
|
-
const BASE = process.env.ZEN_BASE_URL ?? DEFAULT_BASE;
|
|
46
|
-
const CATALOG_URL = process.env.ZEN_CATALOG_URL ?? CATALOG;
|
|
47
|
-
const GENERATION = process.env.ZEN_GENERATION ?? "1.x";
|
|
48
|
-
const READ_BUDGET = 4096;
|
|
49
|
-
const ASKED_CEILING = 1_000_000;
|
|
50
|
-
// How many ids are asked at once. Each ask stops as soon as the answer starts,
|
|
51
|
-
// so the connection and the upstream load stay modest while the 45 s abort of
|
|
52
|
-
// one id no longer holds up every id behind it.
|
|
53
|
-
const POOL = 4;
|
|
54
|
-
|
|
55
|
-
// The ids the plugin would offer: the listing's, kept where the catalogue calls
|
|
56
|
-
// them free — or, for an id neither the catalogue nor the docs describe, where
|
|
57
|
-
// the id names itself free and nothing else names its protocol, which is what
|
|
58
|
-
// models.mjs offers on its own say-so. An id the snapshot already calls
|
|
59
|
-
// something this plugin has no SDK for is not offered by anyone: `jev-1.13-free`
|
|
60
|
-
// is SystemOne, and stays out here for the same reason buildModels leaves it out.
|
|
61
|
-
// "Free" is models.mjs's own answer, retirement included, so the audited set and
|
|
62
|
-
// the offered set cannot drift apart. Exported for the tests.
|
|
63
|
-
export function offeredIds(listing, catalog, snapshot = bootstrapProtocols) {
|
|
64
|
-
return listing.filter((id) => {
|
|
65
|
-
const protocol = snapshot?.[id];
|
|
66
|
-
if (protocol !== undefined && !Object.hasOwn(SDK, protocol)) return false;
|
|
67
|
-
const model = catalog?.[id];
|
|
68
|
-
if (!model) return /-free$/.test(id);
|
|
69
|
-
return isFree(id, model);
|
|
70
|
-
});
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
// Offered ids by the upstream model serving them, which is the route's own
|
|
74
|
-
// answer and not a guess from the id's shape.
|
|
75
|
-
export function groupByUpstream(rows) {
|
|
76
|
-
const groups = new Map();
|
|
77
|
-
for (const row of rows) {
|
|
78
|
-
if (!row.upstream) continue;
|
|
79
|
-
groups.set(row.upstream, [...(groups.get(row.upstream) ?? []), row]);
|
|
80
|
-
}
|
|
81
|
-
return groups;
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
// What is worth a measurement, in the order it should be read. `covered` are the
|
|
85
|
-
// ids limits.mjs already carries evidence for, which are said once and not
|
|
86
|
-
// raised again. Exported for the tests.
|
|
87
|
-
export function findings(rows, covered = new Set()) {
|
|
88
|
-
const out = [];
|
|
89
|
-
for (const row of rows) {
|
|
90
|
-
const declared = row.declared ?? {};
|
|
91
|
-
// An id limits.mjs already carries evidence for is not asked for again: its
|
|
92
|
-
// numbers are in the table and repeating them here would bury what is new.
|
|
93
|
-
// Its status is still printed in the table above, so a model that dies is
|
|
94
|
-
// not hidden by this — only its numbers stop being a question.
|
|
95
|
-
if (covered.has(row.id)) continue;
|
|
96
|
-
if (row.ceiling !== undefined && row.ceiling !== declared.output)
|
|
97
|
-
out.push(
|
|
98
|
-
`${row.id}: the route names a ceiling of ${row.ceiling} completion tokens, the catalogue declares ${declared.output ?? "none"}`,
|
|
99
|
-
);
|
|
100
|
-
if (row.window !== undefined && row.window !== declared.context)
|
|
101
|
-
out.push(
|
|
102
|
-
`${row.id}: a refusal named a window of ${row.window}, the catalogue declares ${declared.context ?? "none"}`,
|
|
103
|
-
);
|
|
104
|
-
const named = row.ceiling !== undefined || row.window !== undefined;
|
|
105
|
-
// A model that does not speak the shape this audit asks with is not a limits
|
|
106
|
-
// finding and must not sit in this list for ever: it is said once, beside the
|
|
107
|
-
// ids that were not asked at all.
|
|
108
|
-
if (protocolScoped(row)) continue;
|
|
109
|
-
if (row.status !== 200 && row.status !== 0 && !named) {
|
|
110
|
-
// What is left is worth reading: a 410, and a 400 that named nothing. A 0
|
|
111
|
-
// is a question that was never answered — timeout, DNS, a closed socket —
|
|
112
|
-
// and the audit's own rule is that being unable to ask is not a finding
|
|
113
|
-
// about limits, so the id, its status and its message stay in the table.
|
|
114
|
-
out.push(`${row.id}: HTTP ${row.status}${row.says ? ` — ${row.says}` : ""}`);
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
for (const [upstream, list] of groupByUpstream(rows)) {
|
|
118
|
-
const largest = (axis) => Math.max(...list.map((row) => row.declared?.[axis] ?? 0));
|
|
119
|
-
if (list.length < 2) continue;
|
|
120
|
-
if (new Set(list.map((row) => `${row.declared?.context}/${row.declared?.output}`)).size < 2) continue;
|
|
121
|
-
for (const row of list) {
|
|
122
|
-
if (covered.has(row.id)) continue;
|
|
123
|
-
if ((row.declared?.context ?? 0) >= largest("context") && (row.declared?.output ?? 0) >= largest("output"))
|
|
124
|
-
continue;
|
|
125
|
-
const siblings = list
|
|
126
|
-
.filter((other) => other.id !== row.id)
|
|
127
|
-
.map((other) => `${other.id} (${other.declared?.context}/${other.declared?.output})`)
|
|
128
|
-
.join(", ");
|
|
129
|
-
out.push(
|
|
130
|
-
`${row.id}: declared ${row.declared?.context}/${row.declared?.output}, and ${upstream} is also served as ${siblings}`,
|
|
131
|
-
);
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
return out;
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
function says(text) {
|
|
138
|
-
const match = text.match(/"(?:message|param)"\s*:\s*"([^"]{0,160})"/);
|
|
139
|
-
if (match) return match[1];
|
|
140
|
-
// Some upstreams refuse in words that are not under a `message` key at all,
|
|
141
|
-
// and a refusal nobody can read is not a finding.
|
|
142
|
-
const flat = text.replace(/\s+/g, " ").trim();
|
|
143
|
-
return flat ? flat.slice(0, 160) : "";
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
// A refusal that names the protocol rather than a limit. This audit asks the
|
|
147
|
-
// chat shape, so a model that speaks Responses or Messages answers by saying so;
|
|
148
|
-
// that is the host's business, not a limit to measure. The wording has to name
|
|
149
|
-
// another shape or say it cannot be spoken at all: `format` alone is how an
|
|
150
|
-
// ordinary `invalid request format` is worded, which is a question that was
|
|
151
|
-
// asked and refused and belongs in the candidate list, not beside the ids that
|
|
152
|
-
// were never asked.
|
|
153
|
-
const OTHER_SHAPE = /protocol|\b(?:responses|messages)\b|anthropic/i;
|
|
154
|
-
|
|
155
|
-
export function protocolScoped(row) {
|
|
156
|
-
return OTHER_SHAPE.test(row?.says ?? "");
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
async function getJSON(url) {
|
|
160
|
-
const response = await fetch(url, { signal: AbortSignal.timeout(30000) });
|
|
161
|
-
if (!response.ok) throw new Error(`${url} answered HTTP ${response.status}`);
|
|
162
|
-
return response.json();
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
// One cheap question per id. The ceiling asked for is past anything this route
|
|
166
|
-
// serves, so an upstream that validates it says what it allows, and the stream
|
|
167
|
-
// is dropped as soon as it starts rather than paying for an answer nobody reads.
|
|
168
|
-
async function ask(id, names) {
|
|
169
|
-
const { headers, body } = chatRequest({
|
|
170
|
-
model: id,
|
|
171
|
-
generation: GENERATION,
|
|
172
|
-
profile: PROFILES[GENERATION],
|
|
173
|
-
names,
|
|
174
|
-
maxTokens: ASKED_CEILING,
|
|
175
|
-
});
|
|
176
|
-
const controller = new AbortController();
|
|
177
|
-
const timer = setTimeout(() => controller.abort(), 45000);
|
|
178
|
-
try {
|
|
179
|
-
const response = await fetch(`${BASE}/chat/completions`, {
|
|
180
|
-
method: "POST",
|
|
181
|
-
headers,
|
|
182
|
-
body: JSON.stringify(body),
|
|
183
|
-
signal: controller.signal,
|
|
184
|
-
});
|
|
185
|
-
let text = "";
|
|
186
|
-
if (response.ok) {
|
|
187
|
-
const reader = response.body?.getReader();
|
|
188
|
-
if (reader) {
|
|
189
|
-
const decoder = new TextDecoder();
|
|
190
|
-
while (text.length < READ_BUDGET) {
|
|
191
|
-
const { value, done } = await reader.read();
|
|
192
|
-
if (done) break;
|
|
193
|
-
text += decoder.decode(value, { stream: true });
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
} else {
|
|
197
|
-
text = (await response.text()).slice(0, READ_BUDGET);
|
|
198
|
-
}
|
|
199
|
-
const upstream = response.headers.get("x-opencode-upstream-model-id") ?? undefined;
|
|
200
|
-
const endpoint = response.headers.get("x-opencode-endpoint-id") ?? undefined;
|
|
201
|
-
// limits.mjs reads refusals, and a refusal is the only place this route
|
|
202
|
-
// states a limit. The body of a 200 is the model's own prose, which an
|
|
203
|
-
// answer about token limits — or one that merely mentions them — would turn
|
|
204
|
-
// into a ceiling nobody stated, and a `named` row then skips its status line
|
|
205
|
-
// in findings() as well.
|
|
206
|
-
return {
|
|
207
|
-
status: response.status,
|
|
208
|
-
upstream,
|
|
209
|
-
endpoint,
|
|
210
|
-
window: response.ok ? undefined : namedContextWindow(text),
|
|
211
|
-
ceiling: response.ok ? undefined : namedOutputCap(text),
|
|
212
|
-
says: response.ok ? "" : says(text),
|
|
213
|
-
};
|
|
214
|
-
} catch (error) {
|
|
215
|
-
return { status: 0, says: error.message, upstream: undefined, endpoint: undefined };
|
|
216
|
-
} finally {
|
|
217
|
-
clearTimeout(timer);
|
|
218
|
-
controller.abort();
|
|
219
|
-
}
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
// Whether this module is the file that was run rather than one the tests
|
|
223
|
-
// import. `import.meta.main` is the direct answer and the one Bun gives, but it
|
|
224
|
-
// arrived in Node 22.18 and nothing before that has it — and an audit that cannot
|
|
225
|
-
// tell whether it was run is an audit that reports success without asking.
|
|
226
|
-
function invokedDirectly() {
|
|
227
|
-
// `typeof` rather than `??`: a runtime without the property throws on the
|
|
228
|
-
// member access itself, where there is nothing to fall back from.
|
|
229
|
-
if (typeof import.meta.main === "boolean") return import.meta.main;
|
|
230
|
-
try {
|
|
231
|
-
return process.argv[1] !== undefined && fileURLToPath(import.meta.url) === process.argv[1];
|
|
232
|
-
} catch {
|
|
233
|
-
return undefined;
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
// Only a run asks the route. The tests import the helpers above and must not
|
|
238
|
-
// reach the network.
|
|
239
|
-
const entry = invokedDirectly();
|
|
240
|
-
if (entry === undefined) {
|
|
241
|
-
console.error(
|
|
242
|
-
"Could not tell whether this file was run or imported, so nothing was asked: neither import.meta.main nor process.argv[1] named an entry point. Run it with `bun run limits:audit`.",
|
|
243
|
-
);
|
|
244
|
-
process.exit(1);
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
if (entry) {
|
|
248
|
-
// A generation nobody captured is a typo or a Zen that has moved on, and is
|
|
249
|
-
// worth saying in one line rather than throwing a TypeError out of the first
|
|
250
|
-
// line of the run.
|
|
251
|
-
if (!Object.hasOwn(PROFILES, GENERATION)) {
|
|
252
|
-
console.error(
|
|
253
|
-
`limits audit — ZEN_GENERATION "${GENERATION}" is not a captured generation (${Object.keys(PROFILES).join(", ")})`,
|
|
254
|
-
);
|
|
255
|
-
process.exitCode = 1;
|
|
256
|
-
} else {
|
|
257
|
-
try {
|
|
258
|
-
await run();
|
|
259
|
-
} catch (error) {
|
|
260
|
-
// Being unable to ask is not a finding about limits, so it is a sentence
|
|
261
|
-
// and an exit code rather than a stack trace about a fetch that failed.
|
|
262
|
-
console.error(`\nlimits audit — unable to ask: ${error?.message ?? error}`);
|
|
263
|
-
console.error("The route or the catalogue could not be read. Check the network and ZEN_BASE_URL, then ask again.");
|
|
264
|
-
process.exitCode = 1;
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
async function run() {
|
|
270
|
-
const names = [...PROFILES[GENERATION].required];
|
|
271
|
-
const [listing, catalog] = await Promise.all([getJSON(`${BASE}/models`), getJSON(CATALOG_URL)]);
|
|
272
|
-
const models = catalog?.opencode?.models ?? {};
|
|
273
|
-
const ids = offeredIds(
|
|
274
|
-
(Array.isArray(listing?.data) ? listing.data : []).map((entry) => entry?.id).filter(Boolean),
|
|
275
|
-
models,
|
|
276
|
-
);
|
|
277
|
-
const free = Object.keys(models).filter(
|
|
278
|
-
(id) => /free/i.test(id) || (models[id].cost?.input === 0 && models[id].cost?.output === 0),
|
|
279
|
-
);
|
|
280
|
-
console.log(`limits audit — ${BASE}`);
|
|
281
|
-
console.log(`${ids.length} free ids are offered; the catalogue still describes ${free.length}\n`);
|
|
282
|
-
|
|
283
|
-
const rows = [];
|
|
284
|
-
// This asks the chat shape only. An id the snapshot places on another protocol
|
|
285
|
-
// is left to the host that speaks it, and said once rather than reported as a
|
|
286
|
-
// finding it cannot be.
|
|
287
|
-
const asked = ids.filter((id) => (bootstrapProtocols[id] ?? "chat") === "chat");
|
|
288
|
-
const elsewhere = ids.filter((id) => !asked.includes(id));
|
|
289
|
-
// The asks are independent, and each one can sit until its 45 s abort, so a
|
|
290
|
-
// few run together: 4 at a time overlaps the waiting without opening a
|
|
291
|
-
// connection per id. Answers are gathered first and printed in `asked` order,
|
|
292
|
-
// which is the order the listing gave.
|
|
293
|
-
for (let at = 0; at < asked.length; at += POOL) {
|
|
294
|
-
const batch = await Promise.all(
|
|
295
|
-
asked.slice(at, at + POOL).map(async (id) => {
|
|
296
|
-
const answer = await ask(id, names);
|
|
297
|
-
return { id, declared: models[id]?.limit ?? {}, ...answer };
|
|
298
|
-
}),
|
|
299
|
-
);
|
|
300
|
-
rows.push(...batch);
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
// What the plugin declares is the catalogue's limit with limits.mjs over it,
|
|
304
|
-
// so that is what is printed; the catalogue's own pair is shown beside it when
|
|
305
|
-
// the two differ, since that is the under-report being answered.
|
|
306
|
-
for (const { id, declared, ...answer } of rows) {
|
|
307
|
-
const effective = { ...declared, ...limitOverrides(id, declared) };
|
|
308
|
-
const moved = effective.context !== declared.context || effective.output !== declared.output;
|
|
309
|
-
const pair = (limit) => `${limit.context ?? "—"}/${limit.output ?? "—"}`;
|
|
310
|
-
const mark = answer.status === 200 ? " " : "! ";
|
|
311
|
-
console.log(
|
|
312
|
-
`${mark}${id.padEnd(34)} ${pair(effective).padStart(17)}` +
|
|
313
|
-
`${moved ? ` (catalogue ${pair(declared)})` : ""} ${answer.status}` +
|
|
314
|
-
`${answer.upstream ? ` ${answer.upstream}${answer.endpoint ? ` via ${answer.endpoint}` : ""}` : ""}` +
|
|
315
|
-
`${answer.ceiling !== undefined ? ` ceiling=${answer.ceiling}` : ""}${answer.window !== undefined ? ` window=${answer.window}` : ""}` +
|
|
316
|
-
`${answer.says ? ` — ${answer.says}` : ""}`,
|
|
317
|
-
);
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
const answered = rows.filter((row) => row.status === 200);
|
|
321
|
-
const candidates = findings(rows, new Set(Object.keys(LIMITS)));
|
|
322
|
-
const skipped = [...elsewhere, ...rows.filter(protocolScoped).map((row) => row.id)];
|
|
323
|
-
if (skipped.length)
|
|
324
|
-
console.log(
|
|
325
|
-
`\nnot asked, this audit asks the chat shape only: ${skipped.map((id) => `${id}${bootstrapProtocols[id] ? ` (${bootstrapProtocols[id]})` : ""}`).join(", ")}`,
|
|
326
|
-
);
|
|
327
|
-
console.log(`\nlimits.mjs already carries evidence for: ${Object.keys(LIMITS).join(", ")}`);
|
|
328
|
-
console.log(`\n${candidates.length} candidate${candidates.length === 1 ? "" : "s"}:`);
|
|
329
|
-
for (const line of candidates) console.log(` ${line}`);
|
|
330
|
-
if (!candidates.length) console.log(" none — every offered id is either measured or declared at what the route states");
|
|
331
|
-
|
|
332
|
-
if (!answered.length) {
|
|
333
|
-
console.log(
|
|
334
|
-
asked.length
|
|
335
|
-
? "\nNothing answered, so this says nothing about limits. Check the network or ZEN_BASE_URL and ask again."
|
|
336
|
-
: "\nNo offered id speaks the chat shape, so nothing was asked and this says nothing about limits.",
|
|
337
|
-
);
|
|
338
|
-
process.exitCode = 1;
|
|
339
|
-
}
|
|
340
|
-
}
|