@volter/twin-openai 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1556 @@
1
+ // OpenAI capability manifest — the EXPECTED REAL-PRODUCT SURFACE (the target), authored
2
+ // top-down from what the OpenAI API actually does — NOT from what this twin has built. This is
3
+ // the honest denominator: most entries start as `todo` and coverage reads LOW until the twin
4
+ // truly reaches 100% of the API. `verify()` (required to count as done) is ground truth;
5
+ // `expected:'done'` only on capabilities we genuinely claim, so a broken one shows as a
6
+ // regression. Grow this toward the API's *full* surface every cycle.
7
+ //
8
+ // THE HONEST CARVE-OUTS: `openai.chat.inference` and `openai.embeddings.real_vectors` are the
9
+ // out-of-scope entries — the twin returns a DETERMINISTIC STUB for model generation (no
10
+ // weights) and DETERMINISTIC pseudo-vectors for embeddings, while the protocol envelope (shape/
11
+ // streaming/tool_calls/usage) is faithful. `openai.images.real_pixels` and
12
+ // `openai.files.real_bytes` carve out the binary content. Surfaced here + in README ## Coverage.
13
+ import { mkdtempSync, rmSync } from 'node:fs';
14
+ import { tmpdir } from 'node:os';
15
+ import { join } from 'node:path';
16
+ import { checkCapabilities, type CapabilityReport, type CapabilitySpec, verifyBoundary, isInfrastructureError, harnessError } from '@volter/twin-tooling';
17
+ import { handleOpenAITwinRequest, type OpenAIResponseEnvelope } from './openai-twin.ts';
18
+ import { buildSignedOpenAIWebhook, verifyOpenAIWebhook, OpenAIWebhookVerificationError, computeOpenAIWebhookSignature } from './openai-webhooks.ts';
19
+ import type { SseEvent } from './openai-types.ts';
20
+
21
+ // (OpenAI is an API-first vendor with no meaningful product UI worth mirroring — ARCHITECTURE.md
22
+ // C1b — so this pack ships no mirror, and there are no UI capabilities to verify.)
23
+
24
+ // ── API verify: drive REAL requests against a fresh temp root, then assert status/shape ──
25
+ type Step = { m: string; p: string; b?: unknown };
26
+ type Body = Record<string, any>;
27
+
28
+ /** Run a sequence of real OpenAI requests against an isolated root; return all responses. */
29
+ async function withRoot(steps: (h: (s: Step) => Promise<OpenAIResponseEnvelope>) => Promise<boolean>): Promise<boolean> {
30
+ const root = mkdtempSync(join(tmpdir(), 'openai-cap-'));
31
+ const h = (s: Step) => handleOpenAITwinRequest({ method: s.m, path: s.p, body: s.b === undefined ? undefined : JSON.stringify(s.b), root });
32
+ try {
33
+ return await verifyBoundary('openai.withRoot', () => steps(h));
34
+ } finally {
35
+ rmSync(root, { recursive: true, force: true });
36
+ }
37
+ }
38
+
39
+ /** Collect the streaming SSE events for a request against an isolated root. */
40
+ function withStream(body: unknown, fn: (events: SseEvent[], final: OpenAIResponseEnvelope) => boolean): Promise<boolean> {
41
+ return new Promise<boolean>((resolve, reject) => {
42
+ const root = mkdtempSync(join(tmpdir(), 'openai-cap-'));
43
+ const events: SseEvent[] = [];
44
+ handleOpenAITwinRequest({ method: 'POST', path: bodyPath(body), body: JSON.stringify(body), root, sseSink: (e) => events.push(e) })
45
+ .then((final) => resolve(fn(events, final)))
46
+ .catch((err) => { if (isInfrastructureError(err)) reject(harnessError('openai.withStream', err)); else resolve(false); })
47
+ .finally(() => rmSync(root, { recursive: true, force: true }));
48
+ });
49
+ }
50
+ function bodyPath(body: unknown): string {
51
+ return body && typeof body === 'object' && 'input' in (body as Record<string, unknown>) ? '/v1/responses' : '/v1/chat/completions';
52
+ }
53
+
54
+ /** Like withRoot, but the request helper passes request HEADERS through (for auth/rate-limit/
55
+ * idempotency, which the trusted no-headers withRoot helper deliberately never triggers). */
56
+ type StepH = Step & { headers?: Record<string, string> };
57
+ async function withRootH(steps: (h: (s: StepH) => Promise<OpenAIResponseEnvelope>) => Promise<boolean>): Promise<boolean> {
58
+ const root = mkdtempSync(join(tmpdir(), 'openai-cap-'));
59
+ const h = (s: StepH) => handleOpenAITwinRequest({ method: s.m, path: s.p, body: s.b === undefined ? undefined : JSON.stringify(s.b), root, ...(s.headers ? { headers: s.headers } : {}) });
60
+ try {
61
+ return await verifyBoundary('openai.withRootH', () => steps(h));
62
+ } finally {
63
+ rmSync(root, { recursive: true, force: true });
64
+ }
65
+ }
66
+
67
+ const ok = (r: OpenAIResponseEnvelope) => r.status >= 200 && r.status < 300;
68
+ const id = (r: OpenAIResponseEnvelope) => (r.body as Body)?.id as string;
69
+ const field = (r: OpenAIResponseEnvelope, k: string) => (r.body as Body)?.[k];
70
+
71
+ // ── shorthands (mirror the stripe/anthropic manifests) ──
72
+ const done = (id: string, area: string, title: string, dimension: CapabilitySpec['dimension'], tier: CapabilitySpec['tier'], verify: CapabilitySpec['verify']): CapabilitySpec => ({ id, area, title, dimension, tier, expected: 'done', verify });
73
+ const todo = (id: string, area: string, title: string, dimension: CapabilitySpec['dimension'], tier: CapabilitySpec['tier']): CapabilitySpec => ({ id, area, title, dimension, tier, expected: 'todo' });
74
+ const outOfScope = (id: string, area: string, title: string, dimension: CapabilitySpec['dimension'], tier: CapabilitySpec['tier'], reason: string): CapabilitySpec => ({ id, area, title, dimension, tier, expected: 'todo', outOfScope: reason });
75
+
76
+ const CHAT = (extra: Record<string, unknown> = {}) => ({ model: 'gpt-4o', messages: [{ role: 'user', content: 'hello twin' }], ...extra });
77
+
78
+ export const OPENAI_CAPABILITIES: CapabilitySpec[] = [
79
+ // ── THE HONEST CARVE-OUTS ─────────────────────────────────────────────────────────────
80
+ outOfScope('openai.chat.inference', 'chat', 'Real model inference (generation from model weights)', 'api', 'core',
81
+ 'Out of scope: the twin cannot run the model — chat/responses return a DETERMINISTIC STUB completion (clearly labeled), never real model output. The protocol envelope (shape/streaming/tool_calls/usage) is faithful; only the generated text is a stub.'),
82
+ outOfScope('openai.embeddings.real_vectors', 'embeddings', 'Real embedding vectors (semantic values)', 'api', 'core',
83
+ 'Out of scope: the twin cannot run the embedding model — /v1/embeddings returns DETERMINISTIC pseudo-vectors seeded from the input hash. The shape, dimensions, and determinism are faithful; the values carry no semantic meaning.'),
84
+ outOfScope('openai.images.real_pixels', 'images', 'Real image pixels (generated image bytes)', 'api', 'common',
85
+ 'Out of scope: the twin cannot run the image model — /v1/images/generations returns the faithful response shape with a placeholder URL/revised_prompt; no real pixels are produced.'),
86
+ outOfScope('openai.files.real_bytes', 'files', 'Real opaque file storage of arbitrary bytes', 'api', 'niche',
87
+ 'Out of scope: the twin stores file metadata + supplied text content faithfully, but does not persist arbitrary binary blobs as a real object store would.'),
88
+
89
+ // ── Chat Completions (the protocol envelope — faithful) ────────────────────────────────
90
+ done('openai.chat.create', 'chat', 'Chat: create → faithful envelope (id/object/created/model/choices/usage)', 'api', 'core', () =>
91
+ withRoot(async (h) => {
92
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
93
+ if (!ok(r)) return false;
94
+ const b = r.body as Body;
95
+ if (b.object !== 'chat.completion' || b.model !== 'gpt-4o' || typeof b.created !== 'number') return false;
96
+ if (!String(b.id).startsWith('chatcmpl-')) return false;
97
+ const c = b.choices?.[0];
98
+ if (!c || c.message?.role !== 'assistant' || typeof c.message?.content !== 'string' || c.finish_reason !== 'stop') return false;
99
+ const u = b.usage;
100
+ return typeof u.prompt_tokens === 'number' && typeof u.completion_tokens === 'number' && u.total_tokens === u.prompt_tokens + u.completion_tokens;
101
+ }),
102
+ ),
103
+ done('openai.chat.stub_labeled', 'chat', 'Stub completion is clearly labeled as a twin stub (not real output)', 'api', 'core', () =>
104
+ withRoot(async (h) => {
105
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
106
+ const text = (r.body as Body).choices?.[0]?.message?.content as string;
107
+ return ok(r) && text.includes('[twin-stub') && text.includes('hello twin');
108
+ }),
109
+ ),
110
+ done('openai.chat.validation', 'chat', 'Chat validation (model required, non-empty messages, valid roles)', 'api', 'core', () =>
111
+ withRoot(async (h) => {
112
+ const noModel = await h({ m: 'POST', p: '/v1/chat/completions', b: { messages: [{ role: 'user', content: 'x' }] } });
113
+ const noMsg = await h({ m: 'POST', p: '/v1/chat/completions', b: { model: 'gpt-4o' } });
114
+ const empty = await h({ m: 'POST', p: '/v1/chat/completions', b: { model: 'gpt-4o', messages: [] } });
115
+ return noModel.status === 400 && noMsg.status === 400 && empty.status === 400 && (noModel.body as Body).error?.type === 'invalid_request_error';
116
+ }),
117
+ ),
118
+ done('openai.chat.n_choices', 'chat', 'Chat: n returns multiple choices (indexed)', 'api', 'common', () =>
119
+ withRoot(async (h) => {
120
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ n: 3 }) });
121
+ const choices = (r.body as Body).choices as Body[];
122
+ return ok(r) && choices.length === 3 && choices[0]!.index === 0 && choices[2]!.index === 2;
123
+ }),
124
+ ),
125
+ done('openai.chat.max_tokens', 'chat', 'Chat: max_tokens/max_completion_tokens caps output (finish_reason length)', 'api', 'common', () =>
126
+ withRoot(async (h) => {
127
+ const a = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ max_tokens: 1, messages: [{ role: 'user', content: 'please produce a long answer that exceeds one token' }] }) });
128
+ const b = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ max_completion_tokens: 1, messages: [{ role: 'user', content: 'please produce a long answer that exceeds one token' }] }) });
129
+ return ok(a) && ok(b) && (a.body as Body).choices[0].finish_reason === 'length' && (b.body as Body).choices[0].finish_reason === 'length';
130
+ }),
131
+ ),
132
+ done('openai.chat.system_message', 'chat', 'Chat: system/developer messages count toward prompt_tokens', 'api', 'common', () =>
133
+ withRoot(async (h) => {
134
+ const without = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
135
+ const withSys = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ messages: [{ role: 'system', content: 'You are a careful, verbose assistant.' }, { role: 'user', content: 'hello twin' }] }) });
136
+ return ok(without) && ok(withSys) && (withSys.body as Body).usage.prompt_tokens > (without.body as Body).usage.prompt_tokens;
137
+ }),
138
+ ),
139
+ done('openai.chat.multi_turn', 'chat', 'Chat: multi-turn user/assistant history accepted', 'api', 'common', () =>
140
+ withRoot(async (h) => {
141
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ messages: [{ role: 'user', content: 'first' }, { role: 'assistant', content: 'reply' }, { role: 'user', content: 'second' }] }) });
142
+ return ok(r) && (r.body as Body).object === 'chat.completion';
143
+ }),
144
+ ),
145
+ done('openai.chat.content_parts', 'chat', 'Chat: content-part array input (text/image_url) accepted', 'api', 'common', () =>
146
+ withRoot(async (h) => {
147
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ messages: [{ role: 'user', content: [{ type: 'text', text: 'describe' }, { type: 'image_url', image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' } }] }] }) });
148
+ return ok(r) && (r.body as Body).usage.prompt_tokens > 0;
149
+ }),
150
+ ),
151
+ done('openai.chat.stop', 'chat', 'Chat: stop sequence truncates the stub text', 'api', 'common', () =>
152
+ withRoot(async (h) => {
153
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ stop: ['Echoing'] }) });
154
+ const text = (r.body as Body).choices[0].message.content as string;
155
+ return ok(r) && !text.includes('Echoing');
156
+ }),
157
+ ),
158
+ done('openai.chat.temperature_accepted', 'chat', 'Chat: temperature/top_p accepted (ignored for the stub)', 'api', 'niche', () =>
159
+ withRoot(async (h) => {
160
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ temperature: 0.7, top_p: 0.9 }) });
161
+ return ok(r) && (r.body as Body).object === 'chat.completion';
162
+ }),
163
+ ),
164
+ done('openai.chat.deterministic_usage', 'chat', 'Chat: usage token counts deterministic for a fixed request', 'api', 'common', () =>
165
+ withRoot(async (h) => {
166
+ const a = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
167
+ const b = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
168
+ if (!ok(a) || !ok(b)) return false;
169
+ const ua = (a.body as Body).usage as { prompt_tokens?: number; completion_tokens?: number; total_tokens?: number } | undefined;
170
+ const ub = (b.body as Body).usage as { prompt_tokens?: number; completion_tokens?: number; total_tokens?: number } | undefined;
171
+ // usage must be a REAL populated token-count object, not just "the same as each other" (undefined === undefined is trivially true).
172
+ if (!ua || typeof ua.prompt_tokens !== 'number' || ua.prompt_tokens <= 0) return false;
173
+ if (typeof ua.completion_tokens !== 'number' || ua.completion_tokens <= 0) return false;
174
+ if (ua.total_tokens !== ua.prompt_tokens + ua.completion_tokens) return false;
175
+ if (JSON.stringify(ua) !== JSON.stringify(ub)) return false;
176
+ // the id must be a real, well-formed, content-derived id (deterministic hash), not merely equal-because-both-undefined.
177
+ const ida = id(a);
178
+ const idb = id(b);
179
+ return typeof ida === 'string' && ida.startsWith('chatcmpl-') && ida === idb;
180
+ }),
181
+ ),
182
+ done('openai.chat.system_fingerprint', 'chat', 'Chat: response carries system_fingerprint', 'api', 'niche', () =>
183
+ withRoot(async (h) => {
184
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
185
+ return ok(r) && typeof (r.body as Body).system_fingerprint === 'string';
186
+ }),
187
+ ),
188
+ done('openai.chat.json_mode', 'chat', 'response_format json_object / json_schema → valid JSON content', 'api', 'common', () =>
189
+ withRoot(async (h) => {
190
+ const obj = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ response_format: { type: 'json_object' } }) });
191
+ if (!ok(obj)) return false;
192
+ const objText = (obj.body as Body).choices[0].message.content as string;
193
+ let parsedObj: any; try { parsedObj = JSON.parse(objText); } catch (err) { if (isInfrastructureError(err)) throw harnessError('openai.chat.json_mode', err); return false; }
194
+ if (typeof parsedObj !== 'object') return false;
195
+ const schema = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ response_format: { type: 'json_schema', json_schema: { name: 'r', schema: { type: 'object', properties: { title: { type: 'string' }, count: { type: 'integer' }, ok: { type: 'boolean' } } } } } }) });
196
+ const schemaText = (schema.body as Body).choices[0].message.content as string;
197
+ let parsed: any; try { parsed = JSON.parse(schemaText); } catch (err) { if (isInfrastructureError(err)) throw harnessError('openai.chat.json_mode', err); return false; }
198
+ // every declared property is present + type-appropriate
199
+ return 'title' in parsed && typeof parsed.title === 'string' && parsed.count === 0 && parsed.ok === false;
200
+ }),
201
+ ),
202
+ done('openai.chat.logprobs', 'chat', 'logprobs + top_logprobs in choices (faithful per-token shape)', 'api', 'niche', () =>
203
+ withRoot(async (h) => {
204
+ // no logprobs param → choices[].logprobs is null
205
+ const off = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
206
+ if (!ok(off) || (off.body as Body).choices[0].logprobs !== null) return false;
207
+ // logprobs:true → content[] with token/logprob/bytes; re-joining tokens reconstructs the text
208
+ const on = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ logprobs: true }) });
209
+ const lp = (on.body as Body).choices[0].logprobs;
210
+ if (!lp || !Array.isArray(lp.content) || lp.content.length === 0) return false;
211
+ const tok = lp.content[0];
212
+ if (typeof tok.token !== 'string' || typeof tok.logprob !== 'number' || tok.logprob > 0 || !Array.isArray(tok.bytes)) return false;
213
+ const joined = lp.content.map((t: Body) => t.token).join('');
214
+ if (joined !== (on.body as Body).choices[0].message.content) return false;
215
+ // top_logprobs:2 → each token carries up to 2 alternatives
216
+ const top = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ logprobs: true, top_logprobs: 2 }) });
217
+ const t0 = (top.body as Body).choices[0].logprobs.content[0];
218
+ if (t0.top_logprobs.length !== 2) return false;
219
+ // top_logprobs without logprobs:true → 400
220
+ const bad = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ top_logprobs: 2 }) });
221
+ const oob = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ logprobs: true, top_logprobs: 99 }) });
222
+ return bad.status === 400 && oob.status === 400;
223
+ }),
224
+ ),
225
+ done('openai.chat.seed', 'chat', 'seed accepted + reflected (distinct seed → distinct id; same seed deterministic)', 'api', 'niche', () =>
226
+ withRoot(async (h) => {
227
+ const a = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ seed: 42 }) });
228
+ const a2 = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ seed: 42 }) });
229
+ const b = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ seed: 7 }) });
230
+ if (!ok(a) || !ok(b)) return false;
231
+ // same seed → identical id (deterministic); different seed → different id
232
+ if (id(a) !== id(a2) || id(a) === id(b)) return false;
233
+ const bad = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ seed: 1.5 }) });
234
+ return bad.status === 400 && (bad.body as Body).error?.param === 'seed';
235
+ }),
236
+ ),
237
+ done('openai.chat.logit_bias', 'chat', 'logit_bias accepted + validated (object of [-100,100] numbers)', 'api', 'niche', () =>
238
+ withRoot(async (h) => {
239
+ const ok1 = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ logit_bias: { '50256': -100, '1734': 25 } }) });
240
+ if (!ok(ok1) || (ok1.body as Body).object !== 'chat.completion') return false;
241
+ const notObj = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ logit_bias: [1, 2] }) });
242
+ const oob = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ logit_bias: { '50256': 500 } }) });
243
+ return notObj.status === 400 && oob.status === 400 && (oob.body as Body).error?.param === 'logit_bias';
244
+ }),
245
+ ),
246
+ done('openai.chat.audio', 'chat', 'Audio input/output modalities (input_audio parts + audio output envelope; bytes are a labeled stub)', 'api', 'niche', () =>
247
+ withRoot(async (h) => {
248
+ // INPUT: an input_audio content part is accepted + counts toward prompt_tokens.
249
+ const withAudioIn = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ messages: [{ role: 'user', content: [{ type: 'input_audio', input_audio: { data: 'BASE64AUDIO', format: 'wav' } }] }] }) });
250
+ if (!ok(withAudioIn) || (withAudioIn.body as Body).usage.prompt_tokens <= 0) return false;
251
+ // OUTPUT: modalities:['text','audio'] + audio:{voice,format} → message.audio envelope.
252
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ modalities: ['text', 'audio'], audio: { voice: 'alloy', format: 'mp3' } }) });
253
+ if (!ok(r)) return false;
254
+ const msg = (r.body as Body).choices[0].message;
255
+ if (msg.content !== null || !msg.audio) return false;
256
+ const a = msg.audio;
257
+ if (typeof a.id !== 'string' || typeof a.data !== 'string' || typeof a.transcript !== 'string' || typeof a.expires_at !== 'number') return false;
258
+ // the bytes are a LABELED stub (decode the base64 and assert the twin-stub marker + no real synthesis claim)
259
+ const decoded = Buffer.from(a.data, 'base64').toString('utf8');
260
+ if (!decoded.includes('[twin-stub') || !decoded.includes('no real audio synthesis')) return false;
261
+ if (!a.transcript.includes('[twin-stub') || !a.transcript.includes('hello twin')) return false;
262
+ // modalities includes 'audio' but audio missing → 400
263
+ const bad = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ modalities: ['text', 'audio'] }) });
264
+ return bad.status === 400 && (bad.body as Body).error?.param === 'audio';
265
+ }),
266
+ ),
267
+ done('openai.chat.prediction', 'chat', 'Predicted outputs (prediction param → echoed + accepted_prediction_tokens)', 'api', 'niche', () =>
268
+ withRoot(async (h) => {
269
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ prediction: { type: 'content', content: 'the predicted text body' } }) });
270
+ if (!ok(r)) return false;
271
+ const b = r.body as Body;
272
+ const text = b.choices[0].message.content as string;
273
+ if (!text.includes('the predicted text body') || !text.includes('[twin-stub')) return false;
274
+ const details = b.usage.completion_tokens_details;
275
+ if (!details || details.accepted_prediction_tokens <= 0 || details.rejected_prediction_tokens !== 0) return false;
276
+ const bad = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ prediction: { type: 'content' } }) });
277
+ return bad.status === 400 && (bad.body as Body).error?.param === 'prediction';
278
+ }),
279
+ ),
280
+ done('openai.chat.store_metadata', 'chat', 'store + metadata → stored completion retrieve / list / messages / delete', 'api', 'niche', () =>
281
+ withRoot(async (h) => {
282
+ // store:false (default) → not retrievable
283
+ const ns = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
284
+ const nsGet = await h({ m: 'GET', p: `/v1/chat/completions/${id(ns)}` });
285
+ if (nsGet.status !== 404) return false;
286
+ // store:true + metadata → retrievable; metadata reflected
287
+ const c = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ store: true, metadata: { tag: 'unit' } }) });
288
+ if (!ok(c) || (c.body as Body).metadata?.tag !== 'unit') return false;
289
+ const g = await h({ m: 'GET', p: `/v1/chat/completions/${id(c)}` });
290
+ if (!ok(g) || id(g) !== id(c) || (g.body as Body).object !== 'chat.completion') return false;
291
+ const list = await h({ m: 'GET', p: '/v1/chat/completions' });
292
+ if (!ok(list) || (list.body as Body).object !== 'list' || ((list.body as Body).data as Body[]).length !== 1) return false;
293
+ const msgs = await h({ m: 'GET', p: `/v1/chat/completions/${id(c)}/messages` });
294
+ if (!ok(msgs) || ((msgs.body as Body).data as Body[]).length < 1) return false;
295
+ const del = await h({ m: 'DELETE', p: `/v1/chat/completions/${id(c)}` });
296
+ if (!ok(del) || (del.body as Body).deleted !== true) return false;
297
+ const after = await h({ m: 'GET', p: `/v1/chat/completions/${id(c)}` });
298
+ return after.status === 404;
299
+ }),
300
+ ),
301
+ done('openai.chat.stream_options_usage', 'chat', 'stream_options.include_usage emits a final usage-only chunk', 'api', 'common', () =>
302
+ withStream(CHAT({ stream: true, stream_options: { include_usage: true } }), (events, final) => {
303
+ const usageChunk = events.filter((e) => !e.done).find((e) => e.data!.usage !== undefined && (e.data!.choices as Body[]).length === 0);
304
+ if (!usageChunk) return false;
305
+ const u = usageChunk.data!.usage as Body;
306
+ const fu = (final.body as Body).usage;
307
+ return u.total_tokens === fu.total_tokens && u.prompt_tokens === fu.prompt_tokens;
308
+ }),
309
+ ),
310
+
311
+ // ── Streaming ─────────────────────────────────────────────────────────────────────────
312
+ done('openai.streaming.chunks', 'streaming', 'Streaming chat.completion.chunk sequence ends with [DONE]', 'api', 'core', () =>
313
+ withStream(CHAT({ stream: true }), (events) => {
314
+ const data = events.filter((e) => !e.done);
315
+ const hasRole = data.some((e) => (e.data!.choices as Body[])[0]?.delta?.role === 'assistant');
316
+ const allChunks = data.every((e) => e.data!.object === 'chat.completion.chunk');
317
+ const doneLast = events.length > 0 && events[events.length - 1]!.done === true;
318
+ return hasRole && allChunks && doneLast;
319
+ }),
320
+ ),
321
+ done('openai.streaming.reconstruct', 'streaming', 'Streaming content deltas reconstruct the full message text', 'api', 'core', () =>
322
+ withStream(CHAT({ stream: true }), (events, final) => {
323
+ const text = events.filter((e) => !e.done).map((e) => (e.data!.choices as Body[])[0]?.delta?.content ?? '').join('');
324
+ const full = (final.body as Body).choices[0].message.content as string;
325
+ return text === full && text.includes('[twin-stub');
326
+ }),
327
+ ),
328
+ done('openai.streaming.finish_reason', 'streaming', 'Streaming final chunk carries finish_reason', 'api', 'common', () =>
329
+ withStream(CHAT({ stream: true }), (events) => {
330
+ const withFinish = events.filter((e) => !e.done).find((e) => (e.data!.choices as Body[])[0]?.finish_reason === 'stop');
331
+ return !!withFinish;
332
+ }),
333
+ ),
334
+ done('openai.streaming.tool_calls', 'streaming', 'Streaming a tool_call emits function name + arguments deltas', 'api', 'common', () =>
335
+ withStream(CHAT({ stream: true, tools: [{ type: 'function', function: { name: 'lookup', parameters: { type: 'object' } } }] }), (events) => {
336
+ const data = events.filter((e) => !e.done);
337
+ const nameDelta = data.find((e) => (e.data!.choices as Body[])[0]?.delta?.tool_calls?.[0]?.function?.name === 'lookup');
338
+ const finish = data.find((e) => (e.data!.choices as Body[])[0]?.finish_reason === 'tool_calls');
339
+ return !!nameDelta && !!finish;
340
+ }),
341
+ ),
342
+ done('openai.streaming.usage_chunk', 'streaming', 'No usage chunk without include_usage; present + last (before [DONE]) with it', 'api', 'common', async () => {
343
+ const without = await withStream(CHAT({ stream: true }), (events) =>
344
+ !events.filter((e) => !e.done).some((e) => e.data!.usage !== undefined));
345
+ const withU = await withStream(CHAT({ stream: true, stream_options: { include_usage: true } }), (events) => {
346
+ const idxUsage = events.findIndex((e) => !e.done && e.data!.usage !== undefined);
347
+ const idxDone = events.findIndex((e) => e.done === true);
348
+ return idxUsage >= 0 && idxDone === events.length - 1 && idxUsage === idxDone - 1;
349
+ });
350
+ return without && withU;
351
+ }),
352
+
353
+ // ── Tool / function calling ───────────────────────────────────────────────────────────
354
+ done('openai.tools.tool_calls', 'tools', 'Tools provided → tool_calls + finish_reason tool_calls', 'api', 'core', () =>
355
+ withRoot(async (h) => {
356
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ tools: [{ type: 'function', function: { name: 'get_weather', parameters: { type: 'object', properties: { city: { type: 'string' } } } } }] }) });
357
+ if (!ok(r)) return false;
358
+ const c = (r.body as Body).choices[0];
359
+ const call = c.message?.tool_calls?.[0];
360
+ return c.finish_reason === 'tool_calls' && c.message.content === null && call?.type === 'function' && call.function.name === 'get_weather' && String(call.id).startsWith('call_');
361
+ }),
362
+ ),
363
+ done('openai.tools.legacy_functions', 'tools', 'Legacy functions param → tool_call for the named function', 'api', 'common', () =>
364
+ withRoot(async (h) => {
365
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ functions: [{ name: 'legacy_fn', parameters: { type: 'object' } }] }) });
366
+ const call = (r.body as Body).choices[0]?.message?.tool_calls?.[0];
367
+ return ok(r) && call?.function?.name === 'legacy_fn';
368
+ }),
369
+ ),
370
+ done('openai.tools.no_tools_text', 'tools', 'No tools → plain text content + finish_reason stop', 'api', 'common', () =>
371
+ withRoot(async (h) => {
372
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
373
+ const c = (r.body as Body).choices[0];
374
+ return ok(r) && typeof c.message.content === 'string' && c.finish_reason === 'stop' && !c.message.tool_calls;
375
+ }),
376
+ ),
377
+ done('openai.tools.tool_choice', 'tools', 'tool_choice none→text / required+named→that tool / parallel_tool_calls', 'api', 'common', () =>
378
+ withRoot(async (h) => {
379
+ const tools = [
380
+ { type: 'function', function: { name: 'get_weather', parameters: { type: 'object', properties: { city: { type: 'string' } } } } },
381
+ { type: 'function', function: { name: 'get_time', parameters: { type: 'object', properties: { tz: { type: 'string' } } } } },
382
+ ];
383
+ // none → no tool_calls, plain text + finish_reason stop
384
+ const none = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ tools, tool_choice: 'none' }) });
385
+ const nc = (none.body as Body).choices[0];
386
+ if (!(nc.finish_reason === 'stop' && typeof nc.message.content === 'string' && !nc.message.tool_calls)) return false;
387
+ // named → exactly that tool
388
+ const named = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ tools, tool_choice: { type: 'function', function: { name: 'get_time' } } }) });
389
+ const calls = (named.body as Body).choices[0].message.tool_calls as Body[];
390
+ if (!(calls.length === 1 && calls[0]!.function.name === 'get_time')) return false;
391
+ // default (auto) + parallel_tool_calls true → one call per provided tool
392
+ const parallel = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ tools }) });
393
+ const pcalls = (parallel.body as Body).choices[0].message.tool_calls as Body[];
394
+ if (pcalls.length !== 2) return false;
395
+ // parallel_tool_calls false → collapse to a single call
396
+ const single = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ tools, parallel_tool_calls: false }) });
397
+ return ((single.body as Body).choices[0].message.tool_calls as Body[]).length === 1;
398
+ }),
399
+ ),
400
+ done('openai.tools.strict', 'tools', 'Strict function schemas → arguments validate against the declared schema', 'api', 'common', () =>
401
+ withRoot(async (h) => {
402
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ tool_choice: { type: 'function', function: { name: 'set_user' } }, tools: [{ type: 'function', function: { name: 'set_user', strict: true, parameters: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' }, active: { type: 'boolean' }, role: { type: 'string', enum: ['admin', 'user'] } }, required: ['name', 'age', 'active', 'role'] } } }] }) });
403
+ const call = (r.body as Body).choices[0]?.message?.tool_calls?.[0];
404
+ if (!call) return false;
405
+ let args: any; try { args = JSON.parse(call.function.arguments); } catch (err) { if (isInfrastructureError(err)) throw harnessError('openai.tools.strict', err); return false; }
406
+ // every declared property is present with a schema-typed value (so a strict parse succeeds)
407
+ return typeof args.name === 'string' && args.age === 0 && args.active === false && args.role === 'admin';
408
+ }),
409
+ ),
410
+
411
+ // ── Responses API ─────────────────────────────────────────────────────────────────────
412
+ done('openai.responses.create', 'responses', 'Responses: create → faithful envelope (id/object/output/usage)', 'api', 'core', () =>
413
+ withRoot(async (h) => {
414
+ const r = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'hello responses' } });
415
+ if (!ok(r)) return false;
416
+ const b = r.body as Body;
417
+ if (b.object !== 'response' || b.status !== 'completed' || !String(b.id).startsWith('resp-')) return false;
418
+ const item = b.output?.[0];
419
+ if (item?.type !== 'message' || item.content?.[0]?.type !== 'output_text') return false;
420
+ return typeof b.usage.input_tokens === 'number' && typeof b.usage.output_tokens === 'number';
421
+ }),
422
+ ),
423
+ done('openai.responses.stub_labeled', 'responses', 'Responses stub output is clearly labeled (not real output)', 'api', 'core', () =>
424
+ withRoot(async (h) => {
425
+ const r = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'echo me responses' } });
426
+ const text = (r.body as Body).output_text as string;
427
+ return ok(r) && text.includes('[twin-stub') && text.includes('echo me responses');
428
+ }),
429
+ ),
430
+ done('openai.responses.input_items', 'responses', 'Responses accepts an array of input items (role/content)', 'api', 'common', () =>
431
+ withRoot(async (h) => {
432
+ const r = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: [{ role: 'user', content: [{ type: 'input_text', text: 'structured input' }] }] } });
433
+ return ok(r) && (r.body as Body).output?.[0]?.type === 'message';
434
+ }),
435
+ ),
436
+ done('openai.responses.validation', 'responses', 'Responses validation (model + input required)', 'api', 'common', () =>
437
+ withRoot(async (h) => {
438
+ const noModel = await h({ m: 'POST', p: '/v1/responses', b: { input: 'x' } });
439
+ const noInput = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o' } });
440
+ return noModel.status === 400 && noInput.status === 400;
441
+ }),
442
+ ),
443
+ done('openai.responses.streaming', 'responses', 'Responses streaming events (created → output_text.delta → completed → [DONE])', 'api', 'common', () =>
444
+ withStream({ model: 'gpt-4o', input: 'stream responses', stream: true }, (events) => {
445
+ const types = events.filter((e) => !e.done).map((e) => e.data!.type);
446
+ const doneLast = events[events.length - 1]?.done === true;
447
+ return types.includes('response.created') && types.includes('response.output_text.delta') && types.includes('response.completed') && doneLast;
448
+ }),
449
+ ),
450
+ done('openai.responses.retrieve', 'responses', 'Responses: stored by default, retrieve + delete by id (+ 404)', 'api', 'common', () =>
451
+ withRoot(async (h) => {
452
+ const c = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'store me' } });
453
+ if (!ok(c)) return false;
454
+ const g = await h({ m: 'GET', p: `/v1/responses/${id(c)}` });
455
+ if (!ok(g) || id(g) !== id(c) || (g.body as Body).object !== 'response' || (g.body as Body).output_text !== (c.body as Body).output_text) return false;
456
+ const del = await h({ m: 'DELETE', p: `/v1/responses/${id(c)}` });
457
+ if (!ok(del) || (del.body as Body).deleted !== true) return false;
458
+ const after = await h({ m: 'GET', p: `/v1/responses/${id(c)}` });
459
+ const missing = await h({ m: 'GET', p: '/v1/responses/resp-nope' });
460
+ // store:false must NOT be retrievable
461
+ const ns = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'ephemeral', store: false } });
462
+ const nsGet = await h({ m: 'GET', p: `/v1/responses/${id(ns)}` });
463
+ return after.status === 404 && missing.status === 404 && nsGet.status === 404;
464
+ }),
465
+ ),
466
+ outOfScope('openai.responses.tools', 'responses', 'Responses built-in tools (web_search/file_search/computer_use)', 'api', 'niche',
467
+ 'Out of scope: the hosted built-in tools (web_search, computer_use, code_interpreter) require live model+infrastructure side effects the twin cannot reproduce offline — faking a web_search result or a computer_use action would fabricate output and violate the honest-stub contract. (User-defined function tools ARE modeled via openai.tools.*.)'),
468
+ done('openai.responses.reasoning', 'responses', 'Responses reasoning items + reasoning.effort (faithful reasoning item shape; labeled stub summary)', 'api', 'niche', () =>
469
+ withRoot(async (h) => {
470
+ // no reasoning param → no reasoning item, no reasoning_tokens
471
+ const plain = await h({ m: 'POST', p: '/v1/responses', b: { model: 'o4-mini', input: 'think about this' } });
472
+ if (!ok(plain)) return false;
473
+ if ((plain.body as Body).output.some((o: Body) => o.type === 'reasoning')) return false;
474
+ if ((plain.body as Body).usage.output_tokens_details !== undefined) return false;
475
+ // reasoning.effort:'high' → a reasoning item (labeled stub) BEFORE the message item + reasoning_tokens
476
+ const r = await h({ m: 'POST', p: '/v1/responses', b: { model: 'o4-mini', input: 'think about this', reasoning: { effort: 'high' } } });
477
+ if (!ok(r)) return false;
478
+ const b = r.body as Body;
479
+ const items = b.output as Body[];
480
+ const reasoning = items.find((o) => o.type === 'reasoning');
481
+ const message = items.find((o) => o.type === 'message');
482
+ if (!reasoning || !message) return false;
483
+ // reasoning item is first; faithful shape (id/summary[]) with a labeled-stub summary
484
+ if (items[0]!.type !== 'reasoning' || !String(reasoning.id).startsWith('rs-twin-')) return false;
485
+ const summary = reasoning.summary?.[0];
486
+ if (summary?.type !== 'summary_text' || !String(summary.text).includes('[twin-stub') || !String(summary.text).includes('effort=high')) return false;
487
+ // usage.output_tokens_details.reasoning_tokens > 0 and folded into output_tokens
488
+ const rt = b.usage.output_tokens_details?.reasoning_tokens;
489
+ if (typeof rt !== 'number' || rt <= 0) return false;
490
+ if (b.reasoning?.effort !== 'high') return false;
491
+ // higher effort → more reasoning tokens (deterministic budget)
492
+ const low = await h({ m: 'POST', p: '/v1/responses', b: { model: 'o4-mini', input: 'think about this', reasoning: { effort: 'low' } } });
493
+ if ((low.body as Body).usage.output_tokens_details.reasoning_tokens >= rt) return false;
494
+ // invalid effort → 400
495
+ const bad = await h({ m: 'POST', p: '/v1/responses', b: { model: 'o4-mini', input: 'x', reasoning: { effort: 'turbo' } } });
496
+ return bad.status === 400 && (bad.body as Body).error?.param === 'reasoning.effort';
497
+ }),
498
+ ),
499
+ done('openai.responses.previous_response', 'responses', 'previous_response_id chaining (server-side state; 404 unknown)', 'api', 'common', () =>
500
+ withRoot(async (h) => {
501
+ const first = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'first turn' } });
502
+ if (!ok(first)) return false;
503
+ const second = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'second turn', previous_response_id: id(first) } });
504
+ if (!ok(second) || (second.body as Body).previous_response_id !== id(first)) return false;
505
+ // chaining folds the prior turn into the prompt → larger input_tokens than the same turn unchained
506
+ const unchained = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'second turn' } });
507
+ if (!((second.body as Body).usage.input_tokens > (unchained.body as Body).usage.input_tokens)) return false;
508
+ const bad = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'x', previous_response_id: 'resp-nope' } });
509
+ return bad.status === 404;
510
+ }),
511
+ ),
512
+ done('openai.responses.input_items_list', 'responses', 'List the input items of a stored response', 'api', 'niche', () =>
513
+ withRoot(async (h) => {
514
+ const c = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'an input turn' } });
515
+ const items = await h({ m: 'GET', p: `/v1/responses/${id(c)}/input_items` });
516
+ const data = (items.body as Body).data as Body[];
517
+ const missing = await h({ m: 'GET', p: '/v1/responses/resp-nope/input_items' });
518
+ return ok(items) && (items.body as Body).object === 'list' && data.length >= 1 && data[0]!.type === 'message' && missing.status === 404;
519
+ }),
520
+ ),
521
+
522
+ // ── Embeddings ────────────────────────────────────────────────────────────────────────
523
+ done('openai.embeddings.create', 'embeddings', 'Embeddings: faithful list envelope + per-input embedding objects', 'api', 'core', () =>
524
+ withRoot(async (h) => {
525
+ const r = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'embed me' } });
526
+ if (!ok(r)) return false;
527
+ const b = r.body as Body;
528
+ const e = b.data?.[0];
529
+ return b.object === 'list' && e?.object === 'embedding' && e.index === 0 && Array.isArray(e.embedding) && e.embedding.length === 1536 && typeof b.usage.prompt_tokens === 'number';
530
+ }),
531
+ ),
532
+ done('openai.embeddings.deterministic', 'embeddings', 'Embeddings: same input → identical pseudo-vector', 'api', 'core', () =>
533
+ withRoot(async (h) => {
534
+ const a = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'same' } });
535
+ const b = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'same' } });
536
+ const va = (a.body as Body).data[0].embedding, vb = (b.body as Body).data[0].embedding;
537
+ const vc = (await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'different' } })).body as Body;
538
+ return JSON.stringify(va) === JSON.stringify(vb) && JSON.stringify(va) !== JSON.stringify(vc.data[0].embedding);
539
+ }),
540
+ ),
541
+ done('openai.embeddings.batch', 'embeddings', 'Embeddings: array input → one indexed embedding per item', 'api', 'common', () =>
542
+ withRoot(async (h) => {
543
+ const r = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: ['one', 'two', 'three'] } });
544
+ const data = (r.body as Body).data as Body[];
545
+ return ok(r) && data.length === 3 && data[2]!.index === 2;
546
+ }),
547
+ ),
548
+ done('openai.embeddings.dimensions', 'embeddings', 'Embeddings: dimensions param controls vector length; large default 3072', 'api', 'common', () =>
549
+ withRoot(async (h) => {
550
+ const custom = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'x', dimensions: 256 } });
551
+ const large = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-large', input: 'x' } });
552
+ return (custom.body as Body).data[0].embedding.length === 256 && (large.body as Body).data[0].embedding.length === 3072;
553
+ }),
554
+ ),
555
+ done('openai.embeddings.normalized', 'embeddings', 'Embeddings: pseudo-vectors are L2-normalized (unit length)', 'api', 'niche', () =>
556
+ withRoot(async (h) => {
557
+ const r = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'normalize', dimensions: 64 } });
558
+ const v = (r.body as Body).data[0].embedding as number[];
559
+ const norm = Math.sqrt(v.reduce((s, x) => s + x * x, 0));
560
+ return Math.abs(norm - 1) < 1e-6;
561
+ }),
562
+ ),
563
+ done('openai.embeddings.validation', 'embeddings', 'Embeddings: model + input required (400 otherwise)', 'api', 'common', () =>
564
+ withRoot(async (h) => {
565
+ const noModel = await h({ m: 'POST', p: '/v1/embeddings', b: { input: 'x' } });
566
+ const noInput = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small' } });
567
+ return noModel.status === 400 && noInput.status === 400;
568
+ }),
569
+ ),
570
+ done('openai.embeddings.base64', 'embeddings', 'encoding_format base64 → base64 Float32 string decoding to the float vector', 'api', 'niche', () =>
571
+ withRoot(async (h) => {
572
+ const flt = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'b64', dimensions: 8 } });
573
+ const b64 = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'b64', dimensions: 8, encoding_format: 'base64' } });
574
+ if (!ok(flt) || !ok(b64)) return false;
575
+ const floats = (flt.body as Body).data[0].embedding as number[];
576
+ const encoded = (b64.body as Body).data[0].embedding;
577
+ if (typeof encoded !== 'string') return false; // base64 returns a STRING, not an array
578
+ // decode the base64 little-endian Float32 buffer and compare to the float vector
579
+ const bin = atob(encoded);
580
+ const buf = new ArrayBuffer(bin.length);
581
+ const bytes = new Uint8Array(buf);
582
+ for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
583
+ const view = new DataView(buf);
584
+ if (bytes.length !== floats.length * 4) return false;
585
+ for (let i = 0; i < floats.length; i++) {
586
+ if (Math.abs(view.getFloat32(i * 4, true) - floats[i]!) > 1e-5) return false;
587
+ }
588
+ return true;
589
+ }),
590
+ ),
591
+
592
+ // ── Models ────────────────────────────────────────────────────────────────────────────
593
+ done('openai.models.list', 'models', 'Models: list (object:list, data array of model objects)', 'api', 'core', () =>
594
+ withRoot(async (h) => {
595
+ const r = await h({ m: 'GET', p: '/v1/models' });
596
+ const b = r.body as Body;
597
+ return ok(r) && b.object === 'list' && Array.isArray(b.data) && b.data.length > 0 && b.data[0].object === 'model';
598
+ }),
599
+ ),
600
+ done('openai.models.retrieve', 'models', 'Models: retrieve by id (+ 404 model_not_found)', 'api', 'core', () =>
601
+ withRoot(async (h) => {
602
+ const r = await h({ m: 'GET', p: '/v1/models/gpt-4o' });
603
+ if (!ok(r) || (r.body as Body).object !== 'model' || (r.body as Body).id !== 'gpt-4o') return false;
604
+ const missing = await h({ m: 'GET', p: '/v1/models/nope-9' });
605
+ return missing.status === 404 && (missing.body as Body).error?.code === 'model_not_found';
606
+ }),
607
+ ),
608
+ done('openai.models.delete', 'models', 'Delete a fine-tuned model (job output); base models refuse; 404 unknown', 'api', 'niche', () =>
609
+ withRoot(async (h) => {
610
+ const job = await h({ m: 'POST', p: '/v1/fine_tuning/jobs', b: { model: 'gpt-4o-mini', training_file: 'file-train' } });
611
+ const ftModel = (job.body as Body).fine_tuned_model as string;
612
+ // the minted fine-tuned model is retrievable + listed
613
+ const ret = await h({ m: 'GET', p: `/v1/models/${encodeURIComponent(ftModel)}` });
614
+ if (!ok(ret) || (ret.body as Body).id !== ftModel) return false;
615
+ const listed = ((await h({ m: 'GET', p: '/v1/models' })).body as Body).data as Body[];
616
+ if (!listed.some((m) => m.id === ftModel)) return false;
617
+ // delete it → deleted:true, then gone from retrieve + list
618
+ const del = await h({ m: 'DELETE', p: `/v1/models/${encodeURIComponent(ftModel)}` });
619
+ if (!ok(del) || (del.body as Body).deleted !== true) return false;
620
+ const after = await h({ m: 'GET', p: `/v1/models/${encodeURIComponent(ftModel)}` });
621
+ if (after.status !== 404) return false;
622
+ // base catalog model cannot be deleted; unknown → 404
623
+ const base = await h({ m: 'DELETE', p: '/v1/models/gpt-4o' });
624
+ const missing = await h({ m: 'DELETE', p: '/v1/models/ft:nope' });
625
+ return base.status === 400 && missing.status === 404;
626
+ }),
627
+ ),
628
+
629
+ // ── Moderations ───────────────────────────────────────────────────────────────────────
630
+ done('openai.moderations.create', 'moderations', 'Moderations: faithful results shape (categories/scores/flagged)', 'api', 'core', () =>
631
+ withRoot(async (h) => {
632
+ const r = await h({ m: 'POST', p: '/v1/moderations', b: { input: 'a harmless sentence' } });
633
+ if (!ok(r)) return false;
634
+ const res = (r.body as Body).results?.[0];
635
+ return String((r.body as Body).id).startsWith('modr-') && res && res.flagged === false && typeof res.categories === 'object' && typeof res.category_scores === 'object';
636
+ }),
637
+ ),
638
+ done('openai.moderations.flagging', 'moderations', 'Moderations: deterministic flagging on the keyword heuristic', 'api', 'common', () =>
639
+ withRoot(async (h) => {
640
+ const r = await h({ m: 'POST', p: '/v1/moderations', b: { input: 'I will kill the process' } });
641
+ const res = (r.body as Body).results?.[0];
642
+ return ok(r) && res.flagged === true && res.categories.violence === true;
643
+ }),
644
+ ),
645
+ done('openai.moderations.batch', 'moderations', 'Moderations: array input → one result per item', 'api', 'common', () =>
646
+ withRoot(async (h) => {
647
+ const r = await h({ m: 'POST', p: '/v1/moderations', b: { input: ['safe', 'hateful slur'] } });
648
+ const results = (r.body as Body).results as Body[];
649
+ return ok(r) && results.length === 2 && results[0]!.flagged === false && results[1]!.flagged === true;
650
+ }),
651
+ ),
652
+
653
+ // ── Files (stateful) ──────────────────────────────────────────────────────────────────
654
+ done('openai.files.upload', 'files', 'Files: upload (stateful) with purpose/filename/bytes/status', 'api', 'core', () =>
655
+ withRoot(async (h) => {
656
+ const r = await h({ m: 'POST', p: '/v1/files', b: { purpose: 'fine-tune', filename: 'train.jsonl', content: '{"x":1}' } });
657
+ if (!ok(r) || (r.body as Body).object !== 'file' || !String(id(r)).startsWith('file-')) return false;
658
+ const b = r.body as Body;
659
+ return b.purpose === 'fine-tune' && b.filename === 'train.jsonl' && b.status === 'processed' && b.bytes > 0;
660
+ }),
661
+ ),
662
+ done('openai.files.retrieve', 'files', 'Files: retrieve by id (+ 404 unknown)', 'api', 'core', () =>
663
+ withRoot(async (h) => {
664
+ const c = await h({ m: 'POST', p: '/v1/files', b: { purpose: 'batch', filename: 'a.jsonl', content: '{}' } });
665
+ const g = await h({ m: 'GET', p: `/v1/files/${id(c)}` });
666
+ const missing = await h({ m: 'GET', p: '/v1/files/file-nope' });
667
+ return ok(g) && id(g) === id(c) && missing.status === 404;
668
+ }),
669
+ ),
670
+ done('openai.files.list', 'files', 'Files: list (object:list, newest first)', 'api', 'core', () =>
671
+ withRoot(async (h) => {
672
+ await h({ m: 'POST', p: '/v1/files', b: { purpose: 'batch', filename: 'a.jsonl', content: '{}' } });
673
+ await h({ m: 'POST', p: '/v1/files', b: { purpose: 'batch', filename: 'b.jsonl', content: '{}' } });
674
+ const l = await h({ m: 'GET', p: '/v1/files' });
675
+ return ok(l) && (l.body as Body).object === 'list' && (l.body as Body).data.length === 2;
676
+ }),
677
+ ),
678
+ done('openai.files.content', 'files', 'Files: download content of an uploaded file', 'api', 'common', () =>
679
+ withRoot(async (h) => {
680
+ const c = await h({ m: 'POST', p: '/v1/files', b: { purpose: 'batch', filename: 'a.jsonl', content: 'hello-bytes' } });
681
+ const content = await h({ m: 'GET', p: `/v1/files/${id(c)}/content` });
682
+ return ok(content) && content.body === 'hello-bytes';
683
+ }),
684
+ ),
685
+ done('openai.files.delete', 'files', 'Files: delete returns the deleted stub + removes from list', 'api', 'common', () =>
686
+ withRoot(async (h) => {
687
+ const c = await h({ m: 'POST', p: '/v1/files', b: { purpose: 'batch', filename: 'a.jsonl', content: '{}' } });
688
+ const del = await h({ m: 'DELETE', p: `/v1/files/${id(c)}` });
689
+ if (!ok(del) || (del.body as Body).deleted !== true) return false;
690
+ const l = await h({ m: 'GET', p: '/v1/files' });
691
+ return (l.body as Body).data.length === 0;
692
+ }),
693
+ ),
694
+ done('openai.files.purpose_required', 'files', 'Files: purpose required (400 invalid_request_error)', 'api', 'common', () =>
695
+ withRoot(async (h) => {
696
+ const r = await h({ m: 'POST', p: '/v1/files', b: { filename: 'a.jsonl', content: '{}' } });
697
+ return r.status === 400 && (r.body as Body).error?.param === 'purpose';
698
+ }),
699
+ ),
700
+
701
+ // ── Batches (stateful) ────────────────────────────────────────────────────────────────
702
+ done('openai.batches.create', 'batches', 'Batches: create (stateful) with endpoint/input_file_id/status', 'api', 'core', () =>
703
+ withRoot(async (h) => {
704
+ const r = await h({ m: 'POST', p: '/v1/batches', b: { input_file_id: 'file-x', endpoint: '/v1/chat/completions', completion_window: '24h' } });
705
+ if (!ok(r) || (r.body as Body).object !== 'batch' || !String(id(r)).startsWith('batch-')) return false;
706
+ const b = r.body as Body;
707
+ return b.endpoint === '/v1/chat/completions' && b.status === 'completed' && typeof b.output_file_id === 'string';
708
+ }),
709
+ ),
710
+ done('openai.batches.retrieve', 'batches', 'Batches: retrieve by id (+ 404 unknown)', 'api', 'core', () =>
711
+ withRoot(async (h) => {
712
+ const c = await h({ m: 'POST', p: '/v1/batches', b: { input_file_id: 'file-x', endpoint: '/v1/chat/completions', completion_window: '24h' } });
713
+ const g = await h({ m: 'GET', p: `/v1/batches/${id(c)}` });
714
+ const missing = await h({ m: 'GET', p: '/v1/batches/batch-nope' });
715
+ return ok(g) && id(g) === id(c) && missing.status === 404;
716
+ }),
717
+ ),
718
+ done('openai.batches.list', 'batches', 'Batches: list (object:list)', 'api', 'core', () =>
719
+ withRoot(async (h) => {
720
+ await h({ m: 'POST', p: '/v1/batches', b: { input_file_id: 'file-x', endpoint: '/v1/chat/completions', completion_window: '24h' } });
721
+ const l = await h({ m: 'GET', p: '/v1/batches' });
722
+ return ok(l) && (l.body as Body).object === 'list' && (l.body as Body).data.length === 1;
723
+ }),
724
+ ),
725
+ done('openai.batches.cancel', 'batches', 'Batches: cancel sets status cancelled (404 unknown)', 'api', 'common', () =>
726
+ withRoot(async (h) => {
727
+ const c = await h({ m: 'POST', p: '/v1/batches', b: { input_file_id: 'file-x', endpoint: '/v1/chat/completions', completion_window: '24h' } });
728
+ const cancel = await h({ m: 'POST', p: `/v1/batches/${id(c)}/cancel` });
729
+ const missing = await h({ m: 'POST', p: '/v1/batches/batch-nope/cancel' });
730
+ return ok(cancel) && field(cancel, 'status') === 'cancelled' && missing.status === 404;
731
+ }),
732
+ ),
733
+ done('openai.batches.validation', 'batches', 'Batches: input_file_id/endpoint/completion_window required', 'api', 'common', () =>
734
+ withRoot(async (h) => {
735
+ const r = await h({ m: 'POST', p: '/v1/batches', b: { endpoint: '/v1/chat/completions', completion_window: '24h' } });
736
+ return r.status === 400 && (r.body as Body).error?.param === 'input_file_id';
737
+ }),
738
+ ),
739
+ done('openai.batches.persistence', 'batches', 'Batches persist across requests (kernel-backed, not a side store)', 'api', 'common', () =>
740
+ withRoot(async (h) => {
741
+ const c = await h({ m: 'POST', p: '/v1/batches', b: { input_file_id: 'file-x', endpoint: '/v1/chat/completions', completion_window: '24h' } });
742
+ if (!ok(c)) return false;
743
+ const cid = id(c);
744
+ // a real, well-formed batch id + object/fields must exist — not merely "both sides undefined".
745
+ if (typeof cid !== 'string' || !cid.startsWith('batch-')) return false;
746
+ const cb = c.body as Body;
747
+ if (cb.object !== 'batch' || cb.input_file_id !== 'file-x' || cb.status !== 'completed') return false;
748
+ const g1 = await h({ m: 'GET', p: `/v1/batches/${cid}` });
749
+ const g2 = await h({ m: 'GET', p: `/v1/batches/${cid}` });
750
+ // the GET must actually round-trip the persisted resource's real fields (kernel-backed) —
751
+ // not merely a matching-because-undefined id.
752
+ const same = (r: OpenAIResponseEnvelope) => {
753
+ const rb = r.body as Body;
754
+ return rb.object === 'batch' && rb.input_file_id === cb.input_file_id && rb.endpoint === cb.endpoint &&
755
+ rb.status === cb.status && rb.output_file_id === cb.output_file_id;
756
+ };
757
+ return ok(g1) && ok(g2) && id(g1) === cid && id(g2) === cid && same(g1) && same(g2);
758
+ }),
759
+ ),
760
+
761
+ // ── Fine-tuning (stateful) ────────────────────────────────────────────────────────────
762
+ done('openai.fine_tuning.create', 'fine_tuning', 'Fine-tuning: create job (stateful) with model/training_file/status', 'api', 'core', () =>
763
+ withRoot(async (h) => {
764
+ const r = await h({ m: 'POST', p: '/v1/fine_tuning/jobs', b: { model: 'gpt-4o-mini', training_file: 'file-train' } });
765
+ if (!ok(r) || (r.body as Body).object !== 'fine_tuning.job' || !String(id(r)).startsWith('ftjob-')) return false;
766
+ const b = r.body as Body;
767
+ return b.model === 'gpt-4o-mini' && b.status === 'succeeded' && String(b.fine_tuned_model).startsWith('ft:');
768
+ }),
769
+ ),
770
+ done('openai.fine_tuning.retrieve', 'fine_tuning', 'Fine-tuning: retrieve + list jobs', 'api', 'common', () =>
771
+ withRoot(async (h) => {
772
+ const c = await h({ m: 'POST', p: '/v1/fine_tuning/jobs', b: { model: 'gpt-4o-mini', training_file: 'file-train' } });
773
+ const g = await h({ m: 'GET', p: `/v1/fine_tuning/jobs/${id(c)}` });
774
+ const l = await h({ m: 'GET', p: '/v1/fine_tuning/jobs' });
775
+ return ok(g) && id(g) === id(c) && (l.body as Body).data.length === 1;
776
+ }),
777
+ ),
778
+ done('openai.fine_tuning.events', 'fine_tuning', 'Fine-tuning: job events list', 'api', 'common', () =>
779
+ withRoot(async (h) => {
780
+ const c = await h({ m: 'POST', p: '/v1/fine_tuning/jobs', b: { model: 'gpt-4o-mini', training_file: 'file-train' } });
781
+ const ev = await h({ m: 'GET', p: `/v1/fine_tuning/jobs/${id(c)}/events` });
782
+ const data = (ev.body as Body).data as Body[];
783
+ return ok(ev) && data.length > 0 && data[0]!.object === 'fine_tuning.job.event';
784
+ }),
785
+ ),
786
+ done('openai.fine_tuning.cancel', 'fine_tuning', 'Fine-tuning: cancel job (status cancelled)', 'api', 'common', () =>
787
+ withRoot(async (h) => {
788
+ const c = await h({ m: 'POST', p: '/v1/fine_tuning/jobs', b: { model: 'gpt-4o-mini', training_file: 'file-train' } });
789
+ const cancel = await h({ m: 'POST', p: `/v1/fine_tuning/jobs/${id(c)}/cancel` });
790
+ return ok(cancel) && field(cancel, 'status') === 'cancelled';
791
+ }),
792
+ ),
793
+ done('openai.fine_tuning.validation', 'fine_tuning', 'Fine-tuning: model + training_file required', 'api', 'common', () =>
794
+ withRoot(async (h) => {
795
+ const r = await h({ m: 'POST', p: '/v1/fine_tuning/jobs', b: { model: 'gpt-4o-mini' } });
796
+ return r.status === 400 && (r.body as Body).error?.param === 'training_file';
797
+ }),
798
+ ),
799
+ done('openai.fine_tuning.checkpoints', 'fine_tuning', 'Fine-tuning checkpoints list (succeeded job → checkpoint; 404 unknown)', 'api', 'niche', () =>
800
+ withRoot(async (h) => {
801
+ const job = await h({ m: 'POST', p: '/v1/fine_tuning/jobs', b: { model: 'gpt-4o-mini', training_file: 'file-train' } });
802
+ const ck = await h({ m: 'GET', p: `/v1/fine_tuning/jobs/${id(job)}/checkpoints` });
803
+ if (!ok(ck) || (ck.body as Body).object !== 'list') return false;
804
+ const data = (ck.body as Body).data as Body[];
805
+ if (data.length !== 1 || data[0]!.object !== 'fine_tuning.job.checkpoint' || data[0]!.fine_tuning_job_id !== id(job) || typeof data[0]!.step_number !== 'number') return false;
806
+ const missing = await h({ m: 'GET', p: '/v1/fine_tuning/jobs/ftjob-nope/checkpoints' });
807
+ return missing.status === 404;
808
+ }),
809
+ ),
810
+
811
+ // ── Vector stores (stateful) ──────────────────────────────────────────────────────────
812
+ done('openai.vector_stores.create', 'vector_stores', 'Vector stores: create (stateful) with name/status/file_counts', 'api', 'core', () =>
813
+ withRoot(async (h) => {
814
+ const r = await h({ m: 'POST', p: '/v1/vector_stores', b: { name: 'docs' } });
815
+ if (!ok(r) || (r.body as Body).object !== 'vector_store' || !String(id(r)).startsWith('vs-')) return false;
816
+ const b = r.body as Body;
817
+ return b.name === 'docs' && b.status === 'completed' && typeof b.file_counts === 'object';
818
+ }),
819
+ ),
820
+ done('openai.vector_stores.crud', 'vector_stores', 'Vector stores: retrieve + list + delete', 'api', 'common', () =>
821
+ withRoot(async (h) => {
822
+ const c = await h({ m: 'POST', p: '/v1/vector_stores', b: { name: 'docs' } });
823
+ const g = await h({ m: 'GET', p: `/v1/vector_stores/${id(c)}` });
824
+ const l = await h({ m: 'GET', p: '/v1/vector_stores' });
825
+ const del = await h({ m: 'DELETE', p: `/v1/vector_stores/${id(c)}` });
826
+ const after = await h({ m: 'GET', p: '/v1/vector_stores' });
827
+ return ok(g) && (l.body as Body).data.length === 1 && (del.body as Body).deleted === true && (after.body as Body).data.length === 0;
828
+ }),
829
+ ),
830
+ done('openai.vector_stores.files', 'vector_stores', 'Vector stores: attach + list files', 'api', 'common', () =>
831
+ withRoot(async (h) => {
832
+ const c = await h({ m: 'POST', p: '/v1/vector_stores', b: { name: 'docs' } });
833
+ const add = await h({ m: 'POST', p: `/v1/vector_stores/${id(c)}/files`, b: { file_id: 'file-abc' } });
834
+ if (!ok(add) || (add.body as Body).file_id !== 'file-abc' || (add.body as Body).object !== 'vector_store.file') return false;
835
+ const l = await h({ m: 'GET', p: `/v1/vector_stores/${id(c)}/files` });
836
+ return ok(l) && (l.body as Body).data.length === 1 && (l.body as Body).data[0].file_id === 'file-abc';
837
+ }),
838
+ ),
839
+ done('openai.vector_stores.seed_files', 'vector_stores', 'Vector stores: create with file_ids seeds attached files', 'api', 'common', () =>
840
+ withRoot(async (h) => {
841
+ const c = await h({ m: 'POST', p: '/v1/vector_stores', b: { name: 'docs', file_ids: ['file-a', 'file-b'] } });
842
+ const l = await h({ m: 'GET', p: `/v1/vector_stores/${id(c)}/files` });
843
+ return ok(c) && (l.body as Body).data.length === 2;
844
+ }),
845
+ ),
846
+ done('openai.vector_stores.search', 'vector_stores', 'Vector store search: deterministic ranked results page (+ query required, 404 store)', 'api', 'common', () =>
847
+ withRoot(async (h) => {
848
+ const c = await h({ m: 'POST', p: '/v1/vector_stores', b: { name: 'docs', file_ids: ['file-a', 'file-b', 'file-c'] } });
849
+ const s = await h({ m: 'POST', p: `/v1/vector_stores/${id(c)}/search`, b: { query: 'hello' } });
850
+ if (!ok(s) || (s.body as Body).object !== 'vector_store.search_results.page') return false;
851
+ const data = (s.body as Body).data as Body[];
852
+ if (data.length !== 3 || typeof data[0]!.score !== 'number') return false;
853
+ // ranked descending + deterministic across runs
854
+ if (!(data[0]!.score >= data[1]!.score && data[1]!.score >= data[2]!.score)) return false;
855
+ const s2 = await h({ m: 'POST', p: `/v1/vector_stores/${id(c)}/search`, b: { query: 'hello' } });
856
+ if (JSON.stringify((s2.body as Body).data) !== JSON.stringify(data)) return false;
857
+ const maxR = await h({ m: 'POST', p: `/v1/vector_stores/${id(c)}/search`, b: { query: 'hello', max_num_results: 1 } });
858
+ if (((maxR.body as Body).data as Body[]).length !== 1) return false;
859
+ const noQuery = await h({ m: 'POST', p: `/v1/vector_stores/${id(c)}/search`, b: {} });
860
+ const badStore = await h({ m: 'POST', p: '/v1/vector_stores/vs-nope/search', b: { query: 'x' } });
861
+ return noQuery.status === 400 && badStore.status === 404;
862
+ }),
863
+ ),
864
+ done('openai.vector_stores.file_batches', 'vector_stores', 'Vector store file batches: bulk attach → batch object + retrieve + list batch files', 'api', 'niche', () =>
865
+ withRoot(async (h) => {
866
+ const vs = await h({ m: 'POST', p: '/v1/vector_stores', b: { name: 'docs' } });
867
+ const batch = await h({ m: 'POST', p: `/v1/vector_stores/${id(vs)}/file_batches`, b: { file_ids: ['file-a', 'file-b', 'file-c'] } });
868
+ if (!ok(batch)) return false;
869
+ const bb = batch.body as Body;
870
+ if (bb.object !== 'vector_store.file_batch' || bb.status !== 'completed' || bb.file_counts?.total !== 3) return false;
871
+ // retrieve the batch
872
+ const get = await h({ m: 'GET', p: `/v1/vector_stores/${id(vs)}/file_batches/${bb.id}` });
873
+ if (!ok(get) || (get.body as Body).id !== bb.id) return false;
874
+ // list this batch's files
875
+ const files = await h({ m: 'GET', p: `/v1/vector_stores/${id(vs)}/file_batches/${bb.id}/files` });
876
+ if (!ok(files) || ((files.body as Body).data as Body[]).length !== 3) return false;
877
+ // the files are also attached to the store as a whole
878
+ const all = await h({ m: 'GET', p: `/v1/vector_stores/${id(vs)}/files` });
879
+ if (((all.body as Body).data as Body[]).length !== 3) return false;
880
+ // validation + 404 store + 404 batch
881
+ const noIds = await h({ m: 'POST', p: `/v1/vector_stores/${id(vs)}/file_batches`, b: {} });
882
+ const badStore = await h({ m: 'POST', p: '/v1/vector_stores/vs-nope/file_batches', b: { file_ids: ['x'] } });
883
+ const badBatch = await h({ m: 'GET', p: `/v1/vector_stores/${id(vs)}/file_batches/vsfb-nope` });
884
+ return noIds.status === 400 && badStore.status === 404 && badBatch.status === 404;
885
+ }),
886
+ ),
887
+
888
+ // ── Images ────────────────────────────────────────────────────────────────────────────
889
+ done('openai.images.generations_shape', 'images', 'Images: generations return the faithful response shape (placeholder URL)', 'api', 'common', () =>
890
+ withRoot(async (h) => {
891
+ const r = await h({ m: 'POST', p: '/v1/images/generations', b: { model: 'dall-e-3', prompt: 'a cat', n: 1 } });
892
+ if (!ok(r)) return false;
893
+ const d = (r.body as Body).data?.[0];
894
+ return typeof (r.body as Body).created === 'number' && typeof d?.url === 'string' && String(d.revised_prompt).includes('[twin-stub');
895
+ }),
896
+ ),
897
+ done('openai.images.validation', 'images', 'Images: prompt required (400)', 'api', 'common', () =>
898
+ withRoot(async (h) => {
899
+ const r = await h({ m: 'POST', p: '/v1/images/generations', b: { model: 'dall-e-3' } });
900
+ return r.status === 400 && (r.body as Body).error?.param === 'prompt';
901
+ }),
902
+ ),
903
+ done('openai.images.edits', 'images', 'Image edits (image+prompt) + variations (image) → faithful response shape', 'api', 'niche', () =>
904
+ withRoot(async (h) => {
905
+ const edit = await h({ m: 'POST', p: '/v1/images/edits', b: { image: 'base.png', prompt: 'add a hat' } });
906
+ if (!ok(edit) || typeof (edit.body as Body).created !== 'number' || typeof (edit.body as Body).data[0].url !== 'string') return false;
907
+ const varn = await h({ m: 'POST', p: '/v1/images/variations', b: { image: 'base.png', n: 2 } });
908
+ if (!ok(varn) || ((varn.body as Body).data as Body[]).length !== 2) return false;
909
+ // validation: edits require image AND prompt; variations require image
910
+ const noImg = await h({ m: 'POST', p: '/v1/images/edits', b: { prompt: 'x' } });
911
+ const noPrompt = await h({ m: 'POST', p: '/v1/images/edits', b: { image: 'base.png' } });
912
+ const varNoImg = await h({ m: 'POST', p: '/v1/images/variations', b: {} });
913
+ return noImg.status === 400 && noPrompt.status === 400 && varNoImg.status === 400;
914
+ }),
915
+ ),
916
+
917
+ // ── Audio ─────────────────────────────────────────────────────────────────────────────
918
+ done('openai.audio.transcriptions', 'audio', 'Audio transcriptions: labeled stub transcript, json + verbose_json + text shapes', 'api', 'common', () =>
919
+ withRoot(async (h) => {
920
+ const j = await h({ m: 'POST', p: '/v1/audio/transcriptions', b: { file: 'speech.mp3', model: 'whisper-1' } });
921
+ if (!ok(j) || typeof (j.body as Body).text !== 'string' || !String((j.body as Body).text).includes('[twin-stub')) return false;
922
+ const v = await h({ m: 'POST', p: '/v1/audio/transcriptions', b: { file: 'speech.mp3', model: 'whisper-1', response_format: 'verbose_json' } });
923
+ if (!ok(v) || (v.body as Body).task !== 'transcription' || !Array.isArray((v.body as Body).segments)) return false;
924
+ const t = await h({ m: 'POST', p: '/v1/audio/transcriptions', b: { file: 'speech.mp3', model: 'whisper-1', response_format: 'text' } });
925
+ if (!ok(t) || typeof t.body !== 'string') return false;
926
+ const noFile = await h({ m: 'POST', p: '/v1/audio/transcriptions', b: { model: 'whisper-1' } });
927
+ const noModel = await h({ m: 'POST', p: '/v1/audio/transcriptions', b: { file: 'speech.mp3' } });
928
+ return noFile.status === 400 && (noFile.body as Body).error?.param === 'file' && noModel.status === 400;
929
+ }),
930
+ ),
931
+ done('openai.audio.translations', 'audio', 'Audio translations: labeled stub, task translation, english language', 'api', 'niche', () =>
932
+ withRoot(async (h) => {
933
+ const v = await h({ m: 'POST', p: '/v1/audio/translations', b: { file: 'foreign.mp3', model: 'whisper-1', response_format: 'verbose_json' } });
934
+ const noFile = await h({ m: 'POST', p: '/v1/audio/translations', b: { model: 'whisper-1' } });
935
+ return ok(v) && (v.body as Body).task === 'translation' && (v.body as Body).language === 'english' && noFile.status === 400;
936
+ }),
937
+ ),
938
+ done('openai.audio.speech', 'audio', 'Text-to-speech: deterministic labeled audio payload (+ model/input/voice required)', 'api', 'common', () =>
939
+ withRoot(async (h) => {
940
+ const r = await h({ m: 'POST', p: '/v1/audio/speech', b: { model: 'tts-1', input: 'hello world', voice: 'alloy' } });
941
+ if (!ok(r) || (r.body as Body).object !== 'audio.speech' || typeof (r.body as Body).audio_base64 !== 'string') return false;
942
+ // deterministic: same input → same payload; decoded marker labels it a stub
943
+ const r2 = await h({ m: 'POST', p: '/v1/audio/speech', b: { model: 'tts-1', input: 'hello world', voice: 'alloy' } });
944
+ if ((r.body as Body).audio_base64 !== (r2.body as Body).audio_base64) return false;
945
+ if (!atob((r.body as Body).audio_base64 as string).includes('[twin-stub-audio]')) return false;
946
+ const noVoice = await h({ m: 'POST', p: '/v1/audio/speech', b: { model: 'tts-1', input: 'x' } });
947
+ const noInput = await h({ m: 'POST', p: '/v1/audio/speech', b: { model: 'tts-1', voice: 'alloy' } });
948
+ return noVoice.status === 400 && (noVoice.body as Body).error?.param === 'voice' && noInput.status === 400;
949
+ }),
950
+ ),
951
+ outOfScope('openai.audio.realtime', 'audio', 'Realtime API (websocket sessions)', 'api', 'niche',
952
+ 'Out of scope: the Realtime API is a stateful bidirectional WebSocket carrying live audio to/from a running speech model — both the transport and the real-time model inference are irreproducible offline. The twin runs no model and the verify() harness is HTTP/offline, so a faithful realtime session cannot be modeled without fabrication.'),
953
+
954
+ // ── Assistants (beta) ─────────────────────────────────────────────────────────────────
955
+ done('openai.assistants.crud', 'assistants', 'Assistants: create/retrieve/update/list/delete (+ model required, 404)', 'api', 'niche', () =>
956
+ withRoot(async (h) => {
957
+ const noModel = await h({ m: 'POST', p: '/v1/assistants', b: { name: 'a' } });
958
+ if (noModel.status !== 400) return false;
959
+ const c = await h({ m: 'POST', p: '/v1/assistants', b: { model: 'gpt-4o', name: 'Helper', instructions: 'be helpful', tools: [{ type: 'code_interpreter' }] } });
960
+ if (!ok(c) || !String(id(c)).startsWith('asst-') || (c.body as Body).object !== 'assistant' || (c.body as Body).name !== 'Helper') return false;
961
+ const g = await h({ m: 'GET', p: `/v1/assistants/${id(c)}` });
962
+ if (!ok(g) || id(g) !== id(c)) return false;
963
+ const upd = await h({ m: 'POST', p: `/v1/assistants/${id(c)}`, b: { name: 'Renamed' } });
964
+ if (!ok(upd) || (upd.body as Body).name !== 'Renamed' || (upd.body as Body).instructions !== 'be helpful') return false;
965
+ const l = await h({ m: 'GET', p: '/v1/assistants' });
966
+ if (!ok(l) || ((l.body as Body).data as Body[]).length !== 1) return false;
967
+ const del = await h({ m: 'DELETE', p: `/v1/assistants/${id(c)}` });
968
+ if (!ok(del) || (del.body as Body).deleted !== true) return false;
969
+ const after = await h({ m: 'GET', p: `/v1/assistants/${id(c)}` });
970
+ const missing = await h({ m: 'GET', p: '/v1/assistants/asst-nope' });
971
+ return after.status === 404 && missing.status === 404;
972
+ }),
973
+ ),
974
+ done('openai.threads.crud', 'assistants', 'Threads + messages: create/retrieve/update/delete + add/list/retrieve messages', 'api', 'niche', () =>
975
+ withRoot(async (h) => {
976
+ // create a thread with an inline seed message
977
+ const t = await h({ m: 'POST', p: '/v1/threads', b: { messages: [{ role: 'user', content: 'seed turn' }], metadata: { k: 'v' } } });
978
+ if (!ok(t) || !String(id(t)).startsWith('thread-') || (t.body as Body).object !== 'thread') return false;
979
+ const g = await h({ m: 'GET', p: `/v1/threads/${id(t)}` });
980
+ if (!ok(g) || id(g) !== id(t)) return false;
981
+ // the seed message is present
982
+ const seeded = await h({ m: 'GET', p: `/v1/threads/${id(t)}/messages` });
983
+ if (((seeded.body as Body).data as Body[]).length !== 1) return false;
984
+ // add a message
985
+ const m = await h({ m: 'POST', p: `/v1/threads/${id(t)}/messages`, b: { role: 'user', content: 'another turn' } });
986
+ if (!ok(m) || (m.body as Body).object !== 'thread.message' || (m.body as Body).content[0].text.value !== 'another turn') return false;
987
+ const mg = await h({ m: 'GET', p: `/v1/threads/${id(t)}/messages/${id(m)}` });
988
+ if (!ok(mg) || id(mg) !== id(m)) return false;
989
+ const list = await h({ m: 'GET', p: `/v1/threads/${id(t)}/messages` });
990
+ if (((list.body as Body).data as Body[]).length !== 2) return false;
991
+ // content required
992
+ const noContent = await h({ m: 'POST', p: `/v1/threads/${id(t)}/messages`, b: { role: 'user' } });
993
+ if (noContent.status !== 400) return false;
994
+ // update thread metadata
995
+ const upd = await h({ m: 'POST', p: `/v1/threads/${id(t)}`, b: { metadata: { k: 'v2' } } });
996
+ if ((upd.body as Body).metadata?.k !== 'v2') return false;
997
+ // delete
998
+ const del = await h({ m: 'DELETE', p: `/v1/threads/${id(t)}` });
999
+ if (!ok(del) || (del.body as Body).deleted !== true) return false;
1000
+ const after = await h({ m: 'GET', p: `/v1/threads/${id(t)}` });
1001
+ const missingMsg = await h({ m: 'POST', p: '/v1/threads/thread-nope/messages', b: { role: 'user', content: 'x' } });
1002
+ return after.status === 404 && missingMsg.status === 404;
1003
+ }),
1004
+ ),
1005
+ done('openai.runs.crud', 'assistants', 'Runs + run steps: create (stub completion) / retrieve / list / steps / cancel', 'api', 'niche', () =>
1006
+ withRoot(async (h) => {
1007
+ const a = await h({ m: 'POST', p: '/v1/assistants', b: { model: 'gpt-4o', name: 'Runner' } });
1008
+ const t = await h({ m: 'POST', p: '/v1/threads', b: { messages: [{ role: 'user', content: 'do a thing' }] } });
1009
+ // assistant_id required + 404 unknown assistant
1010
+ const noAsst = await h({ m: 'POST', p: `/v1/threads/${id(t)}/runs`, b: {} });
1011
+ const badAsst = await h({ m: 'POST', p: `/v1/threads/${id(t)}/runs`, b: { assistant_id: 'asst-nope' } });
1012
+ if (noAsst.status !== 400 || badAsst.status !== 404) return false;
1013
+ // create a run → completed, stub assistant reply appended to the thread
1014
+ const run = await h({ m: 'POST', p: `/v1/threads/${id(t)}/runs`, b: { assistant_id: id(a) } });
1015
+ if (!ok(run) || !String(id(run)).startsWith('run-') || (run.body as Body).object !== 'thread.run' || (run.body as Body).status !== 'completed') return false;
1016
+ if ((run.body as Body).assistant_id !== id(a) || (run.body as Body).thread_id !== id(t)) return false;
1017
+ // the thread now has the user turn + the stub assistant reply
1018
+ const msgs = ((await h({ m: 'GET', p: `/v1/threads/${id(t)}/messages` })).body as Body).data as Body[];
1019
+ const assistantMsg = msgs.find((mm) => mm.role === 'assistant');
1020
+ if (!assistantMsg || !String(assistantMsg.content[0].text.value).includes('[twin-stub')) return false;
1021
+ // retrieve the run + list runs
1022
+ const gr = await h({ m: 'GET', p: `/v1/threads/${id(t)}/runs/${id(run)}` });
1023
+ if (!ok(gr) || id(gr) !== id(run)) return false;
1024
+ const lr = await h({ m: 'GET', p: `/v1/threads/${id(t)}/runs` });
1025
+ if (((lr.body as Body).data as Body[]).length !== 1) return false;
1026
+ // run steps: a message_creation step pointing at the reply message
1027
+ const steps = await h({ m: 'GET', p: `/v1/threads/${id(t)}/runs/${id(run)}/steps` });
1028
+ const sd = (steps.body as Body).data as Body[];
1029
+ if (sd.length !== 1 || sd[0]!.object !== 'thread.run.step' || sd[0]!.type !== 'message_creation') return false;
1030
+ if (sd[0]!.step_details.message_creation.message_id !== assistantMsg.id) return false;
1031
+ const gs = await h({ m: 'GET', p: `/v1/threads/${id(t)}/runs/${id(run)}/steps/${sd[0]!.id}` });
1032
+ if (!ok(gs) || (gs.body as Body).id !== sd[0]!.id) return false;
1033
+ // cancel
1034
+ const cancel = await h({ m: 'POST', p: `/v1/threads/${id(t)}/runs/${id(run)}/cancel` });
1035
+ if (!ok(cancel) || (cancel.body as Body).status !== 'cancelled') return false;
1036
+ const badRun = await h({ m: 'GET', p: `/v1/threads/${id(t)}/runs/run-nope` });
1037
+ return badRun.status === 404;
1038
+ }),
1039
+ ),
1040
+
1041
+ // ── Admin / org ───────────────────────────────────────────────────────────────────────
1042
+ done('openai.uploads.multipart', 'uploads', 'Uploads API: create → add parts → complete assembles a File (+ cancel, validation)', 'api', 'niche', () =>
1043
+ withRoot(async (h) => {
1044
+ // create requires filename/purpose/bytes/mime_type
1045
+ const noFn = await h({ m: 'POST', p: '/v1/uploads', b: { purpose: 'fine-tune', bytes: 10, mime_type: 'text/plain' } });
1046
+ if (noFn.status !== 400) return false;
1047
+ const up = await h({ m: 'POST', p: '/v1/uploads', b: { filename: 'big.jsonl', purpose: 'fine-tune', bytes: 6, mime_type: 'text/plain' } });
1048
+ if (!ok(up) || (up.body as Body).object !== 'upload' || (up.body as Body).status !== 'pending' || !String(id(up)).startsWith('upload-')) return false;
1049
+ const p1 = await h({ m: 'POST', p: `/v1/uploads/${id(up)}/parts`, b: { data: 'foo' } });
1050
+ const p2 = await h({ m: 'POST', p: `/v1/uploads/${id(up)}/parts`, b: { data: 'bar' } });
1051
+ if (!ok(p1) || !ok(p2) || (p1.body as Body).object !== 'upload.part') return false;
1052
+ // complete in the given part order assembles 'foobar' into a real File
1053
+ const done = await h({ m: 'POST', p: `/v1/uploads/${id(up)}/complete`, b: { part_ids: [(p1.body as Body).id, (p2.body as Body).id] } });
1054
+ if (!ok(done) || (done.body as Body).status !== 'completed' || !(done.body as Body).file) return false;
1055
+ const fileId = (done.body as Body).file.id as string;
1056
+ const content = await h({ m: 'GET', p: `/v1/files/${fileId}/content` });
1057
+ if (content.body !== 'foobar') return false;
1058
+ // adding a part after completion fails; cancel a fresh upload
1059
+ const lateP = await h({ m: 'POST', p: `/v1/uploads/${id(up)}/parts`, b: { data: 'x' } });
1060
+ const up2 = await h({ m: 'POST', p: '/v1/uploads', b: { filename: 'c.jsonl', purpose: 'fine-tune', bytes: 1, mime_type: 'text/plain' } });
1061
+ const cancel = await h({ m: 'POST', p: `/v1/uploads/${id(up2)}/cancel` });
1062
+ return lateP.status === 404 && ok(cancel) && (cancel.body as Body).status === 'cancelled';
1063
+ }),
1064
+ ),
1065
+ done('openai.admin.projects', 'admin', 'Admin: projects create/retrieve/modify/list/archive (+ archived hidden by default)', 'connector', 'niche', () =>
1066
+ withRoot(async (h) => {
1067
+ const noName = await h({ m: 'POST', p: '/v1/organization/projects', b: {} });
1068
+ if (noName.status !== 400) return false;
1069
+ const c = await h({ m: 'POST', p: '/v1/organization/projects', b: { name: 'Alpha' } });
1070
+ if (!ok(c) || (c.body as Body).object !== 'organization.project' || !String(id(c)).startsWith('proj-') || (c.body as Body).status !== 'active') return false;
1071
+ const g = await h({ m: 'GET', p: `/v1/organization/projects/${id(c)}` });
1072
+ if (!ok(g) || id(g) !== id(c)) return false;
1073
+ const upd = await h({ m: 'POST', p: `/v1/organization/projects/${id(c)}`, b: { name: 'Alpha2' } });
1074
+ if ((upd.body as Body).name !== 'Alpha2') return false;
1075
+ const arch = await h({ m: 'POST', p: `/v1/organization/projects/${id(c)}/archive` });
1076
+ if (!ok(arch) || (arch.body as Body).status !== 'archived') return false;
1077
+ // archived project hidden by default, visible with include_archived
1078
+ const def = ((await h({ m: 'GET', p: '/v1/organization/projects' })).body as Body).data as Body[];
1079
+ const withArch = ((await h({ m: 'GET', p: '/v1/organization/projects?include_archived=true' })).body as Body).data as Body[];
1080
+ const missing = await h({ m: 'GET', p: '/v1/organization/projects/proj-nope' });
1081
+ return def.length === 0 && withArch.length === 1 && missing.status === 404;
1082
+ }),
1083
+ ),
1084
+ done('openai.admin.api_keys', 'admin', 'Admin: project API keys create (one-time secret) / list / retrieve (redacted) / delete', 'connector', 'niche', () =>
1085
+ withRoot(async (h) => {
1086
+ const proj = await h({ m: 'POST', p: '/v1/organization/projects', b: { name: 'Keys' } });
1087
+ const pid = id(proj);
1088
+ const key = await h({ m: 'POST', p: `/v1/organization/projects/${pid}/api_keys`, b: { name: 'CI key' } });
1089
+ if (!ok(key)) return false;
1090
+ const kb = key.body as Body;
1091
+ // creation returns the one-time secret value (a synthetic twin key, not a real OpenAI key)
1092
+ if (kb.object !== 'organization.project.api_key' || typeof kb.value !== 'string' || !String(kb.value).startsWith('sk-twin-')) return false;
1093
+ // retrieve only shows the redacted value, never the secret
1094
+ const g = await h({ m: 'GET', p: `/v1/organization/projects/${pid}/api_keys/${kb.id}` });
1095
+ if (!ok(g) || (g.body as Body).value !== undefined || typeof (g.body as Body).redacted_value !== 'string') return false;
1096
+ const list = await h({ m: 'GET', p: `/v1/organization/projects/${pid}/api_keys` });
1097
+ if (((list.body as Body).data as Body[]).length !== 1) return false;
1098
+ const del = await h({ m: 'DELETE', p: `/v1/organization/projects/${pid}/api_keys/${kb.id}` });
1099
+ if (!ok(del) || (del.body as Body).deleted !== true) return false;
1100
+ const after = await h({ m: 'GET', p: `/v1/organization/projects/${pid}/api_keys/${kb.id}` });
1101
+ return after.status === 404;
1102
+ }),
1103
+ ),
1104
+ done('openai.usage.costs', 'admin', 'Usage + costs reporting endpoints computed over REAL recorded usage', 'api', 'niche', () =>
1105
+ withRoot(async (h) => {
1106
+ // with no traffic the report is genuinely EMPTY (nothing hardcoded)
1107
+ const empty = await h({ m: 'GET', p: '/v1/organization/usage/completions' });
1108
+ if (!ok(empty) || (empty.body as Body).object !== 'page' || ((empty.body as Body).data as Body[]).length !== 0) return false;
1109
+ const emptyCosts = await h({ m: 'GET', p: '/v1/organization/costs' });
1110
+ if (((emptyCosts.body as Body).data as Body[]).length !== 0) return false;
1111
+ // generate real usage: two chat calls (gpt-4o) + one embeddings call
1112
+ const c1 = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
1113
+ const c2 = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ messages: [{ role: 'user', content: 'another prompt entirely' }] }) });
1114
+ await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'embed me' } });
1115
+ // completions usage aggregates the two chat calls under gpt-4o
1116
+ const usage = await h({ m: 'GET', p: '/v1/organization/usage/completions' });
1117
+ const buckets = (usage.body as Body).data as Body[];
1118
+ if (buckets.length !== 1) return false;
1119
+ const results = buckets[0]!.results as Body[];
1120
+ const gpt4o = results.find((r) => r.model === 'gpt-4o');
1121
+ if (!gpt4o || gpt4o.object !== 'organization.usage.completions.result' || gpt4o.num_model_requests !== 2) return false;
1122
+ // the recorded input/output tokens equal the sum of the two completions' real usage
1123
+ const expectedIn = (c1.body as Body).usage.prompt_tokens + (c2.body as Body).usage.prompt_tokens;
1124
+ const expectedOut = (c1.body as Body).usage.completion_tokens + (c2.body as Body).usage.completion_tokens;
1125
+ if (gpt4o.input_tokens !== expectedIn || gpt4o.output_tokens !== expectedOut) return false;
1126
+ // embeddings usage is filtered out of the completions view, present under embeddings
1127
+ if (results.some((r) => r.model.startsWith('text-embedding'))) return false;
1128
+ const emb = await h({ m: 'GET', p: '/v1/organization/usage/embeddings' });
1129
+ const embResults = ((emb.body as Body).data as Body[])[0]!.results as Body[];
1130
+ if (!embResults.some((r) => r.model === 'text-embedding-3-small' && r.object === 'organization.usage.embeddings.result')) return false;
1131
+ // costs: a non-zero amount computed from the recorded usage (not hardcoded)
1132
+ const costs = await h({ m: 'GET', p: '/v1/organization/costs' });
1133
+ const costResults = ((costs.body as Body).data as Body[])[0]!.results as Body[];
1134
+ const completionsCost = costResults.find((r) => r.line_item === 'completions');
1135
+ if (!completionsCost || completionsCost.object !== 'organization.costs.result' || completionsCost.amount.currency !== 'usd' || !(completionsCost.amount.value > 0)) return false;
1136
+ // the cost matches the price-table computation over the recorded tokens (gpt-4o: $2.5/$10 per 1M)
1137
+ const expectedCost = (expectedIn / 1e6) * 2.5 + (expectedOut / 1e6) * 10;
1138
+ return Math.abs(completionsCost.amount.value - expectedCost) < 1e-9;
1139
+ }),
1140
+ ),
1141
+
1142
+ // ── Evals API (config → runs → output items) ──────────────────────────────────────────
1143
+ done('openai.evals.api', 'evals', 'Evals API — create/run/list evals + eval runs + output items', 'api', 'niche', () =>
1144
+ withRoot(async (h) => {
1145
+ // create an eval, read it back, assert values
1146
+ const ev = await h({ m: 'POST', p: '/v1/evals', b: { name: 'qa-eval', data_source_config: { type: 'custom', item_schema: { type: 'object' } }, testing_criteria: [{ type: 'string_check', name: 'match', input: '{{item.x}}', reference: 'y', operation: 'eq' }] } });
1147
+ if (!ok(ev) || field(ev, 'object') !== 'eval' || field(ev, 'name') !== 'qa-eval') return false;
1148
+ const evalId = id(ev);
1149
+ const got = await h({ m: 'GET', p: `/v1/evals/${evalId}` });
1150
+ // the testing_criteria round-trips through the kernel byte-for-byte (not fabricated)
1151
+ const tc = field(got, 'testing_criteria') as Body[];
1152
+ if (!ok(got) || id(got) !== evalId || !Array.isArray(tc) || tc[0]?.name !== 'match' || tc[0]?.operation !== 'eq') return false;
1153
+ const dsc = field(got, 'data_source_config') as Body;
1154
+ if (dsc?.type !== 'custom') return false;
1155
+ const list = await h({ m: 'GET', p: '/v1/evals' });
1156
+ if (!ok(list) || !(list.body as Body).data.some((e: Body) => e.id === evalId)) return false;
1157
+ // kick off a run; assert completed + result_counts + per_model_usage reflects the run model
1158
+ const run = await h({ m: 'POST', p: `/v1/evals/${evalId}/runs`, b: { name: 'run-1', data_source: { type: 'completions', model: 'gpt-4.1', source: { type: 'file_content', content: [] } } } });
1159
+ if (!ok(run) || field(run, 'object') !== 'eval.run' || field(run, 'status') !== 'completed') return false;
1160
+ const rc = field(run, 'result_counts') as Body;
1161
+ if (rc.passed !== 1 || rc.total !== 1 || field(run, 'eval_id') !== evalId) return false;
1162
+ // the run echoes the model from the request data_source (not a hardcoded default)
1163
+ if (field(run, 'model') !== 'gpt-4.1') return false;
1164
+ const pmu = field(run, 'per_model_usage') as Body[];
1165
+ if (!Array.isArray(pmu) || pmu[0]?.model_name !== 'gpt-4.1') return false;
1166
+ const runId = id(run);
1167
+ const runGet = await h({ m: 'GET', p: `/v1/evals/${evalId}/runs/${runId}` });
1168
+ if (!ok(runGet) || id(runGet) !== runId) return false;
1169
+ const runList = await h({ m: 'GET', p: `/v1/evals/${evalId}/runs` });
1170
+ if (!ok(runList) || !(runList.body as Body).data.some((r: Body) => r.id === runId)) return false;
1171
+ // output items
1172
+ const items = await h({ m: 'GET', p: `/v1/evals/${evalId}/runs/${runId}/output_items` });
1173
+ if (!ok(items)) return false;
1174
+ const it = (items.body as Body).data?.[0] as Body;
1175
+ if (!it || it.object !== 'eval.run.output_item' || it.status !== 'pass' || it.run_id !== runId) return false;
1176
+ // negative: missing testing_criteria → 400; missing data_source_config → 400; unknown eval id → 404
1177
+ const noCriteria = await h({ m: 'POST', p: '/v1/evals', b: { data_source_config: { type: 'custom' } } });
1178
+ const noConfig = await h({ m: 'POST', p: '/v1/evals', b: { testing_criteria: [] } });
1179
+ const missing = await h({ m: 'GET', p: '/v1/evals/eval-nope' });
1180
+ const missingRun = await h({ m: 'POST', p: '/v1/evals/eval-nope/runs', b: { data_source: {} } });
1181
+ return noCriteria.status === 400 && noConfig.status === 400 && missing.status === 404 && missingRun.status === 404;
1182
+ }),
1183
+ ),
1184
+
1185
+ // ── Containers API (code-interpreter sandboxes → container files) ──────────────────────
1186
+ done('openai.containers.api', 'containers', 'Containers API — create/list/delete containers + container files (code-interpreter sandboxes)', 'api', 'niche', () =>
1187
+ withRoot(async (h) => {
1188
+ const c = await h({ m: 'POST', p: '/v1/containers', b: { name: 'sandbox-1' } });
1189
+ if (!ok(c) || field(c, 'object') !== 'container' || field(c, 'status') !== 'running' || field(c, 'name') !== 'sandbox-1') return false;
1190
+ const cid = id(c);
1191
+ const got = await h({ m: 'GET', p: `/v1/containers/${cid}` });
1192
+ if (!ok(got) || id(got) !== cid) return false;
1193
+ const list = await h({ m: 'GET', p: '/v1/containers' });
1194
+ if (!ok(list) || !(list.body as Body).data.some((x: Body) => x.id === cid)) return false;
1195
+ // container file with inline text content → read back metadata + content
1196
+ const cf = await h({ m: 'POST', p: `/v1/containers/${cid}/files`, b: { path: '/mnt/data/a.txt', content: 'hello sandbox' } });
1197
+ if (!ok(cf) || field(cf, 'object') !== 'container.file' || field(cf, 'container_id') !== cid || field(cf, 'bytes') !== 'hello sandbox'.length || field(cf, 'path') !== '/mnt/data/a.txt') return false;
1198
+ const fid = id(cf);
1199
+ const content = await h({ m: 'GET', p: `/v1/containers/${cid}/files/${fid}/content` });
1200
+ if (!ok(content) || content.body !== 'hello sandbox') return false;
1201
+ const fileGet = await h({ m: 'GET', p: `/v1/containers/${cid}/files/${fid}` });
1202
+ if (!ok(fileGet) || id(fileGet) !== fid) return false;
1203
+ const del = await h({ m: 'DELETE', p: `/v1/containers/${cid}/files/${fid}` });
1204
+ if (!ok(del) || (del.body as Body).deleted !== true) return false;
1205
+ const afterDel = await h({ m: 'GET', p: `/v1/containers/${cid}/files/${fid}` });
1206
+ if (afterDel.status !== 404) return false;
1207
+ // delete the container; subsequent GET → 404
1208
+ const delC = await h({ m: 'DELETE', p: `/v1/containers/${cid}` });
1209
+ const gone = await h({ m: 'GET', p: `/v1/containers/${cid}` });
1210
+ // negative: create without name → 400; unknown container → 404
1211
+ const bad = await h({ m: 'POST', p: '/v1/containers', b: {} });
1212
+ const missing = await h({ m: 'GET', p: '/v1/containers/cntr-nope' });
1213
+ return ok(delC) && gone.status === 404 && bad.status === 400 && missing.status === 404;
1214
+ }),
1215
+ ),
1216
+
1217
+ // ── Background responses (queued → poll → completed | cancel) ──────────────────────────
1218
+ done('openai.responses.background', 'responses', 'Background responses (background:true → queued status + poll + cancel)', 'api', 'niche', () =>
1219
+ withRoot(async (h) => {
1220
+ // background:true → immediate queued response with no output yet
1221
+ const bg = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'background task A', background: true } });
1222
+ if (!ok(bg) || field(bg, 'status') !== 'queued' || field(bg, 'background') !== true) return false;
1223
+ if (field(bg, 'output_text') !== null || ((field(bg, 'output') as unknown[]) ?? []).length !== 0) return false;
1224
+ const rid = id(bg);
1225
+ // first poll → transitions to completed with the real stub output + usage
1226
+ const poll = await h({ m: 'GET', p: `/v1/responses/${rid}` });
1227
+ if (!ok(poll) || field(poll, 'status') !== 'completed') return false;
1228
+ const text = field(poll, 'output_text');
1229
+ if (typeof text !== 'string' || !text.includes('[twin-stub')) return false;
1230
+ // usage must be coherent: total = input + output, and output > 0 (the stub produced text)
1231
+ const usage = field(poll, 'usage') as Body;
1232
+ if (!usage || typeof usage.output_tokens !== 'number' || usage.output_tokens <= 0) return false;
1233
+ if (usage.total_tokens !== usage.input_tokens + usage.output_tokens) return false;
1234
+ // the completion is billable: a usage_record was recorded → the usage report is non-empty
1235
+ const report = await h({ m: 'GET', p: '/v1/organization/usage/responses' });
1236
+ const buckets = (report.body as Body).data as Body[];
1237
+ if (!ok(report) || buckets.length === 0) return false;
1238
+ // a second poll stays completed (idempotent) with the same output text (no re-billing visible)
1239
+ const poll2 = await h({ m: 'GET', p: `/v1/responses/${rid}` });
1240
+ if (field(poll2, 'status') !== 'completed' || field(poll2, 'output_text') !== text) return false;
1241
+ // cancel a still-queued background response → cancelled
1242
+ const bg2 = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'background task B', background: true } });
1243
+ const rid2 = id(bg2);
1244
+ const cancel = await h({ m: 'POST', p: `/v1/responses/${rid2}/cancel` });
1245
+ if (!ok(cancel) || field(cancel, 'status') !== 'cancelled') return false;
1246
+ // negative: cancel an already-completed response → 400; cancel unknown id → 404
1247
+ const reCancel = await h({ m: 'POST', p: `/v1/responses/${rid}/cancel` });
1248
+ const missing = await h({ m: 'POST', p: '/v1/responses/resp-nope/cancel' });
1249
+ return reCancel.status === 400 && missing.status === 404;
1250
+ }),
1251
+ ),
1252
+
1253
+ // ── Fine-tuning pause / resume ─────────────────────────────────────────────────────────
1254
+ done('openai.fine_tuning.pause_resume', 'fine_tuning', 'Fine-tuning job pause / resume — status transitions', 'api', 'niche', () =>
1255
+ withRoot(async (h) => {
1256
+ const job = await h({ m: 'POST', p: '/v1/fine_tuning/jobs', b: { model: 'gpt-4o-mini', training_file: 'file-twin-1' } });
1257
+ if (!ok(job) || field(job, 'status') !== 'succeeded') return false;
1258
+ const jid = id(job);
1259
+ // pause → status 'paused'
1260
+ const paused = await h({ m: 'POST', p: `/v1/fine_tuning/jobs/${jid}/pause` });
1261
+ if (!ok(paused) || field(paused, 'status') !== 'paused') return false;
1262
+ const readPaused = await h({ m: 'GET', p: `/v1/fine_tuning/jobs/${jid}` });
1263
+ if (field(readPaused, 'status') !== 'paused') return false;
1264
+ // resume → back to 'succeeded', and the persisted state round-trips
1265
+ const resumed = await h({ m: 'POST', p: `/v1/fine_tuning/jobs/${jid}/resume` });
1266
+ if (!ok(resumed) || field(resumed, 'status') !== 'succeeded') return false;
1267
+ const readResumed = await h({ m: 'GET', p: `/v1/fine_tuning/jobs/${jid}` });
1268
+ if (field(readResumed, 'status') !== 'succeeded') return false;
1269
+ // negative: resuming the now-succeeded (non-paused) job → 400; pause unknown job → 404
1270
+ const badResume = await h({ m: 'POST', p: `/v1/fine_tuning/jobs/${jid}/resume` });
1271
+ const missing = await h({ m: 'POST', p: '/v1/fine_tuning/jobs/ftjob-nope/pause' });
1272
+ return badResume.status === 400 && missing.status === 404;
1273
+ }),
1274
+ ),
1275
+
1276
+ // ── Webhook signature verification (REAL HMAC, Standard-Webhooks scheme) ────────────────
1277
+ done('openai.webhooks.verify', 'webhooks', 'Webhook event signature verification (REAL signing-secret HMAC, Standard-Webhooks)', 'api', 'niche', async () => {
1278
+ // a 32-byte base64 signing secret (whsec_<base64>), like the vendor mints.
1279
+ const secret = 'whsec_' + Buffer.from('0123456789abcdef0123456789abcdef').toString('base64');
1280
+ const signed = buildSignedOpenAIWebhook({ type: 'response.completed', resourceId: 'resp-twin-abc', secret, occurredAt: '2026-01-01T00:00:00Z', webhookId: 'wh-1' });
1281
+ // a good delivery verifies and returns the parsed event with faithful shape
1282
+ const ev = verifyOpenAIWebhook(signed.body, signed.headers, secret);
1283
+ if (ev.object !== 'event' || ev.type !== 'response.completed' || ev.data.id !== 'resp-twin-abc') return false;
1284
+ // the signature header is the Standard-Webhooks `v1,<base64>` form computed over id.ts.payload
1285
+ const recomputed = computeOpenAIWebhookSignature('wh-1', Math.floor(Date.parse('2026-01-01T00:00:00Z') / 1000), signed.body, secret);
1286
+ if (signed.headers['webhook-signature'] !== recomputed) return false;
1287
+ // a TAMPERED payload must be rejected (the real verifier throws)
1288
+ let tamperedRejected = false;
1289
+ try { verifyOpenAIWebhook(signed.body + ' ', signed.headers, secret); } catch (e) { tamperedRejected = e instanceof OpenAIWebhookVerificationError; }
1290
+ if (!tamperedRejected) return false;
1291
+ // a wrong secret must be rejected
1292
+ let wrongSecretRejected = false;
1293
+ try { verifyOpenAIWebhook(signed.body, signed.headers, 'whsec_' + Buffer.from('ffffffffffffffffffffffffffffffff').toString('base64')); } catch (e) { wrongSecretRejected = e instanceof OpenAIWebhookVerificationError; }
1294
+ if (!wrongSecretRejected) return false;
1295
+ // missing headers must be rejected
1296
+ let missingRejected = false;
1297
+ try { verifyOpenAIWebhook(signed.body, {}, secret); } catch (e) { missingRejected = e instanceof OpenAIWebhookVerificationError; }
1298
+ return missingRejected;
1299
+ }),
1300
+
1301
+ // ── Genuinely-unmodeled real surfaces (honest todos — the API has these; the twin does not yet) ──
1302
+ todo('openai.evals.run_cancel_delete', 'evals', 'Evals — cancel a run + delete an eval/run', 'api', 'niche'),
1303
+ todo('openai.evals.update', 'evals', 'Evals — update an eval (name/metadata) via POST /v1/evals/:id', 'api', 'niche'),
1304
+ todo('openai.responses.background_stream_resume', 'responses', 'Background responses — resume an in-progress stream via starting_after cursor', 'api', 'niche'),
1305
+ todo('openai.fine_tuning.checkpoint_permissions', 'fine_tuning', 'Fine-tuning checkpoint permissions (per-project grant/list/revoke)', 'api', 'niche'),
1306
+ todo('openai.admin.invites_and_users', 'admin', 'Admin API — organization invites + users (list/retrieve/modify/delete)', 'api', 'niche'),
1307
+
1308
+ // ── Errors / protocol ─────────────────────────────────────────────────────────────────
1309
+ done('openai.errors.not_found', 'errors', 'Vendor-faithful 404 on unknown route', 'api', 'core', () =>
1310
+ withRoot(async (h) => {
1311
+ const r = await h({ m: 'GET', p: '/v1/nonexistent' });
1312
+ return r.status === 404 && typeof (r.body as Body).error === 'object' && (r.body as Body).error.type === 'invalid_request_error';
1313
+ }),
1314
+ ),
1315
+ done('openai.errors.invalid_request', 'errors', 'Vendor-faithful 400 invalid_request_error envelope (error.message/type/param/code)', 'api', 'core', () =>
1316
+ withRoot(async (h) => {
1317
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: '{bad json' as any });
1318
+ const e = (r.body as Body).error;
1319
+ return r.status === 400 && e && typeof e.message === 'string' && e.type === 'invalid_request_error' && 'param' in e && 'code' in e;
1320
+ }),
1321
+ ),
1322
+ done('openai.read_only_guard', 'errors', 'Read-only mode rejects mutations (405 vendor-shaped error)', 'api', 'common', () => {
1323
+ const root = mkdtempSync(join(tmpdir(), 'openai-cap-'));
1324
+ return handleOpenAITwinRequest({ method: 'POST', path: '/v1/chat/completions', body: JSON.stringify(CHAT()), root, readOnly: true })
1325
+ .then((r) => r.status === 405 && (r.body as Body).error?.type === 'invalid_request_error')
1326
+ .finally(() => rmSync(root, { recursive: true, force: true }));
1327
+ }),
1328
+ done('openai.auth.faked', 'errors', 'Auth is modeled but permissive — any non-empty key is accepted; trusted in-process calls pass', 'api', 'common', () =>
1329
+ withRoot(async (h) => {
1330
+ // trusted in-process calls (no headers/apiKey) are NOT auth-gated → succeed with a REAL model catalog.
1331
+ const trusted = await h({ m: 'GET', p: '/v1/models' });
1332
+ if (!ok(trusted)) return false;
1333
+ const tb = trusted.body as Body;
1334
+ if (tb.object !== 'list' || !Array.isArray(tb.data) || tb.data.length === 0) return false;
1335
+ // a request that carries a valid bearer key is accepted (the twin can't validate real keys),
1336
+ // and it must return the SAME real, non-empty catalog content — not just any 2xx status.
1337
+ const withKey = await handleOpenAITwinRequest({ method: 'GET', path: '/v1/models', headers: { authorization: 'Bearer sk-anything' } });
1338
+ if (!ok(withKey)) return false;
1339
+ const wb = withKey.body as Body;
1340
+ return wb.object === 'list' && Array.isArray(wb.data) && wb.data.length === tb.data.length && JSON.stringify(wb) === JSON.stringify(tb);
1341
+ }),
1342
+ ),
1343
+ done('openai.errors.auth_401', 'errors', 'Authentication error (401) — modeled auth: missing/invalid credential on a request carrying an auth surface', 'api', 'niche', () =>
1344
+ withRootH(async (h) => {
1345
+ // a request carrying an auth surface (headers present) but no credential → 401
1346
+ const missing = await h({ m: 'GET', p: '/v1/models', headers: {} });
1347
+ if (missing.status !== 401) return false;
1348
+ const me = (missing.body as Body).error;
1349
+ if (me?.type !== 'invalid_request_error' || me?.code !== 'invalid_api_key') return false;
1350
+ // an empty bearer is also missing
1351
+ const emptyBearer = await h({ m: 'GET', p: '/v1/models', headers: { authorization: 'Bearer ' } });
1352
+ if (emptyBearer.status !== 401) return false;
1353
+ // the reserved sentinel exercises the invalid-key path deterministically → 401
1354
+ const invalid = await h({ m: 'GET', p: '/v1/models', headers: { authorization: 'Bearer sk-invalid' } });
1355
+ if (invalid.status !== 401) return false;
1356
+ // a valid bearer passes; and a trusted call WITHOUT any auth surface is NOT gated
1357
+ const valid = await h({ m: 'GET', p: '/v1/models', headers: { authorization: 'Bearer sk-twin-good' } });
1358
+ const trusted = await handleOpenAITwinRequest({ method: 'GET', path: '/v1/models' });
1359
+ return ok(valid) && ok(trusted);
1360
+ }),
1361
+ ),
1362
+ done('openai.errors.rate_limit_429', 'errors', 'Rate limit (429) — deterministic opt-in trigger → vendor envelope + Retry-After + x-ratelimit-* headers', 'api', 'niche', () =>
1363
+ withRootH(async (h) => {
1364
+ const auth = { authorization: 'Bearer sk-twin-good' };
1365
+ // no trigger → normal success
1366
+ const normal = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT(), headers: auth });
1367
+ if (!ok(normal)) return false;
1368
+ // the deterministic trigger header → 429 with the vendor error envelope + rate-limit headers
1369
+ const limited = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT(), headers: { ...auth, 'x-twin-force-rate-limit': '1' } });
1370
+ if (limited.status !== 429) return false;
1371
+ const e = (limited.body as Body).error;
1372
+ if (e?.type !== 'rate_limit_exceeded' || e?.code !== 'rate_limit_exceeded') return false;
1373
+ const hdrs = limited.headers;
1374
+ if (!hdrs || hdrs['retry-after'] !== '1') return false;
1375
+ return hdrs['x-ratelimit-remaining-requests'] === '0' && typeof hdrs['x-ratelimit-limit-tokens'] === 'string';
1376
+ }),
1377
+ ),
1378
+ done('openai.protocol.idempotency', 'errors', 'Idempotency-Key header dedup — re-issue replays the same response; a new key → a new result', 'api', 'niche', () =>
1379
+ withRootH(async (h) => {
1380
+ const auth = { authorization: 'Bearer sk-twin-good' };
1381
+ // a mutation (file create) with an Idempotency-Key → re-issue replays the SAME id (no double create)
1382
+ const first = await h({ m: 'POST', p: '/v1/files', b: { purpose: 'batch', filename: 'a.jsonl', content: '{}' }, headers: { ...auth, 'idempotency-key': 'key-A' } });
1383
+ if (!ok(first)) return false;
1384
+ const id1 = (first.body as Body).id as string;
1385
+ const replay = await h({ m: 'POST', p: '/v1/files', b: { purpose: 'batch', filename: 'a.jsonl', content: '{}' }, headers: { ...auth, 'idempotency-key': 'key-A' } });
1386
+ if (!ok(replay) || (replay.body as Body).id !== id1) return false;
1387
+ // the side effect happened ONCE: exactly one file exists
1388
+ const listed = (await h({ m: 'GET', p: '/v1/files', headers: auth })).body as Body;
1389
+ if ((listed.data as Body[]).length !== 1) return false;
1390
+ // a DIFFERENT key → a new, distinct create
1391
+ const second = await h({ m: 'POST', p: '/v1/files', b: { purpose: 'batch', filename: 'a.jsonl', content: '{}' }, headers: { ...auth, 'idempotency-key': 'key-B' } });
1392
+ if (!ok(second) || (second.body as Body).id === id1) return false;
1393
+ const listed2 = (await h({ m: 'GET', p: '/v1/files', headers: auth })).body as Body;
1394
+ // even the replayed body is byte-identical to the first response (full envelope replay)
1395
+ return (listed2.data as Body[]).length === 2 && JSON.stringify(replay.body) === JSON.stringify(first.body);
1396
+ }),
1397
+ ),
1398
+ done('openai.protocol.pagination', 'errors', 'Cursor pagination: after/limit + has_more/first_id/last_id across list endpoints', 'api', 'common', () =>
1399
+ withRoot(async (h) => {
1400
+ // create 3 files; page with limit=2 → has_more + last_id; then after=last_id → final page
1401
+ for (const f of ['a', 'b', 'c']) await h({ m: 'POST', p: '/v1/files', b: { purpose: 'batch', filename: `${f}.jsonl`, content: '{}' } });
1402
+ const p1 = await h({ m: 'GET', p: '/v1/files?limit=2' });
1403
+ const b1 = p1.body as Body;
1404
+ if (!ok(p1) || b1.data.length !== 2 || b1.has_more !== true || b1.last_id !== b1.data[1].id || b1.first_id !== b1.data[0].id) return false;
1405
+ const p2 = await h({ m: 'GET', p: `/v1/files?limit=2&after=${b1.last_id}` });
1406
+ const b2 = p2.body as Body;
1407
+ if (b2.data.length !== 1 || b2.has_more !== false) return false;
1408
+ // the two pages partition the set (no overlap)
1409
+ if (b2.data[0].id === b1.data[0].id || b2.data[0].id === b1.data[1].id) return false;
1410
+ // an unknown cursor → empty page (vendor returns nothing past the end)
1411
+ const empty = await h({ m: 'GET', p: '/v1/files?after=file-nope' });
1412
+ return (empty.body as Body).data.length === 0;
1413
+ }),
1414
+ ),
1415
+
1416
+ // ── Connector (pull/push over an injected client) ─────────────────────────────────────
1417
+ done('openai.connector.read_surface', 'connector', 'Connector read surface (models + files + batches in one pass)', 'connector', 'core', () =>
1418
+ withRoot(async (h) => {
1419
+ const models = await h({ m: 'GET', p: '/v1/models' });
1420
+ const files = await h({ m: 'GET', p: '/v1/files' });
1421
+ const batches = await h({ m: 'GET', p: '/v1/batches' });
1422
+ return ok(models) && Array.isArray((models.body as Body).data) && ok(files) && ok(batches);
1423
+ }),
1424
+ ),
1425
+ done('openai.connector.pull_map', 'connector', 'Connector pulls + maps real state into the twin (offline, injected client)', 'connector', 'core', async () => {
1426
+ const { mapBatch, mapFile, syncOpenAIFromReal } = await import('./openai-connector.ts');
1427
+ const { projectResources } = await import('@volter/twin');
1428
+ const root = mkdtempSync(join(tmpdir(), 'openai-cap-'));
1429
+ try {
1430
+ const mappedFile = mapFile({ id: 'file-real', purpose: 'batch', bytes: 12, status: 'processed', created_at: 1 });
1431
+ const mappedBatch = mapBatch({ id: 'batch-real', status: 'completed', created_at: 1, request_counts: { total: 1 } });
1432
+ if (mappedFile.type !== 'file' || mappedBatch.fields.status !== 'completed') return false;
1433
+ const fake = async (_m: 'GET' | 'POST' | 'DELETE', p: string) => {
1434
+ if (p.startsWith('/v1/files')) return { data: [{ id: 'file-real', purpose: 'batch', bytes: 12, status: 'processed', created_at: 1 }] };
1435
+ if (p.startsWith('/v1/batches')) return { data: [{ id: 'batch-real', status: 'completed', created_at: 1, request_counts: { total: 1 } }] };
1436
+ return { data: [] };
1437
+ };
1438
+ const res = await syncOpenAIFromReal(fake, { root, occurredAt: '2026-06-15T00:00:00Z' });
1439
+ if (res.deltasAppended < 1) return false;
1440
+ const got = projectResources('openai', root).find((r) => r.id === 'file-real');
1441
+ const again = await syncOpenAIFromReal(fake, { root, occurredAt: '2026-06-15T00:00:00Z' });
1442
+ return !!got && got.status === 'processed' && again.deltasAppended === 0;
1443
+ } catch (err) {
1444
+ if (isInfrastructureError(err)) throw harnessError('openai.connector.pull_map', err);
1445
+ return false;
1446
+ } finally {
1447
+ rmSync(root, { recursive: true, force: true });
1448
+ }
1449
+ }),
1450
+ done('openai.connector.push_confirm', 'connector', 'Connector pushes pending local writes to real + confirms them (offline, injected client)', 'connector', 'core', async () => {
1451
+ const { pushPendingOpenAIActions, openaiRequestForAction } = await import('./openai-connector.ts');
1452
+ const { pendingActions } = await import('@volter/twin');
1453
+ const root = mkdtempSync(join(tmpdir(), 'openai-cap-'));
1454
+ try {
1455
+ const created = await handleOpenAITwinRequest({ method: 'POST', path: '/v1/batches', body: JSON.stringify({ input_file_id: 'file-x', endpoint: '/v1/chat/completions', completion_window: '24h' }), root, occurredAt: '2026-06-15T00:00:00Z' });
1456
+ if (created.status !== 200) return false;
1457
+ if (pendingActions('openai', root).length === 0) return false;
1458
+ const createReq = openaiRequestForAction({ operation: 'batch.create', subject: { type: 'batch', id: 'batch-twin-1' } });
1459
+ const cancelReq = openaiRequestForAction({ operation: 'batch.cancel', subject: { type: 'batch', id: 'batch-twin-1' } });
1460
+ const delReq = openaiRequestForAction({ operation: 'file.delete', subject: { type: 'file', id: 'file-twin-1' } });
1461
+ if (createReq.path !== '/v1/batches' || cancelReq.path !== '/v1/batches/batch-twin-1/cancel' || delReq.method !== 'DELETE') return false;
1462
+ const calls: string[] = [];
1463
+ const fake = async (m: 'GET' | 'POST' | 'DELETE', p: string) => { calls.push(`${m} ${p}`); return { id: 'batch-real-1' }; };
1464
+ const res = await pushPendingOpenAIActions(fake, { root, occurredAt: '2026-06-15T00:00:01Z' });
1465
+ if (res.pushed < 1 || calls.length < 1) return false;
1466
+ const after = await pushPendingOpenAIActions(fake, { root, occurredAt: '2026-06-15T00:00:02Z' });
1467
+ return pendingActions('openai', root).length === 0 && after.pushed === 0;
1468
+ } catch (err) {
1469
+ if (isInfrastructureError(err)) throw harnessError('openai.connector.push_confirm', err);
1470
+ return false;
1471
+ } finally {
1472
+ rmSync(root, { recursive: true, force: true });
1473
+ }
1474
+ }),
1475
+ done('openai.connector.unsupported_op_fails', 'connector', 'Connector refuses to silently drop an unsupported push op', 'connector', 'common', async () => {
1476
+ const { pushOpenAIAction } = await import('./openai-connector.ts');
1477
+ try {
1478
+ const fake = async () => ({ id: 'x' });
1479
+ let threw = false;
1480
+ try { await pushOpenAIAction(fake, { operation: 'batch.frobnicate', subject: { type: 'batch', id: 'batch-1' }, fields: {} }); } catch { threw = true; }
1481
+ return threw;
1482
+ } catch (err) {
1483
+ if (isInfrastructureError(err)) throw harnessError('openai.connector.unsupported_op_fails', err);
1484
+ return false;
1485
+ }
1486
+ }),
1487
+ done('openai.connector.full_sync', 'connector', 'Connector full bi-directional sync: push pending then pull all collections (idempotent)', 'connector', 'common', async () => {
1488
+ const { fullSyncOpenAI } = await import('./openai-connector.ts');
1489
+ const { pendingActions } = await import('@volter/twin');
1490
+ const root = mkdtempSync(join(tmpdir(), 'openai-cap-'));
1491
+ try {
1492
+ // a local write becomes a pending action
1493
+ const created = await handleOpenAITwinRequest({ method: 'POST', path: '/v1/batches', body: JSON.stringify({ input_file_id: 'file-x', endpoint: '/v1/chat/completions', completion_window: '24h' }), root, occurredAt: '2026-06-15T00:00:00Z' });
1494
+ if (created.status !== 200 || pendingActions('openai', root).length === 0) return false;
1495
+ const calls: string[] = [];
1496
+ const fake = async (m: 'GET' | 'POST' | 'DELETE', p: string) => {
1497
+ calls.push(`${m} ${p}`);
1498
+ if (m === 'GET' && p.startsWith('/v1/files')) return { data: [{ id: 'file-real', purpose: 'batch', bytes: 5, status: 'processed', created_at: 1 }] };
1499
+ if (m === 'GET' && p.startsWith('/v1/batches')) return { data: [{ id: 'batch-real', status: 'completed', created_at: 1, request_counts: { total: 1 } }] };
1500
+ if (m === 'GET') return { data: [] };
1501
+ return { id: 'batch-real-pushed' };
1502
+ };
1503
+ const res = await fullSyncOpenAI(fake, { root, occurredAt: '2026-06-15T00:00:01Z' });
1504
+ // pushed the pending write, observed real state, appended deltas, swept all 4 collections
1505
+ if (res.pushed < 1 || res.deltasAppended < 1 || res.collections < 4) return false;
1506
+ if (pendingActions('openai', root).length !== 0) return false;
1507
+ // idempotent: a second run with no pending + identical real state changes nothing
1508
+ const again = await fullSyncOpenAI(fake, { root, occurredAt: '2026-06-15T00:00:02Z' });
1509
+ return again.pushed === 0 && again.deltasAppended === 0;
1510
+ } catch (err) {
1511
+ if (isInfrastructureError(err)) throw harnessError('openai.connector.full_sync', err);
1512
+ return false;
1513
+ } finally {
1514
+ rmSync(root, { recursive: true, force: true });
1515
+ }
1516
+ }),
1517
+
1518
+ // ── Conformance harness ───────────────────────────────────────────────────────────────
1519
+ done('openai.conformance.envelopes', 'conformance', 'Offline conformance harness passes (envelope shapes across the surface)', 'connector', 'core', async () => {
1520
+ const { checkOpenAIConformance } = await import('./openai-conformance.ts');
1521
+ const report = await checkOpenAIConformance();
1522
+ return report.ok && report.checksRun >= 6;
1523
+ }),
1524
+ // ── Pull-surface coverage audit gaps (TWIN-46 / G2) — filed as manifest todos, which is
1525
+ // what the demand-ordered build list is drawn from. See pull-audit.json (repo root) for the
1526
+ // per-pack pull-vs-read-surface evidence.
1527
+ todo('openai.connector.pull_model', 'connector', 'Connector: pull the real model catalog (GET /v1/models) into the twin', 'connector', 'core'),
1528
+ todo('openai.connector.pull_assistant', 'connector', 'Connector: pull assistants from the real account', 'connector', 'niche'),
1529
+
1530
+ // ── Missing-area sweep (TWIN-87 / F1) ────────────────────────────────────────────────────
1531
+ // The legacy completions endpoint and org audit logs had NO manifest entry of any status —
1532
+ // neither the twin nor the manifest modeled them, so they never surfaced as gaps. Filed as
1533
+ // honest todos (both genuinely buildable). See OPENAI_AREAS below + openai-capabilities.
1534
+ // test.ts's area-census meta-test, gate-wired so a future whole-area omission fails here.
1535
+ todo('openai.completions_legacy.create', 'completions_legacy', 'Legacy POST /v1/completions (pre-chat text completions endpoint)', 'api', 'niche'),
1536
+ todo('openai.completions_legacy.streaming', 'completions_legacy', 'Legacy completions streaming (text/event-stream deltas)', 'api', 'niche'),
1537
+ todo('openai.admin.audit_logs.list', 'admin', 'Admin API — organization audit logs (GET /v1/organization/audit_logs, filterable)', 'api', 'niche'),
1538
+ todo('openai.admin.audit_logs.get', 'admin', 'Admin API — retrieve a single audit log event by id', 'api', 'niche'),
1539
+
1540
+ ];
1541
+
1542
+ // TWIN-87 committed area census — the vendor's top-level API product areas (docs nav /
1543
+ // OpenAPI tags), authored top-down independent of what a manifest entry happens to already
1544
+ // exist for. openai-capabilities.test.ts's area-census meta-test (assertAreaCensus) fails the
1545
+ // gate if a declared area has zero manifest entries and no named exclusion, OR if a manifest
1546
+ // entry's `area` drifts outside this list — so a whole missing area (legacy completions, org
1547
+ // audit logs were the concrete F1 leak) can never again hide invisibly.
1548
+ export const OPENAI_AREAS = [
1549
+ 'admin', 'assistants', 'audio', 'batches', 'chat', 'completions_legacy', 'conformance',
1550
+ 'connector', 'containers', 'embeddings', 'errors', 'evals', 'files', 'fine_tuning', 'images',
1551
+ 'models', 'moderations', 'responses', 'streaming', 'tools', 'uploads', 'vector_stores', 'webhooks',
1552
+ ] as const;
1553
+
1554
+ export function openaiCapabilities(): Promise<CapabilityReport> {
1555
+ return checkCapabilities('openai', OPENAI_CAPABILITIES);
1556
+ }