@volter/twin-openai 0.1.1 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/README.md +33 -30
  2. package/defaults/handlers.json +10 -0
  3. package/dist/defaults/handlers.json +10 -0
  4. package/dist/src/cli.d.ts +2 -0
  5. package/dist/src/cli.js +29 -0
  6. package/dist/src/generated/surface.gen.json +1 -0
  7. package/dist/src/generated/ui.gen.json +1 -0
  8. package/dist/src/index.d.ts +19 -0
  9. package/dist/src/index.js +72 -0
  10. package/dist/src/manifest.d.ts +6 -0
  11. package/dist/src/manifest.js +323 -0
  12. package/dist/src/openai-budget.d.ts +53 -0
  13. package/dist/src/openai-budget.js +147 -0
  14. package/dist/src/openai-capabilities.d.ts +4 -0
  15. package/dist/src/openai-capabilities.js +1569 -0
  16. package/dist/src/openai-conformance.d.ts +13 -0
  17. package/dist/src/openai-conformance.js +116 -0
  18. package/dist/src/openai-connector.d.ts +86 -0
  19. package/dist/src/openai-connector.js +291 -0
  20. package/dist/src/openai-media.d.ts +43 -0
  21. package/dist/src/openai-media.js +257 -0
  22. package/dist/src/openai-models.d.ts +74 -0
  23. package/dist/src/openai-models.js +148 -0
  24. package/dist/src/openai-scenario.d.ts +51 -0
  25. package/dist/src/openai-scenario.js +166 -0
  26. package/dist/src/openai-server.d.ts +40 -0
  27. package/dist/src/openai-server.js +126 -0
  28. package/dist/src/openai-stub.d.ts +82 -0
  29. package/dist/src/openai-stub.js +256 -0
  30. package/dist/src/openai-twin.d.ts +182 -0
  31. package/dist/src/openai-twin.js +1117 -0
  32. package/dist/src/openai-types.d.ts +194 -0
  33. package/dist/src/openai-types.js +4 -0
  34. package/dist/src/openai-webhooks.d.ts +47 -0
  35. package/dist/src/openai-webhooks.js +99 -0
  36. package/dist/src/screens/api-keys.d.ts +16 -0
  37. package/dist/src/screens/api-keys.js +131 -0
  38. package/dist/src/screens/session.d.ts +22 -0
  39. package/dist/src/screens/session.js +115 -0
  40. package/dist/src/semantics/assistants.d.ts +2 -0
  41. package/dist/src/semantics/assistants.js +331 -0
  42. package/dist/src/semantics/audio.d.ts +2 -0
  43. package/dist/src/semantics/audio.js +27 -0
  44. package/dist/src/semantics/batches.d.ts +4 -0
  45. package/dist/src/semantics/batches.js +86 -0
  46. package/dist/src/semantics/chat-completions.d.ts +3 -0
  47. package/dist/src/semantics/chat-completions.js +58 -0
  48. package/dist/src/semantics/containers.d.ts +2 -0
  49. package/dist/src/semantics/containers.js +147 -0
  50. package/dist/src/semantics/embeddings.d.ts +2 -0
  51. package/dist/src/semantics/embeddings.js +13 -0
  52. package/dist/src/semantics/evals.d.ts +2 -0
  53. package/dist/src/semantics/evals.js +173 -0
  54. package/dist/src/semantics/files.d.ts +13 -0
  55. package/dist/src/semantics/files.js +59 -0
  56. package/dist/src/semantics/fine-tuning.d.ts +4 -0
  57. package/dist/src/semantics/fine-tuning.js +178 -0
  58. package/dist/src/semantics/images.d.ts +2 -0
  59. package/dist/src/semantics/images.js +18 -0
  60. package/dist/src/semantics/index.d.ts +8 -0
  61. package/dist/src/semantics/index.js +46 -0
  62. package/dist/src/semantics/models.d.ts +2 -0
  63. package/dist/src/semantics/models.js +34 -0
  64. package/dist/src/semantics/moderations.d.ts +2 -0
  65. package/dist/src/semantics/moderations.js +12 -0
  66. package/dist/src/semantics/organization.d.ts +2 -0
  67. package/dist/src/semantics/organization.js +67 -0
  68. package/dist/src/semantics/progress.d.ts +22 -0
  69. package/dist/src/semantics/progress.js +63 -0
  70. package/dist/src/semantics/responses.d.ts +3 -0
  71. package/dist/src/semantics/responses.js +153 -0
  72. package/dist/src/semantics/shared.d.ts +32 -0
  73. package/dist/src/semantics/shared.js +69 -0
  74. package/dist/src/semantics/uploads.d.ts +2 -0
  75. package/dist/src/semantics/uploads.js +84 -0
  76. package/dist/src/semantics/vector-stores.d.ts +2 -0
  77. package/dist/src/semantics/vector-stores.js +281 -0
  78. package/dist/test-fixtures/openai-openapi-operations.SOURCE.md +18 -0
  79. package/dist/test-fixtures/openai-openapi-operations.json +1849 -0
  80. package/package.json +21 -10
  81. package/src/cli.ts +9 -7
  82. package/src/generated/surface.gen.json +1 -0
  83. package/src/generated/ui.gen.json +1 -0
  84. package/src/index.ts +20 -10
  85. package/src/manifest.ts +343 -0
  86. package/src/openai-budget.ts +4 -4
  87. package/src/openai-capabilities.ts +177 -195
  88. package/src/openai-conformance.ts +1 -1
  89. package/src/openai-connector.ts +40 -43
  90. package/src/openai-media.ts +225 -0
  91. package/src/openai-models.ts +145 -15
  92. package/src/openai-scenario.ts +46 -10
  93. package/src/openai-server.ts +65 -108
  94. package/src/openai-stub.ts +54 -30
  95. package/src/openai-twin.ts +760 -1665
  96. package/src/openai-types.ts +24 -6
  97. package/src/openai-webhooks.ts +2 -1
  98. package/src/screens/api-keys.tsx +138 -0
  99. package/src/screens/session.tsx +131 -0
  100. package/src/semantics/assistants.ts +336 -0
  101. package/src/semantics/audio.ts +31 -0
  102. package/src/semantics/batches.ts +88 -0
  103. package/src/semantics/chat-completions.ts +66 -0
  104. package/src/semantics/containers.ts +151 -0
  105. package/src/semantics/embeddings.ts +19 -0
  106. package/src/semantics/evals.ts +182 -0
  107. package/src/semantics/files.ts +67 -0
  108. package/src/semantics/fine-tuning.ts +185 -0
  109. package/src/semantics/images.ts +23 -0
  110. package/src/semantics/index.ts +52 -0
  111. package/src/semantics/models.ts +41 -0
  112. package/src/semantics/moderations.ts +14 -0
  113. package/src/semantics/organization.ts +76 -0
  114. package/src/semantics/progress.ts +72 -0
  115. package/src/semantics/responses.ts +151 -0
  116. package/src/semantics/shared.ts +82 -0
  117. package/src/semantics/uploads.ts +92 -0
  118. package/src/semantics/vector-stores.ts +279 -0
  119. package/test-fixtures/openai-openapi-operations.SOURCE.md +4 -5
  120. package/test-fixtures/openai-openapi-operations.json +224 -1334
@@ -0,0 +1,1569 @@
1
+ // OpenAI capability manifest — the EXPECTED REAL-PRODUCT SURFACE (the target), authored
2
+ // top-down from what the OpenAI API actually does — NOT from what this twin has built. This is
3
+ // the honest denominator: most entries start as `todo` and coverage reads LOW until the twin
4
+ // truly reaches 100% of the API. `verify()` (required to count as done) is ground truth;
5
+ // `expected:'done'` only on capabilities we genuinely claim, so a broken one shows as a
6
+ // regression. Grow this toward the API's *full* surface every cycle.
7
+ //
8
+ // Every capability is done or todo. A twin is a deterministic, offline model of the vendor's API
9
+ // contract; the labeled stub completion and the deterministic pseudo-vector ARE the twin's
10
+ // answer (see `openai.chat.stub_labeled`, `openai.embeddings.deterministic`), not a shortfall
11
+ // from a "real" one. Every entry here is either done or todo.
12
+ import { mkdtempSync, rmSync } from 'node:fs';
13
+ import { tmpdir } from 'node:os';
14
+ import { join } from 'node:path';
15
+ import { checkCapabilities, verifyBoundary, isInfrastructureError, harnessError } from '@volter/world-tooling';
16
+ import { handleOpenAITwinRequest } from "./openai-twin.js";
17
+ import { audioFormat, imageFormat, placeholderPng, wavSeconds } from "./openai-media.js";
18
+ import { createOpenAITwinFetch } from "./openai-server.js";
19
+ import { buildSignedOpenAIWebhook, verifyOpenAIWebhook, OpenAIWebhookVerificationError, computeOpenAIWebhookSignature } from "./openai-webhooks.js";
20
+ /** The World instant the checks run at unless a step names one: 2026-01-01, when every model and API family the checks
21
+ * use is live on OpenAI's timeline (./openai-models.ts: dall-e until 2026-05-12, the Assistants API until 2026-08-26,
22
+ * fine-tuning by a new organization until 2026-05-07). */
23
+ const CHECKS_AT = '2026-01-01T00:00:00.000Z';
24
+ /** Run a sequence of real OpenAI requests against an isolated root; return all responses. */
25
+ async function withRoot(steps) {
26
+ const root = mkdtempSync(join(tmpdir(), 'openai-cap-'));
27
+ const h = (s) => handleOpenAITwinRequest({ method: s.m, path: s.p, body: s.b === undefined ? undefined : s.b instanceof FormData ? s.b : JSON.stringify(s.b), root, occurredAt: s.at ?? CHECKS_AT });
28
+ try {
29
+ return await verifyBoundary('openai.withRoot', () => steps(h));
30
+ }
31
+ finally {
32
+ rmSync(root, { recursive: true, force: true });
33
+ }
34
+ }
35
+ /** Collect the streaming SSE events for a request against an isolated root. */
36
+ function withStream(body, fn) {
37
+ return new Promise((resolve, reject) => {
38
+ const root = mkdtempSync(join(tmpdir(), 'openai-cap-'));
39
+ const events = [];
40
+ handleOpenAITwinRequest({ method: 'POST', path: bodyPath(body), body: JSON.stringify(body), root, occurredAt: CHECKS_AT, sseSink: (e) => events.push(e) })
41
+ .then((final) => resolve(fn(events, final)))
42
+ .catch((err) => { if (isInfrastructureError(err))
43
+ reject(harnessError('openai.withStream', err));
44
+ else
45
+ resolve(false); })
46
+ .finally(() => rmSync(root, { recursive: true, force: true }));
47
+ });
48
+ }
49
+ function bodyPath(body) {
50
+ return body && typeof body === 'object' && 'input' in body ? '/v1/responses' : '/v1/chat/completions';
51
+ }
52
+ async function withRootH(steps) {
53
+ const root = mkdtempSync(join(tmpdir(), 'openai-cap-'));
54
+ const h = (s) => handleOpenAITwinRequest({ method: s.m, path: s.p, body: s.b === undefined ? undefined : JSON.stringify(s.b), root, occurredAt: CHECKS_AT, ...(s.headers ? { headers: s.headers } : {}) });
55
+ try {
56
+ return await verifyBoundary('openai.withRootH', () => steps(h));
57
+ }
58
+ finally {
59
+ rmSync(root, { recursive: true, force: true });
60
+ }
61
+ }
62
+ const ok = (r) => r.status >= 200 && r.status < 300;
63
+ const id = (r) => r.body?.id;
64
+ const field = (r, k) => r.body?.[k];
65
+ /** A response's text, as the SDKs' `output_text` aggregates it from its message items (the wire carries no `output_text`). */
66
+ const outputText = (r) => (r.body?.output ?? []).filter((i) => i.type === 'message').flatMap((i) => (i.content ?? [])).filter((c) => c.type === 'output_text').map((c) => String(c.text)).join('');
67
+ /** A multipart upload: text fields and one file part of the given bytes. */
68
+ function form(fields, name, bytes, filename, type) {
69
+ const f = new FormData();
70
+ for (const [k, v] of Object.entries(fields))
71
+ f.append(k, v);
72
+ f.append(name, new File([bytes], filename, { type }));
73
+ return f;
74
+ }
75
+ /** A tenth of a second of 8 kHz 8-bit mono silence as a WAV file. */
76
+ const SILENT_WAV = (() => {
77
+ const data = 800;
78
+ const b = new Uint8Array(44 + data).fill(0x80);
79
+ const v = new DataView(b.buffer);
80
+ b.set([...'RIFF'].map((c) => c.charCodeAt(0)), 0);
81
+ v.setUint32(4, 36 + data, true);
82
+ b.set([...'WAVEfmt '].map((c) => c.charCodeAt(0)), 8);
83
+ v.setUint32(16, 16, true);
84
+ v.setUint16(20, 1, true);
85
+ v.setUint16(22, 1, true);
86
+ v.setUint32(24, 8000, true);
87
+ v.setUint32(28, 8000, true);
88
+ v.setUint16(32, 1, true);
89
+ v.setUint16(34, 8, true);
90
+ b.set([...'data'].map((c) => c.charCodeAt(0)), 36);
91
+ v.setUint32(40, data, true);
92
+ return b;
93
+ })();
94
+ // ── shorthands (mirror the stripe/anthropic manifests) ──
95
+ const done = (id, area, title, dimension, tier, verify) => ({ id, area, title, dimension, tier, expected: 'done', verify });
96
+ const todo = (id, area, title, dimension, tier) => ({ id, area, title, dimension, tier, expected: 'todo' });
97
+ const CHAT = (extra = {}) => ({ model: 'gpt-4o', messages: [{ role: 'user', content: 'hello twin' }], ...extra });
98
+ export const OPENAI_CAPABILITIES = [
99
+ // ── THE HONEST CARVE-OUTS ─────────────────────────────────────────────────────────────
100
+ todo('openai.images.served_bytes', 'images', 'Image URLs resolve: the twin serves deterministic placeholder image bytes at the URLs it returns (today they point at the dead twin.invalid host)', 'api', 'common'),
101
+ todo('openai.files.binary_content', 'files', 'Files: persist and return arbitrary binary content, not only supplied text (a blob store, as the S3 twin already does)', 'api', 'niche'),
102
+ // ── Chat Completions (the protocol envelope — faithful) ────────────────────────────────
103
+ done('openai.chat.create', 'chat', 'Chat: create → faithful envelope (id/object/created/model/choices/usage)', 'api', 'core', () => withRoot(async (h) => {
104
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
105
+ if (!ok(r))
106
+ return false;
107
+ const b = r.body;
108
+ if (b.object !== 'chat.completion' || b.model !== 'gpt-4o' || typeof b.created !== 'number')
109
+ return false;
110
+ if (!String(b.id).startsWith('chatcmpl-'))
111
+ return false;
112
+ const c = b.choices?.[0];
113
+ if (!c || c.message?.role !== 'assistant' || typeof c.message?.content !== 'string' || c.finish_reason !== 'stop')
114
+ return false;
115
+ const u = b.usage;
116
+ return typeof u.prompt_tokens === 'number' && typeof u.completion_tokens === 'number' && u.total_tokens === u.prompt_tokens + u.completion_tokens;
117
+ })),
118
+ done('openai.chat.stub_labeled', 'chat', 'Stub completion is clearly labeled as a twin stub (not real output)', 'api', 'core', () => withRoot(async (h) => {
119
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
120
+ const text = r.body.choices?.[0]?.message?.content;
121
+ return ok(r) && text.includes('[twin-stub') && text.includes('hello twin');
122
+ })),
123
+ done('openai.chat.validation', 'chat', 'Chat validation (model required, non-empty messages, valid roles)', 'api', 'core', () => withRoot(async (h) => {
124
+ const noModel = await h({ m: 'POST', p: '/v1/chat/completions', b: { messages: [{ role: 'user', content: 'x' }] } });
125
+ const noMsg = await h({ m: 'POST', p: '/v1/chat/completions', b: { model: 'gpt-4o' } });
126
+ const empty = await h({ m: 'POST', p: '/v1/chat/completions', b: { model: 'gpt-4o', messages: [] } });
127
+ return noModel.status === 400 && noMsg.status === 400 && empty.status === 400 && noModel.body.error?.type === 'invalid_request_error';
128
+ })),
129
+ done('openai.chat.n_choices', 'chat', 'Chat: n returns multiple choices (indexed)', 'api', 'common', () => withRoot(async (h) => {
130
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ n: 3 }) });
131
+ const choices = r.body.choices;
132
+ return ok(r) && choices.length === 3 && choices[0].index === 0 && choices[2].index === 2;
133
+ })),
134
+ done('openai.chat.max_tokens', 'chat', 'Chat: max_tokens/max_completion_tokens caps output (finish_reason length)', 'api', 'common', () => withRoot(async (h) => {
135
+ const a = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ max_tokens: 1, messages: [{ role: 'user', content: 'please produce a long answer that exceeds one token' }] }) });
136
+ const b = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ max_completion_tokens: 1, messages: [{ role: 'user', content: 'please produce a long answer that exceeds one token' }] }) });
137
+ return ok(a) && ok(b) && a.body.choices[0].finish_reason === 'length' && b.body.choices[0].finish_reason === 'length';
138
+ })),
139
+ done('openai.chat.system_message', 'chat', 'Chat: system/developer messages count toward prompt_tokens', 'api', 'common', () => withRoot(async (h) => {
140
+ const without = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
141
+ const withSys = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ messages: [{ role: 'system', content: 'You are a careful, verbose assistant.' }, { role: 'user', content: 'hello twin' }] }) });
142
+ return ok(without) && ok(withSys) && withSys.body.usage.prompt_tokens > without.body.usage.prompt_tokens;
143
+ })),
144
+ done('openai.chat.multi_turn', 'chat', 'Chat: multi-turn user/assistant history accepted', 'api', 'common', () => withRoot(async (h) => {
145
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ messages: [{ role: 'user', content: 'first' }, { role: 'assistant', content: 'reply' }, { role: 'user', content: 'second' }] }) });
146
+ return ok(r) && r.body.object === 'chat.completion';
147
+ })),
148
+ done('openai.chat.content_parts', 'chat', 'Chat: content-part array input (text/image_url) accepted', 'api', 'common', () => withRoot(async (h) => {
149
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ messages: [{ role: 'user', content: [{ type: 'text', text: 'describe' }, { type: 'image_url', image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' } }] }] }) });
150
+ return ok(r) && r.body.usage.prompt_tokens > 0;
151
+ })),
152
+ done('openai.chat.stop', 'chat', 'Chat: stop sequence truncates the stub text', 'api', 'common', () => withRoot(async (h) => {
153
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ stop: ['Echoing'] }) });
154
+ const text = r.body.choices[0].message.content;
155
+ return ok(r) && !text.includes('Echoing');
156
+ })),
157
+ done('openai.chat.temperature_accepted', 'chat', 'Chat: temperature/top_p accepted (ignored for the stub)', 'api', 'niche', () => withRoot(async (h) => {
158
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ temperature: 0.7, top_p: 0.9 }) });
159
+ return ok(r) && r.body.object === 'chat.completion';
160
+ })),
161
+ done('openai.chat.deterministic_usage', 'chat', 'Chat: usage token counts deterministic for a fixed request', 'api', 'common', () => withRoot(async (h) => {
162
+ const a = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
163
+ const b = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
164
+ if (!ok(a) || !ok(b))
165
+ return false;
166
+ const ua = a.body.usage;
167
+ const ub = b.body.usage;
168
+ // usage must be a REAL populated token-count object, not just "the same as each other" (undefined === undefined is trivially true).
169
+ if (!ua || typeof ua.prompt_tokens !== 'number' || ua.prompt_tokens <= 0)
170
+ return false;
171
+ if (typeof ua.completion_tokens !== 'number' || ua.completion_tokens <= 0)
172
+ return false;
173
+ if (ua.total_tokens !== ua.prompt_tokens + ua.completion_tokens)
174
+ return false;
175
+ if (JSON.stringify(ua) !== JSON.stringify(ub))
176
+ return false;
177
+ // the id must be a real, well-formed, content-derived id (deterministic hash), not merely equal-because-both-undefined.
178
+ const ida = id(a);
179
+ const idb = id(b);
180
+ return typeof ida === 'string' && ida.startsWith('chatcmpl-') && ida === idb;
181
+ })),
182
+ done('openai.chat.system_fingerprint', 'chat', 'Chat: response carries system_fingerprint', 'api', 'niche', () => withRoot(async (h) => {
183
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
184
+ return ok(r) && typeof r.body.system_fingerprint === 'string';
185
+ })),
186
+ done('openai.chat.json_mode', 'chat', 'response_format json_object / json_schema → valid JSON content', 'api', 'common', () => withRoot(async (h) => {
187
+ const obj = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ response_format: { type: 'json_object' } }) });
188
+ if (!ok(obj))
189
+ return false;
190
+ const objText = obj.body.choices[0].message.content;
191
+ let parsedObj;
192
+ try {
193
+ parsedObj = JSON.parse(objText);
194
+ }
195
+ catch (err) {
196
+ if (isInfrastructureError(err))
197
+ throw harnessError('openai.chat.json_mode', err);
198
+ return false;
199
+ }
200
+ if (typeof parsedObj !== 'object')
201
+ return false;
202
+ const schema = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ response_format: { type: 'json_schema', json_schema: { name: 'r', schema: { type: 'object', properties: { title: { type: 'string' }, count: { type: 'integer' }, ok: { type: 'boolean' } } } } } }) });
203
+ const schemaText = schema.body.choices[0].message.content;
204
+ let parsed;
205
+ try {
206
+ parsed = JSON.parse(schemaText);
207
+ }
208
+ catch (err) {
209
+ if (isInfrastructureError(err))
210
+ throw harnessError('openai.chat.json_mode', err);
211
+ return false;
212
+ }
213
+ // every declared property is present + type-appropriate
214
+ return 'title' in parsed && typeof parsed.title === 'string' && parsed.count === 0 && parsed.ok === false;
215
+ })),
216
+ done('openai.chat.logprobs', 'chat', 'logprobs + top_logprobs in choices (faithful per-token shape)', 'api', 'niche', () => withRoot(async (h) => {
217
+ // no logprobs param → choices[].logprobs is null
218
+ const off = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
219
+ if (!ok(off) || off.body.choices[0].logprobs !== null)
220
+ return false;
221
+ // logprobs:true → content[] with token/logprob/bytes; re-joining tokens reconstructs the text
222
+ const on = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ logprobs: true }) });
223
+ const lp = on.body.choices[0].logprobs;
224
+ if (!lp || !Array.isArray(lp.content) || lp.content.length === 0)
225
+ return false;
226
+ const tok = lp.content[0];
227
+ if (typeof tok.token !== 'string' || typeof tok.logprob !== 'number' || tok.logprob > 0 || !Array.isArray(tok.bytes))
228
+ return false;
229
+ const joined = lp.content.map((t) => t.token).join('');
230
+ if (joined !== on.body.choices[0].message.content)
231
+ return false;
232
+ // top_logprobs:2 → each token carries up to 2 alternatives
233
+ const top = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ logprobs: true, top_logprobs: 2 }) });
234
+ const t0 = top.body.choices[0].logprobs.content[0];
235
+ if (t0.top_logprobs.length !== 2)
236
+ return false;
237
+ // top_logprobs without logprobs:true → 400
238
+ const bad = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ top_logprobs: 2 }) });
239
+ const oob = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ logprobs: true, top_logprobs: 99 }) });
240
+ return bad.status === 400 && oob.status === 400;
241
+ })),
242
+ done('openai.chat.seed', 'chat', 'seed accepted + reflected (distinct seed → distinct id; same seed deterministic)', 'api', 'niche', () => withRoot(async (h) => {
243
+ const a = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ seed: 42 }) });
244
+ const a2 = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ seed: 42 }) });
245
+ const b = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ seed: 7 }) });
246
+ if (!ok(a) || !ok(b))
247
+ return false;
248
+ // same seed → identical id (deterministic); different seed → different id
249
+ if (id(a) !== id(a2) || id(a) === id(b))
250
+ return false;
251
+ const bad = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ seed: 1.5 }) });
252
+ return bad.status === 400 && bad.body.error?.param === 'seed';
253
+ })),
254
+ done('openai.chat.logit_bias', 'chat', 'logit_bias accepted + validated (object of [-100,100] numbers)', 'api', 'niche', () => withRoot(async (h) => {
255
+ const ok1 = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ logit_bias: { '50256': -100, '1734': 25 } }) });
256
+ if (!ok(ok1) || ok1.body.object !== 'chat.completion')
257
+ return false;
258
+ const notObj = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ logit_bias: [1, 2] }) });
259
+ const oob = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ logit_bias: { '50256': 500 } }) });
260
+ return notObj.status === 400 && oob.status === 400 && oob.body.error?.param === 'logit_bias';
261
+ })),
262
+ done('openai.chat.audio', 'chat', 'Audio input/output modalities (input_audio parts + audio output envelope; bytes are a labeled stub)', 'api', 'niche', () => withRoot(async (h) => {
263
+ // INPUT: an input_audio content part is accepted + counts toward prompt_tokens.
264
+ const withAudioIn = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ messages: [{ role: 'user', content: [{ type: 'input_audio', input_audio: { data: 'BASE64AUDIO', format: 'wav' } }] }] }) });
265
+ if (!ok(withAudioIn) || withAudioIn.body.usage.prompt_tokens <= 0)
266
+ return false;
267
+ // OUTPUT: modalities:['text','audio'] + audio:{voice,format} → message.audio envelope.
268
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ modalities: ['text', 'audio'], audio: { voice: 'alloy', format: 'mp3' } }) });
269
+ if (!ok(r))
270
+ return false;
271
+ const msg = r.body.choices[0].message;
272
+ if (msg.content !== null || !msg.audio)
273
+ return false;
274
+ const a = msg.audio;
275
+ if (typeof a.id !== 'string' || typeof a.data !== 'string' || typeof a.transcript !== 'string' || typeof a.expires_at !== 'number')
276
+ return false;
277
+ // the bytes are a LABELED stub (decode the base64 and assert the twin-stub marker + no real synthesis claim)
278
+ const decoded = Buffer.from(a.data, 'base64').toString('utf8');
279
+ if (!decoded.includes('[twin-stub') || !decoded.includes('no real audio synthesis'))
280
+ return false;
281
+ if (!a.transcript.includes('[twin-stub') || !a.transcript.includes('hello twin'))
282
+ return false;
283
+ // modalities includes 'audio' but audio missing → 400
284
+ const bad = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ modalities: ['text', 'audio'] }) });
285
+ return bad.status === 400 && bad.body.error?.param === 'audio';
286
+ })),
287
+ done('openai.chat.prediction', 'chat', 'Predicted outputs (prediction param → echoed + accepted_prediction_tokens)', 'api', 'niche', () => withRoot(async (h) => {
288
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ prediction: { type: 'content', content: 'the predicted text body' } }) });
289
+ if (!ok(r))
290
+ return false;
291
+ const b = r.body;
292
+ const text = b.choices[0].message.content;
293
+ if (!text.includes('the predicted text body') || !text.includes('[twin-stub'))
294
+ return false;
295
+ const details = b.usage.completion_tokens_details;
296
+ if (!details || details.accepted_prediction_tokens <= 0 || details.rejected_prediction_tokens !== 0)
297
+ return false;
298
+ const bad = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ prediction: { type: 'content' } }) });
299
+ return bad.status === 400 && bad.body.error?.param === 'prediction';
300
+ })),
301
+ done('openai.chat.store_metadata', 'chat', 'store + metadata → stored completion retrieve / list / messages / delete', 'api', 'niche', () => withRoot(async (h) => {
302
+ // store:false (default) → not retrievable
303
+ const ns = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
304
+ const nsGet = await h({ m: 'GET', p: `/v1/chat/completions/${id(ns)}` });
305
+ if (nsGet.status !== 404)
306
+ return false;
307
+ // store:true + metadata → retrievable; metadata reflected
308
+ const c = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ store: true, metadata: { tag: 'unit' } }) });
309
+ if (!ok(c) || c.body.metadata?.tag !== 'unit')
310
+ return false;
311
+ const g = await h({ m: 'GET', p: `/v1/chat/completions/${id(c)}` });
312
+ if (!ok(g) || id(g) !== id(c) || g.body.object !== 'chat.completion')
313
+ return false;
314
+ const list = await h({ m: 'GET', p: '/v1/chat/completions' });
315
+ if (!ok(list) || list.body.object !== 'list' || list.body.data.length !== 1)
316
+ return false;
317
+ const msgs = await h({ m: 'GET', p: `/v1/chat/completions/${id(c)}/messages` });
318
+ if (!ok(msgs) || msgs.body.data.length < 1)
319
+ return false;
320
+ const del = await h({ m: 'DELETE', p: `/v1/chat/completions/${id(c)}` });
321
+ if (!ok(del) || del.body.deleted !== true)
322
+ return false;
323
+ const after = await h({ m: 'GET', p: `/v1/chat/completions/${id(c)}` });
324
+ return after.status === 404;
325
+ })),
326
+ done('openai.chat.stream_options_usage', 'chat', 'stream_options.include_usage emits a final usage-only chunk', 'api', 'common', () => withStream(CHAT({ stream: true, stream_options: { include_usage: true } }), (events, final) => {
327
+ const usageChunk = events.filter((e) => !e.done).find((e) => e.data.usage !== undefined && e.data.choices.length === 0);
328
+ if (!usageChunk)
329
+ return false;
330
+ const u = usageChunk.data.usage;
331
+ const fu = final.body.usage;
332
+ return u.total_tokens === fu.total_tokens && u.prompt_tokens === fu.prompt_tokens;
333
+ })),
334
+ // ── Streaming ─────────────────────────────────────────────────────────────────────────
335
+ done('openai.streaming.chunks', 'streaming', 'Streaming chat.completion.chunk sequence ends with [DONE]', 'api', 'core', () => withStream(CHAT({ stream: true }), (events) => {
336
+ const data = events.filter((e) => !e.done);
337
+ const hasRole = data.some((e) => e.data.choices[0]?.delta?.role === 'assistant');
338
+ const allChunks = data.every((e) => e.data.object === 'chat.completion.chunk');
339
+ const doneLast = events.length > 0 && events[events.length - 1].done === true;
340
+ return hasRole && allChunks && doneLast;
341
+ })),
342
+ done('openai.streaming.reconstruct', 'streaming', 'Streaming content deltas reconstruct the full message text', 'api', 'core', () => withStream(CHAT({ stream: true }), (events, final) => {
343
+ const text = events.filter((e) => !e.done).map((e) => e.data.choices[0]?.delta?.content ?? '').join('');
344
+ const full = final.body.choices[0].message.content;
345
+ return text === full && text.includes('[twin-stub');
346
+ })),
347
+ done('openai.streaming.finish_reason', 'streaming', 'Streaming final chunk carries finish_reason', 'api', 'common', () => withStream(CHAT({ stream: true }), (events) => {
348
+ const withFinish = events.filter((e) => !e.done).find((e) => e.data.choices[0]?.finish_reason === 'stop');
349
+ return !!withFinish;
350
+ })),
351
+ done('openai.streaming.tool_calls', 'streaming', 'Streaming a tool_call emits function name + arguments deltas', 'api', 'common', () => withStream(CHAT({ stream: true, tools: [{ type: 'function', function: { name: 'lookup', parameters: { type: 'object' } } }] }), (events) => {
352
+ const data = events.filter((e) => !e.done);
353
+ const nameDelta = data.find((e) => e.data.choices[0]?.delta?.tool_calls?.[0]?.function?.name === 'lookup');
354
+ const finish = data.find((e) => e.data.choices[0]?.finish_reason === 'tool_calls');
355
+ return !!nameDelta && !!finish;
356
+ })),
357
+ done('openai.streaming.usage_chunk', 'streaming', 'No usage chunk without include_usage; present + last (before [DONE]) with it', 'api', 'common', async () => {
358
+ const without = await withStream(CHAT({ stream: true }), (events) => !events.filter((e) => !e.done).some((e) => e.data.usage !== undefined));
359
+ const withU = await withStream(CHAT({ stream: true, stream_options: { include_usage: true } }), (events) => {
360
+ const idxUsage = events.findIndex((e) => !e.done && e.data.usage !== undefined);
361
+ const idxDone = events.findIndex((e) => e.done === true);
362
+ return idxUsage >= 0 && idxDone === events.length - 1 && idxUsage === idxDone - 1;
363
+ });
364
+ return without && withU;
365
+ }),
366
+ // ── Tool / function calling ───────────────────────────────────────────────────────────
367
+ done('openai.tools.tool_calls', 'tools', 'Tools provided → tool_calls + finish_reason tool_calls', 'api', 'core', () => withRoot(async (h) => {
368
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ tools: [{ type: 'function', function: { name: 'get_weather', parameters: { type: 'object', properties: { city: { type: 'string' } } } } }] }) });
369
+ if (!ok(r))
370
+ return false;
371
+ const c = r.body.choices[0];
372
+ const call = c.message?.tool_calls?.[0];
373
+ return c.finish_reason === 'tool_calls' && c.message.content === null && call?.type === 'function' && call.function.name === 'get_weather' && String(call.id).startsWith('call_');
374
+ })),
375
+ done('openai.tools.legacy_functions', 'tools', 'Legacy functions param → tool_call for the named function', 'api', 'common', () => withRoot(async (h) => {
376
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ functions: [{ name: 'legacy_fn', parameters: { type: 'object' } }] }) });
377
+ const call = r.body.choices[0]?.message?.tool_calls?.[0];
378
+ return ok(r) && call?.function?.name === 'legacy_fn';
379
+ })),
380
+ done('openai.tools.no_tools_text', 'tools', 'No tools → plain text content + finish_reason stop', 'api', 'common', () => withRoot(async (h) => {
381
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
382
+ const c = r.body.choices[0];
383
+ return ok(r) && typeof c.message.content === 'string' && c.finish_reason === 'stop' && !c.message.tool_calls;
384
+ })),
385
+ done('openai.tools.tool_choice', 'tools', 'tool_choice none→text / required+named→that tool / parallel_tool_calls', 'api', 'common', () => withRoot(async (h) => {
386
+ const tools = [
387
+ { type: 'function', function: { name: 'get_weather', parameters: { type: 'object', properties: { city: { type: 'string' } } } } },
388
+ { type: 'function', function: { name: 'get_time', parameters: { type: 'object', properties: { tz: { type: 'string' } } } } },
389
+ ];
390
+ // none → no tool_calls, plain text + finish_reason stop
391
+ const none = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ tools, tool_choice: 'none' }) });
392
+ const nc = none.body.choices[0];
393
+ if (!(nc.finish_reason === 'stop' && typeof nc.message.content === 'string' && !nc.message.tool_calls))
394
+ return false;
395
+ // named → exactly that tool
396
+ const named = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ tools, tool_choice: { type: 'function', function: { name: 'get_time' } } }) });
397
+ const calls = named.body.choices[0].message.tool_calls;
398
+ if (!(calls.length === 1 && calls[0].function.name === 'get_time'))
399
+ return false;
400
+ // default (auto) + parallel_tool_calls true → one call per provided tool
401
+ const parallel = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ tools }) });
402
+ const pcalls = parallel.body.choices[0].message.tool_calls;
403
+ if (pcalls.length !== 2)
404
+ return false;
405
+ // parallel_tool_calls false → collapse to a single call
406
+ const single = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ tools, parallel_tool_calls: false }) });
407
+ return single.body.choices[0].message.tool_calls.length === 1;
408
+ })),
409
+ done('openai.tools.strict', 'tools', 'Strict function schemas → arguments validate against the declared schema', 'api', 'common', () => withRoot(async (h) => {
410
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ tool_choice: { type: 'function', function: { name: 'set_user' } }, tools: [{ type: 'function', function: { name: 'set_user', strict: true, parameters: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' }, active: { type: 'boolean' }, role: { type: 'string', enum: ['admin', 'user'] } }, required: ['name', 'age', 'active', 'role'] } } }] }) });
411
+ const call = r.body.choices[0]?.message?.tool_calls?.[0];
412
+ if (!call)
413
+ return false;
414
+ let args;
415
+ try {
416
+ args = JSON.parse(call.function.arguments);
417
+ }
418
+ catch (err) {
419
+ if (isInfrastructureError(err))
420
+ throw harnessError('openai.tools.strict', err);
421
+ return false;
422
+ }
423
+ // every declared property is present with a schema-typed value (so a strict parse succeeds)
424
+ return typeof args.name === 'string' && args.age === 0 && args.active === false && args.role === 'admin';
425
+ })),
426
+ // ── Responses API ─────────────────────────────────────────────────────────────────────
427
+ done('openai.responses.create', 'responses', 'Responses: create → faithful envelope (id/object/output/usage)', 'api', 'core', () => withRoot(async (h) => {
428
+ const r = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'hello responses' } });
429
+ if (!ok(r))
430
+ return false;
431
+ const b = r.body;
432
+ if (b.object !== 'response' || b.status !== 'completed' || !String(b.id).startsWith('resp-'))
433
+ return false;
434
+ const item = b.output?.[0];
435
+ if (item?.type !== 'message' || item.content?.[0]?.type !== 'output_text')
436
+ return false;
437
+ return typeof b.usage.input_tokens === 'number' && typeof b.usage.output_tokens === 'number';
438
+ })),
439
+ done('openai.responses.stub_labeled', 'responses', 'Responses stub output is clearly labeled (not real output)', 'api', 'core', () => withRoot(async (h) => {
440
+ const r = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'echo me responses' } });
441
+ const text = outputText(r);
442
+ return ok(r) && text.includes('[twin-stub') && text.includes('echo me responses');
443
+ })),
444
+ done('openai.responses.input_items', 'responses', 'Responses accepts an array of input items (role/content)', 'api', 'common', () => withRoot(async (h) => {
445
+ const r = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: [{ role: 'user', content: [{ type: 'input_text', text: 'structured input' }] }] } });
446
+ return ok(r) && r.body.output?.[0]?.type === 'message';
447
+ })),
448
+ done('openai.responses.validation', 'responses', 'Responses validation (model + input required)', 'api', 'common', () => withRoot(async (h) => {
449
+ const noModel = await h({ m: 'POST', p: '/v1/responses', b: { input: 'x' } });
450
+ const noInput = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o' } });
451
+ return noModel.status === 400 && noInput.status === 400;
452
+ })),
453
+ done('openai.responses.streaming', 'responses', 'Responses streaming events (created → output_text.delta → completed, no sentinel)', 'api', 'common', () => withStream({ model: 'gpt-4o', input: 'stream responses', stream: true }, (events) => {
454
+ const types = events.filter((e) => !e.done).map((e) => e.data.type);
455
+ const completedLast = types[types.length - 1] === 'response.completed' && !events.some((e) => e.done);
456
+ return types.includes('response.created') && types.includes('response.output_text.delta') && completedLast;
457
+ })),
458
+ done('openai.responses.retrieve', 'responses', 'Responses: stored by default, retrieve + delete by id (+ 404)', 'api', 'common', () => withRoot(async (h) => {
459
+ const c = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'store me' } });
460
+ if (!ok(c))
461
+ return false;
462
+ const g = await h({ m: 'GET', p: `/v1/responses/${id(c)}` });
463
+ if (!ok(g) || id(g) !== id(c) || g.body.object !== 'response' || outputText(g) !== outputText(c))
464
+ return false;
465
+ const del = await h({ m: 'DELETE', p: `/v1/responses/${id(c)}` });
466
+ if (!ok(del) || del.body.deleted !== true)
467
+ return false;
468
+ const after = await h({ m: 'GET', p: `/v1/responses/${id(c)}` });
469
+ const missing = await h({ m: 'GET', p: '/v1/responses/resp-nope' });
470
+ // store:false must NOT be retrievable
471
+ const ns = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'ephemeral', store: false } });
472
+ const nsGet = await h({ m: 'GET', p: `/v1/responses/${id(ns)}` });
473
+ return after.status === 404 && missing.status === 404 && nsGet.status === 404;
474
+ })),
475
+ todo('openai.responses.tools', 'responses', 'Responses built-in tools (web_search/file_search/computer_use/code_interpreter): the tool-call envelope with a deterministic, labeled twin-stub result, exactly as chat stubs generation (user-defined function tools are already modeled via openai.tools.*)', 'api', 'niche'),
476
+ done('openai.responses.reasoning', 'responses', 'Responses reasoning items + reasoning.effort (faithful reasoning item shape; labeled stub summary)', 'api', 'niche', () => withRoot(async (h) => {
477
+ // a model that does not reason → no reasoning item, no reasoning_tokens
478
+ const plain = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'think about this' } });
479
+ if (!ok(plain))
480
+ return false;
481
+ if (plain.body.output.some((o) => o.type === 'reasoning'))
482
+ return false;
483
+ if (plain.body.usage.output_tokens_details?.reasoning_tokens !== 0)
484
+ return false;
485
+ // reasoning.effort:'high' with a summary asked for → a reasoning item (labeled stub summary) BEFORE the message item + reasoning_tokens
486
+ const r = await h({ m: 'POST', p: '/v1/responses', b: { model: 'o4-mini', input: 'think about this', reasoning: { effort: 'high', summary: 'auto' } } });
487
+ if (!ok(r))
488
+ return false;
489
+ const b = r.body;
490
+ const items = b.output;
491
+ const reasoning = items.find((o) => o.type === 'reasoning');
492
+ const message = items.find((o) => o.type === 'message');
493
+ if (!reasoning || !message)
494
+ return false;
495
+ // reasoning item is first; faithful shape (id/summary[]) with a labeled-stub summary
496
+ if (items[0].type !== 'reasoning' || !String(reasoning.id).startsWith('rs-twin-'))
497
+ return false;
498
+ const summary = reasoning.summary?.[0];
499
+ if (summary?.type !== 'summary_text' || !String(summary.text).includes('[twin-stub') || !String(summary.text).includes('effort=high'))
500
+ return false;
501
+ // usage.output_tokens_details.reasoning_tokens > 0 and folded into output_tokens
502
+ const rt = b.usage.output_tokens_details?.reasoning_tokens;
503
+ if (typeof rt !== 'number' || rt <= 0)
504
+ return false;
505
+ if (b.reasoning?.effort !== 'high')
506
+ return false;
507
+ // higher effort → more reasoning tokens (deterministic budget)
508
+ const low = await h({ m: 'POST', p: '/v1/responses', b: { model: 'o4-mini', input: 'think about this', reasoning: { effort: 'low' } } });
509
+ if (low.body.usage.output_tokens_details.reasoning_tokens >= rt)
510
+ return false;
511
+ // invalid effort → 400
512
+ const bad = await h({ m: 'POST', p: '/v1/responses', b: { model: 'o4-mini', input: 'x', reasoning: { effort: 'turbo' } } });
513
+ return bad.status === 400 && bad.body.error?.param === 'reasoning.effort';
514
+ })),
515
+ done('openai.responses.previous_response', 'responses', 'previous_response_id chaining (server-side state; 404 unknown)', 'api', 'common', () => withRoot(async (h) => {
516
+ const first = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'first turn' } });
517
+ if (!ok(first))
518
+ return false;
519
+ const second = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'second turn', previous_response_id: id(first) } });
520
+ if (!ok(second) || second.body.previous_response_id !== id(first))
521
+ return false;
522
+ // chaining folds the prior turn into the prompt → larger input_tokens than the same turn unchained
523
+ const unchained = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'second turn' } });
524
+ if (!(second.body.usage.input_tokens > unchained.body.usage.input_tokens))
525
+ return false;
526
+ const bad = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'x', previous_response_id: 'resp-nope' } });
527
+ return bad.status === 404;
528
+ })),
529
+ done('openai.responses.input_items_list', 'responses', 'List the input items of a stored response', 'api', 'niche', () => withRoot(async (h) => {
530
+ const c = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'an input turn' } });
531
+ const items = await h({ m: 'GET', p: `/v1/responses/${id(c)}/input_items` });
532
+ const data = items.body.data;
533
+ const missing = await h({ m: 'GET', p: '/v1/responses/resp-nope/input_items' });
534
+ return ok(items) && items.body.object === 'list' && data.length >= 1 && data[0].type === 'message' && missing.status === 404;
535
+ })),
536
+ // ── Embeddings ────────────────────────────────────────────────────────────────────────
537
+ done('openai.embeddings.create', 'embeddings', 'Embeddings: faithful list envelope + per-input embedding objects', 'api', 'core', () => withRoot(async (h) => {
538
+ const r = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'embed me' } });
539
+ if (!ok(r))
540
+ return false;
541
+ const b = r.body;
542
+ const e = b.data?.[0];
543
+ return b.object === 'list' && e?.object === 'embedding' && e.index === 0 && Array.isArray(e.embedding) && e.embedding.length === 1536 && typeof b.usage.prompt_tokens === 'number';
544
+ })),
545
+ done('openai.embeddings.deterministic', 'embeddings', 'Embeddings: same input → identical pseudo-vector', 'api', 'core', () => withRoot(async (h) => {
546
+ const a = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'same' } });
547
+ const b = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'same' } });
548
+ const va = a.body.data[0].embedding, vb = b.body.data[0].embedding;
549
+ const vc = (await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'different' } })).body;
550
+ return JSON.stringify(va) === JSON.stringify(vb) && JSON.stringify(va) !== JSON.stringify(vc.data[0].embedding);
551
+ })),
552
+ done('openai.embeddings.batch', 'embeddings', 'Embeddings: array input → one indexed embedding per item', 'api', 'common', () => withRoot(async (h) => {
553
+ const r = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: ['one', 'two', 'three'] } });
554
+ const data = r.body.data;
555
+ return ok(r) && data.length === 3 && data[2].index === 2;
556
+ })),
557
+ done('openai.embeddings.dimensions', 'embeddings', 'Embeddings: dimensions param controls vector length; large default 3072', 'api', 'common', () => withRoot(async (h) => {
558
+ const custom = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'x', dimensions: 256 } });
559
+ const large = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-large', input: 'x' } });
560
+ return custom.body.data[0].embedding.length === 256 && large.body.data[0].embedding.length === 3072;
561
+ })),
562
+ done('openai.embeddings.normalized', 'embeddings', 'Embeddings: pseudo-vectors are L2-normalized (unit length)', 'api', 'niche', () => withRoot(async (h) => {
563
+ const r = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'normalize', dimensions: 64 } });
564
+ const v = r.body.data[0].embedding;
565
+ const norm = Math.sqrt(v.reduce((s, x) => s + x * x, 0));
566
+ return Math.abs(norm - 1) < 1e-6;
567
+ })),
568
+ done('openai.embeddings.validation', 'embeddings', 'Embeddings: model + input required (400 otherwise)', 'api', 'common', () => withRoot(async (h) => {
569
+ const noModel = await h({ m: 'POST', p: '/v1/embeddings', b: { input: 'x' } });
570
+ const noInput = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small' } });
571
+ return noModel.status === 400 && noInput.status === 400;
572
+ })),
573
+ done('openai.embeddings.base64', 'embeddings', 'encoding_format base64 → base64 Float32 string decoding to the float vector', 'api', 'niche', () => withRoot(async (h) => {
574
+ const flt = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'b64', dimensions: 8 } });
575
+ const b64 = await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'b64', dimensions: 8, encoding_format: 'base64' } });
576
+ if (!ok(flt) || !ok(b64))
577
+ return false;
578
+ const floats = flt.body.data[0].embedding;
579
+ const encoded = b64.body.data[0].embedding;
580
+ if (typeof encoded !== 'string')
581
+ return false; // base64 returns a STRING, not an array
582
+ // decode the base64 little-endian Float32 buffer and compare to the float vector
583
+ const bin = atob(encoded);
584
+ const buf = new ArrayBuffer(bin.length);
585
+ const bytes = new Uint8Array(buf);
586
+ for (let i = 0; i < bin.length; i++)
587
+ bytes[i] = bin.charCodeAt(i);
588
+ const view = new DataView(buf);
589
+ if (bytes.length !== floats.length * 4)
590
+ return false;
591
+ for (let i = 0; i < floats.length; i++) {
592
+ if (Math.abs(view.getFloat32(i * 4, true) - floats[i]) > 1e-5)
593
+ return false;
594
+ }
595
+ return true;
596
+ })),
597
+ // ── Models ────────────────────────────────────────────────────────────────────────────
598
+ done('openai.models.list', 'models', 'Models: list (object:list, data array of model objects)', 'api', 'core', () => withRoot(async (h) => {
599
+ const r = await h({ m: 'GET', p: '/v1/models' });
600
+ const b = r.body;
601
+ return ok(r) && b.object === 'list' && Array.isArray(b.data) && b.data.length > 0 && b.data[0].object === 'model';
602
+ })),
603
+ done('openai.models.retrieve', 'models', 'Models: retrieve by id (+ 404 model_not_found)', 'api', 'core', () => withRoot(async (h) => {
604
+ const r = await h({ m: 'GET', p: '/v1/models/gpt-4o' });
605
+ if (!ok(r) || r.body.object !== 'model' || r.body.id !== 'gpt-4o')
606
+ return false;
607
+ const missing = await h({ m: 'GET', p: '/v1/models/nope-9' });
608
+ return missing.status === 404 && missing.body.error?.code === 'model_not_found';
609
+ })),
610
+ done('openai.models.delete', 'models', 'Delete a fine-tuned model (job output); base models refuse; 404 unknown', 'api', 'niche', () => withRoot(async (h) => {
611
+ const created = await h({ m: 'POST', p: '/v1/fine_tuning/jobs', b: { model: 'gpt-4o-mini', training_file: 'file-train' } });
612
+ // the model exists once the job has trained: a read observes that
613
+ const job = await h({ m: 'GET', p: `/v1/fine_tuning/jobs/${id(created)}` });
614
+ const ftModel = job.body.fine_tuned_model;
615
+ // the minted fine-tuned model is retrievable + listed
616
+ const ret = await h({ m: 'GET', p: `/v1/models/${encodeURIComponent(ftModel)}` });
617
+ if (!ok(ret) || ret.body.id !== ftModel)
618
+ return false;
619
+ const listed = (await h({ m: 'GET', p: '/v1/models' })).body.data;
620
+ if (!listed.some((m) => m.id === ftModel))
621
+ return false;
622
+ // delete it → deleted:true, then gone from retrieve + list
623
+ const del = await h({ m: 'DELETE', p: `/v1/models/${encodeURIComponent(ftModel)}` });
624
+ if (!ok(del) || del.body.deleted !== true)
625
+ return false;
626
+ const after = await h({ m: 'GET', p: `/v1/models/${encodeURIComponent(ftModel)}` });
627
+ if (after.status !== 404)
628
+ return false;
629
+ // base catalog model cannot be deleted; unknown → 404
630
+ const base = await h({ m: 'DELETE', p: '/v1/models/gpt-4o' });
631
+ const missing = await h({ m: 'DELETE', p: '/v1/models/ft:nope' });
632
+ return base.status === 400 && missing.status === 404;
633
+ })),
634
+ // ── Moderations ───────────────────────────────────────────────────────────────────────
635
+ done('openai.moderations.create', 'moderations', 'Moderations: faithful results shape (categories/scores/flagged)', 'api', 'core', () => withRoot(async (h) => {
636
+ const r = await h({ m: 'POST', p: '/v1/moderations', b: { input: 'a harmless sentence' } });
637
+ if (!ok(r))
638
+ return false;
639
+ const res = r.body.results?.[0];
640
+ return String(r.body.id).startsWith('modr-') && res && res.flagged === false && typeof res.categories === 'object' && typeof res.category_scores === 'object';
641
+ })),
642
+ done('openai.moderations.flagging', 'moderations', 'Moderations: deterministic flagging on the keyword heuristic', 'api', 'common', () => withRoot(async (h) => {
643
+ const r = await h({ m: 'POST', p: '/v1/moderations', b: { input: 'I will kill the process' } });
644
+ const res = r.body.results?.[0];
645
+ return ok(r) && res.flagged === true && res.categories.violence === true;
646
+ })),
647
+ done('openai.moderations.batch', 'moderations', 'Moderations: array input → one result per item', 'api', 'common', () => withRoot(async (h) => {
648
+ const r = await h({ m: 'POST', p: '/v1/moderations', b: { input: ['safe', 'hateful slur'] } });
649
+ const results = r.body.results;
650
+ return ok(r) && results.length === 2 && results[0].flagged === false && results[1].flagged === true;
651
+ })),
652
+ // ── Files (stateful) ──────────────────────────────────────────────────────────────────
653
+ done('openai.files.upload', 'files', 'Files: upload (stateful) with purpose/filename/bytes/status', 'api', 'core', () => withRoot(async (h) => {
654
+ const r = await h({ m: 'POST', p: '/v1/files', b: { purpose: 'fine-tune', filename: 'train.jsonl', content: '{"x":1}' } });
655
+ if (!ok(r) || r.body.object !== 'file' || !String(id(r)).startsWith('file-'))
656
+ return false;
657
+ const b = r.body;
658
+ return b.purpose === 'fine-tune' && b.filename === 'train.jsonl' && b.status === 'processed' && b.bytes > 0;
659
+ })),
660
+ done('openai.files.retrieve', 'files', 'Files: retrieve by id (+ 404 unknown)', 'api', 'core', () => withRoot(async (h) => {
661
+ const c = await h({ m: 'POST', p: '/v1/files', b: { purpose: 'batch', filename: 'a.jsonl', content: '{}' } });
662
+ const g = await h({ m: 'GET', p: `/v1/files/${id(c)}` });
663
+ const missing = await h({ m: 'GET', p: '/v1/files/file-nope' });
664
+ return ok(g) && id(g) === id(c) && missing.status === 404;
665
+ })),
666
+ done('openai.files.list', 'files', 'Files: list (object:list, newest first)', 'api', 'core', () => withRoot(async (h) => {
667
+ await h({ m: 'POST', p: '/v1/files', b: { purpose: 'batch', filename: 'a.jsonl', content: '{}' } });
668
+ await h({ m: 'POST', p: '/v1/files', b: { purpose: 'batch', filename: 'b.jsonl', content: '{}' } });
669
+ const l = await h({ m: 'GET', p: '/v1/files' });
670
+ return ok(l) && l.body.object === 'list' && l.body.data.length === 2;
671
+ })),
672
+ done('openai.files.content', 'files', 'Files: download content of an uploaded file', 'api', 'common', () => withRoot(async (h) => {
673
+ const c = await h({ m: 'POST', p: '/v1/files', b: { purpose: 'batch', filename: 'a.jsonl', content: 'hello-bytes' } });
674
+ const content = await h({ m: 'GET', p: `/v1/files/${id(c)}/content` });
675
+ return ok(content) && content.body === 'hello-bytes';
676
+ })),
677
+ done('openai.files.delete', 'files', 'Files: delete returns the deleted stub + removes from list', 'api', 'common', () => withRoot(async (h) => {
678
+ const c = await h({ m: 'POST', p: '/v1/files', b: { purpose: 'batch', filename: 'a.jsonl', content: '{}' } });
679
+ const del = await h({ m: 'DELETE', p: `/v1/files/${id(c)}` });
680
+ if (!ok(del) || del.body.deleted !== true)
681
+ return false;
682
+ const l = await h({ m: 'GET', p: '/v1/files' });
683
+ return l.body.data.length === 0;
684
+ })),
685
+ done('openai.files.purpose_required', 'files', 'Files: purpose required (400 invalid_request_error)', 'api', 'common', () => withRoot(async (h) => {
686
+ const r = await h({ m: 'POST', p: '/v1/files', b: { filename: 'a.jsonl', content: '{}' } });
687
+ return r.status === 400 && r.body.error?.param === 'purpose';
688
+ })),
689
+ // ── Batches (stateful) ────────────────────────────────────────────────────────────────
690
+ done('openai.batches.create', 'batches', 'Batches: create (stateful) with endpoint/input_file_id/status', 'api', 'core', () => withRoot(async (h) => {
691
+ const r = await h({ m: 'POST', p: '/v1/batches', b: { input_file_id: 'file-x', endpoint: '/v1/chat/completions', completion_window: '24h' } });
692
+ if (!ok(r) || r.body.object !== 'batch' || !String(id(r)).startsWith('batch-'))
693
+ return false;
694
+ const b = r.body;
695
+ if (b.endpoint !== '/v1/chat/completions' || b.status !== 'validating' || b.output_file_id !== null)
696
+ return false;
697
+ // OpenAI works the batch on its own: a read finds it completed with its output file
698
+ const g = await h({ m: 'GET', p: `/v1/batches/${id(r)}` });
699
+ return field(g, 'status') === 'completed' && typeof field(g, 'output_file_id') === 'string';
700
+ })),
701
+ done('openai.batches.retrieve', 'batches', 'Batches: retrieve by id (+ 404 unknown)', 'api', 'core', () => withRoot(async (h) => {
702
+ const c = await h({ m: 'POST', p: '/v1/batches', b: { input_file_id: 'file-x', endpoint: '/v1/chat/completions', completion_window: '24h' } });
703
+ const g = await h({ m: 'GET', p: `/v1/batches/${id(c)}` });
704
+ const missing = await h({ m: 'GET', p: '/v1/batches/batch-nope' });
705
+ return ok(g) && id(g) === id(c) && missing.status === 404;
706
+ })),
707
+ done('openai.batches.list', 'batches', 'Batches: list (object:list)', 'api', 'core', () => withRoot(async (h) => {
708
+ await h({ m: 'POST', p: '/v1/batches', b: { input_file_id: 'file-x', endpoint: '/v1/chat/completions', completion_window: '24h' } });
709
+ const l = await h({ m: 'GET', p: '/v1/batches' });
710
+ return ok(l) && l.body.object === 'list' && l.body.data.length === 1;
711
+ })),
712
+ done('openai.batches.cancel', 'batches', 'Batches: cancel an in-flight batch (cancelling, then cancelled; 409 once done; 404 unknown)', 'api', 'common', () => withRoot(async (h) => {
713
+ const c = await h({ m: 'POST', p: '/v1/batches', b: { input_file_id: 'file-x', endpoint: '/v1/chat/completions', completion_window: '24h' } });
714
+ const cancel = await h({ m: 'POST', p: `/v1/batches/${id(c)}/cancel` });
715
+ if (!ok(cancel) || field(cancel, 'status') !== 'cancelling')
716
+ return false;
717
+ const read = await h({ m: 'GET', p: `/v1/batches/${id(c)}` });
718
+ if (field(read, 'status') !== 'cancelled')
719
+ return false;
720
+ const again = await h({ m: 'POST', p: `/v1/batches/${id(c)}/cancel` });
721
+ const missing = await h({ m: 'POST', p: '/v1/batches/batch-nope/cancel' });
722
+ return again.status === 409 && missing.status === 404;
723
+ })),
724
+ done('openai.batches.validation', 'batches', 'Batches: input_file_id/endpoint/completion_window required', 'api', 'common', () => withRoot(async (h) => {
725
+ const r = await h({ m: 'POST', p: '/v1/batches', b: { endpoint: '/v1/chat/completions', completion_window: '24h' } });
726
+ return r.status === 400 && r.body.error?.param === 'input_file_id';
727
+ })),
728
+ done('openai.batches.persistence', 'batches', 'Batches persist across requests (kernel-backed, not a side store)', 'api', 'common', () => withRoot(async (h) => {
729
+ const c = await h({ m: 'POST', p: '/v1/batches', b: { input_file_id: 'file-x', endpoint: '/v1/chat/completions', completion_window: '24h' } });
730
+ if (!ok(c))
731
+ return false;
732
+ const cid = id(c);
733
+ // a real, well-formed batch id + object/fields must exist — not merely "both sides undefined".
734
+ if (typeof cid !== 'string' || !cid.startsWith('batch-'))
735
+ return false;
736
+ const cb = c.body;
737
+ if (cb.object !== 'batch' || cb.input_file_id !== 'file-x' || cb.status !== 'validating')
738
+ return false;
739
+ const g1 = await h({ m: 'GET', p: `/v1/batches/${cid}` });
740
+ const g2 = await h({ m: 'GET', p: `/v1/batches/${cid}` });
741
+ // the GET must actually round-trip the persisted resource's real fields (kernel-backed) —
742
+ // not merely a matching-because-undefined id.
743
+ const same = (r) => {
744
+ const rb = r.body;
745
+ return rb.object === 'batch' && rb.input_file_id === cb.input_file_id && rb.endpoint === cb.endpoint;
746
+ };
747
+ // the first read observes the vendor's work; the second reads what it stored
748
+ return ok(g1) && ok(g2) && id(g1) === cid && id(g2) === cid && same(g1) && same(g2) &&
749
+ field(g1, 'status') === 'completed' && field(g2, 'status') === 'completed' && field(g2, 'output_file_id') === field(g1, 'output_file_id');
750
+ })),
751
+ // ── Fine-tuning (stateful) ────────────────────────────────────────────────────────────
752
+ done('openai.fine_tuning.create', 'fine_tuning', 'Fine-tuning: create job (stateful) with model/training_file/status', 'api', 'core', () => withRoot(async (h) => {
753
+ const r = await h({ m: 'POST', p: '/v1/fine_tuning/jobs', b: { model: 'gpt-4o-mini', training_file: 'file-train' } });
754
+ if (!ok(r) || r.body.object !== 'fine_tuning.job' || !String(id(r)).startsWith('ftjob-'))
755
+ return false;
756
+ const b = r.body;
757
+ if (b.model !== 'gpt-4o-mini' || b.status !== 'validating_files' || b.fine_tuned_model !== null)
758
+ return false;
759
+ // OpenAI trains on its own: a read finds the job succeeded with its model
760
+ const g = await h({ m: 'GET', p: `/v1/fine_tuning/jobs/${id(r)}` });
761
+ return field(g, 'status') === 'succeeded' && String(field(g, 'fine_tuned_model')).startsWith('ft:');
762
+ })),
763
+ done('openai.fine_tuning.retrieve', 'fine_tuning', 'Fine-tuning: retrieve + list jobs', 'api', 'common', () => withRoot(async (h) => {
764
+ const c = await h({ m: 'POST', p: '/v1/fine_tuning/jobs', b: { model: 'gpt-4o-mini', training_file: 'file-train' } });
765
+ const g = await h({ m: 'GET', p: `/v1/fine_tuning/jobs/${id(c)}` });
766
+ const l = await h({ m: 'GET', p: '/v1/fine_tuning/jobs' });
767
+ return ok(g) && id(g) === id(c) && l.body.data.length === 1;
768
+ })),
769
+ done('openai.fine_tuning.events', 'fine_tuning', 'Fine-tuning: job events list', 'api', 'common', () => withRoot(async (h) => {
770
+ const c = await h({ m: 'POST', p: '/v1/fine_tuning/jobs', b: { model: 'gpt-4o-mini', training_file: 'file-train' } });
771
+ const ev = await h({ m: 'GET', p: `/v1/fine_tuning/jobs/${id(c)}/events` });
772
+ const data = ev.body.data;
773
+ return ok(ev) && data.length > 0 && data[0].object === 'fine_tuning.job.event';
774
+ })),
775
+ done('openai.fine_tuning.cancel', 'fine_tuning', 'Fine-tuning: cancel job (status cancelled)', 'api', 'common', () => withRoot(async (h) => {
776
+ const c = await h({ m: 'POST', p: '/v1/fine_tuning/jobs', b: { model: 'gpt-4o-mini', training_file: 'file-train' } });
777
+ const cancel = await h({ m: 'POST', p: `/v1/fine_tuning/jobs/${id(c)}/cancel` });
778
+ return ok(cancel) && field(cancel, 'status') === 'cancelled';
779
+ })),
780
+ done('openai.fine_tuning.validation', 'fine_tuning', 'Fine-tuning: model + training_file required', 'api', 'common', () => withRoot(async (h) => {
781
+ const r = await h({ m: 'POST', p: '/v1/fine_tuning/jobs', b: { model: 'gpt-4o-mini' } });
782
+ return r.status === 400 && r.body.error?.param === 'training_file';
783
+ })),
784
+ done('openai.fine_tuning.checkpoints', 'fine_tuning', 'Fine-tuning checkpoints list (succeeded job → checkpoint; 404 unknown)', 'api', 'niche', () => withRoot(async (h) => {
785
+ const job = await h({ m: 'POST', p: '/v1/fine_tuning/jobs', b: { model: 'gpt-4o-mini', training_file: 'file-train' } });
786
+ const ck = await h({ m: 'GET', p: `/v1/fine_tuning/jobs/${id(job)}/checkpoints` });
787
+ if (!ok(ck) || ck.body.object !== 'list')
788
+ return false;
789
+ const data = ck.body.data;
790
+ if (data.length !== 1 || data[0].object !== 'fine_tuning.job.checkpoint' || data[0].fine_tuning_job_id !== id(job) || typeof data[0].step_number !== 'number')
791
+ return false;
792
+ const missing = await h({ m: 'GET', p: '/v1/fine_tuning/jobs/ftjob-nope/checkpoints' });
793
+ return missing.status === 404;
794
+ })),
795
+ // ── Vector stores (stateful) ──────────────────────────────────────────────────────────
796
+ done('openai.vector_stores.create', 'vector_stores', 'Vector stores: create (stateful) with name/status/file_counts', 'api', 'core', () => withRoot(async (h) => {
797
+ const r = await h({ m: 'POST', p: '/v1/vector_stores', b: { name: 'docs' } });
798
+ if (!ok(r) || r.body.object !== 'vector_store' || !String(id(r)).startsWith('vs-'))
799
+ return false;
800
+ const b = r.body;
801
+ return b.name === 'docs' && b.status === 'completed' && typeof b.file_counts === 'object';
802
+ })),
803
+ done('openai.vector_stores.crud', 'vector_stores', 'Vector stores: retrieve + list + delete', 'api', 'common', () => withRoot(async (h) => {
804
+ const c = await h({ m: 'POST', p: '/v1/vector_stores', b: { name: 'docs' } });
805
+ const g = await h({ m: 'GET', p: `/v1/vector_stores/${id(c)}` });
806
+ const l = await h({ m: 'GET', p: '/v1/vector_stores' });
807
+ const del = await h({ m: 'DELETE', p: `/v1/vector_stores/${id(c)}` });
808
+ const after = await h({ m: 'GET', p: '/v1/vector_stores' });
809
+ return ok(g) && l.body.data.length === 1 && del.body.deleted === true && after.body.data.length === 0;
810
+ })),
811
+ done('openai.vector_stores.files', 'vector_stores', 'Vector stores: attach + list files', 'api', 'common', () => withRoot(async (h) => {
812
+ const c = await h({ m: 'POST', p: '/v1/vector_stores', b: { name: 'docs' } });
813
+ const add = await h({ m: 'POST', p: `/v1/vector_stores/${id(c)}/files`, b: { file_id: 'file-abc' } });
814
+ if (!ok(add) || add.body.file_id !== 'file-abc' || add.body.object !== 'vector_store.file')
815
+ return false;
816
+ const l = await h({ m: 'GET', p: `/v1/vector_stores/${id(c)}/files` });
817
+ return ok(l) && l.body.data.length === 1 && l.body.data[0].file_id === 'file-abc';
818
+ })),
819
+ done('openai.vector_stores.seed_files', 'vector_stores', 'Vector stores: create with file_ids seeds attached files', 'api', 'common', () => withRoot(async (h) => {
820
+ const c = await h({ m: 'POST', p: '/v1/vector_stores', b: { name: 'docs', file_ids: ['file-a', 'file-b'] } });
821
+ const l = await h({ m: 'GET', p: `/v1/vector_stores/${id(c)}/files` });
822
+ return ok(c) && l.body.data.length === 2;
823
+ })),
824
+ done('openai.vector_stores.search', 'vector_stores', 'Vector store search: deterministic ranked results page (+ query required, 404 store)', 'api', 'common', () => withRoot(async (h) => {
825
+ const c = await h({ m: 'POST', p: '/v1/vector_stores', b: { name: 'docs', file_ids: ['file-a', 'file-b', 'file-c'] } });
826
+ const s = await h({ m: 'POST', p: `/v1/vector_stores/${id(c)}/search`, b: { query: 'hello' } });
827
+ if (!ok(s) || s.body.object !== 'vector_store.search_results.page')
828
+ return false;
829
+ const data = s.body.data;
830
+ if (data.length !== 3 || typeof data[0].score !== 'number')
831
+ return false;
832
+ // ranked descending + deterministic across runs
833
+ if (!(data[0].score >= data[1].score && data[1].score >= data[2].score))
834
+ return false;
835
+ const s2 = await h({ m: 'POST', p: `/v1/vector_stores/${id(c)}/search`, b: { query: 'hello' } });
836
+ if (JSON.stringify(s2.body.data) !== JSON.stringify(data))
837
+ return false;
838
+ const maxR = await h({ m: 'POST', p: `/v1/vector_stores/${id(c)}/search`, b: { query: 'hello', max_num_results: 1 } });
839
+ if (maxR.body.data.length !== 1)
840
+ return false;
841
+ const noQuery = await h({ m: 'POST', p: `/v1/vector_stores/${id(c)}/search`, b: {} });
842
+ const badStore = await h({ m: 'POST', p: '/v1/vector_stores/vs-nope/search', b: { query: 'x' } });
843
+ return noQuery.status === 400 && badStore.status === 404;
844
+ })),
845
+ done('openai.vector_stores.file_batches', 'vector_stores', 'Vector store file batches: bulk attach → batch object + retrieve + list batch files', 'api', 'niche', () => withRoot(async (h) => {
846
+ const vs = await h({ m: 'POST', p: '/v1/vector_stores', b: { name: 'docs' } });
847
+ const batch = await h({ m: 'POST', p: `/v1/vector_stores/${id(vs)}/file_batches`, b: { file_ids: ['file-a', 'file-b', 'file-c'] } });
848
+ if (!ok(batch))
849
+ return false;
850
+ const bb = batch.body;
851
+ if (bb.object !== 'vector_store.files_batch' || bb.status !== 'in_progress' || bb.file_counts?.total !== 3)
852
+ return false;
853
+ // retrieve the batch: the read observes OpenAI's processing done
854
+ const get = await h({ m: 'GET', p: `/v1/vector_stores/${id(vs)}/file_batches/${bb.id}` });
855
+ if (!ok(get) || get.body.id !== bb.id || get.body.status !== 'completed')
856
+ return false;
857
+ // list this batch's files
858
+ const files = await h({ m: 'GET', p: `/v1/vector_stores/${id(vs)}/file_batches/${bb.id}/files` });
859
+ if (!ok(files) || files.body.data.length !== 3)
860
+ return false;
861
+ // the files are also attached to the store as a whole
862
+ const all = await h({ m: 'GET', p: `/v1/vector_stores/${id(vs)}/files` });
863
+ if (all.body.data.length !== 3)
864
+ return false;
865
+ // validation + 404 store + 404 batch
866
+ const noIds = await h({ m: 'POST', p: `/v1/vector_stores/${id(vs)}/file_batches`, b: {} });
867
+ const badStore = await h({ m: 'POST', p: '/v1/vector_stores/vs-nope/file_batches', b: { file_ids: ['x'] } });
868
+ const badBatch = await h({ m: 'GET', p: `/v1/vector_stores/${id(vs)}/file_batches/vsfb-nope` });
869
+ return noIds.status === 400 && badStore.status === 404 && badBatch.status === 404;
870
+ })),
871
+ // ── Images ────────────────────────────────────────────────────────────────────────────
872
+ done('openai.images.generations_shape', 'images', 'Images: generations return the faithful response shape (placeholder URL)', 'api', 'common', () => withRoot(async (h) => {
873
+ const r = await h({ m: 'POST', p: '/v1/images/generations', b: { model: 'dall-e-3', prompt: 'a cat', n: 1 } });
874
+ if (!ok(r))
875
+ return false;
876
+ const d = r.body.data?.[0];
877
+ return typeof r.body.created === 'number' && typeof d?.url === 'string' && String(d.revised_prompt).includes('[twin-stub');
878
+ })),
879
+ done('openai.images.validation', 'images', 'Images: prompt required (400)', 'api', 'common', () => withRoot(async (h) => {
880
+ const r = await h({ m: 'POST', p: '/v1/images/generations', b: { model: 'dall-e-3' } });
881
+ return r.status === 400 && r.body.error?.param === 'prompt';
882
+ })),
883
+ done('openai.images.edits', 'images', 'Image edits (an image file + prompt; base64 PNG from a GPT image model) + variations (a square PNG) → faithful response shape; a text upload refused', 'api', 'niche', () => withRoot(async (h) => {
884
+ const png = placeholderPng({ width: 64, height: 64 }, 'base');
885
+ const edit = await h({ m: 'POST', p: '/v1/images/edits', b: form({ model: 'gpt-image-1', prompt: 'add a hat' }, 'image', png, 'base.png', 'image/png') });
886
+ const b64 = edit.body.data?.[0]?.b64_json;
887
+ if (!ok(edit) || typeof edit.body.created !== 'number' || typeof b64 !== 'string' || imageFormat(Uint8Array.from(atob(b64), (c) => c.charCodeAt(0))) !== 'png')
888
+ return false;
889
+ const varn = await h({ m: 'POST', p: '/v1/images/variations', b: form({ n: '2' }, 'image', png, 'base.png', 'image/png') });
890
+ if (!ok(varn) || varn.body.data.length !== 2)
891
+ return false;
892
+ // validation: edits require image AND prompt, an image file; variations require a square png
893
+ const noImg = await h({ m: 'POST', p: '/v1/images/edits', b: { prompt: 'x' } });
894
+ const noPrompt = await h({ m: 'POST', p: '/v1/images/edits', b: form({ model: 'gpt-image-1' }, 'image', png, 'base.png', 'image/png') });
895
+ const text = await h({ m: 'POST', p: '/v1/images/edits', b: form({ model: 'gpt-image-1', prompt: 'x' }, 'image', new TextEncoder().encode('PNG-base'), 'base.png', 'image/png') });
896
+ const wide = await h({ m: 'POST', p: '/v1/images/variations', b: form({}, 'image', placeholderPng({ width: 64, height: 32 }, 'wide'), 'wide.png', 'image/png') });
897
+ return noImg.status === 400 && noPrompt.status === 400 && text.status === 400 && text.body.error?.param === 'image' && wide.status === 400;
898
+ })),
899
+ // ── Audio ─────────────────────────────────────────────────────────────────────────────
900
+ done('openai.audio.transcriptions', 'audio', 'Audio transcriptions: labeled stub transcript of an audio file, json + verbose_json + text shapes; a text upload refused', 'api', 'common', () => withRoot(async (h) => {
901
+ const speech = (extra) => form({ model: 'whisper-1', ...extra }, 'file', SILENT_WAV, 'speech.wav', 'audio/wav');
902
+ const j = await h({ m: 'POST', p: '/v1/audio/transcriptions', b: speech({}) });
903
+ if (!ok(j) || typeof j.body.text !== 'string' || !String(j.body.text).includes('[twin-stub'))
904
+ return false;
905
+ const v = await h({ m: 'POST', p: '/v1/audio/transcriptions', b: speech({ response_format: 'verbose_json' }) });
906
+ if (!ok(v) || v.body.task !== 'transcribe' || !Array.isArray(v.body.segments))
907
+ return false;
908
+ const t = await h({ m: 'POST', p: '/v1/audio/transcriptions', b: speech({ response_format: 'text' }) });
909
+ if (!ok(t) || typeof t.body !== 'string')
910
+ return false;
911
+ const noFile = await h({ m: 'POST', p: '/v1/audio/transcriptions', b: { model: 'whisper-1' } });
912
+ const noModel = await h({ m: 'POST', p: '/v1/audio/transcriptions', b: form({}, 'file', SILENT_WAV, 'speech.wav', 'audio/wav') });
913
+ const text = await h({ m: 'POST', p: '/v1/audio/transcriptions', b: form({ model: 'whisper-1' }, 'file', new TextEncoder().encode('M4A-speech'), 'speech.m4a', 'audio/mp4') });
914
+ return noFile.status === 400 && noFile.body.error?.param === 'file' && noModel.status === 400 && text.status === 400 && text.body.error?.param === 'file';
915
+ })),
916
+ done('openai.audio.translations', 'audio', 'Audio translations: labeled stub, task translation, english language', 'api', 'niche', () => withRoot(async (h) => {
917
+ const v = await h({ m: 'POST', p: '/v1/audio/translations', b: form({ model: 'whisper-1', response_format: 'verbose_json' }, 'file', SILENT_WAV, 'foreign.wav', 'audio/wav') });
918
+ const noFile = await h({ m: 'POST', p: '/v1/audio/translations', b: { model: 'whisper-1' } });
919
+ return ok(v) && v.body.task === 'translation' && v.body.language === 'english' && noFile.status === 400;
920
+ })),
921
+ done('openai.audio.speech', 'audio', 'Text-to-speech: the audio file itself in the requested format with its Content-Type (mp3 by default, wav), a real file of silence (+ model/input/voice required)', 'api', 'common', async () => {
922
+ const root = mkdtempSync(join(tmpdir(), 'openai-cap-'));
923
+ try {
924
+ return await verifyBoundary('openai.audio.speech', async () => {
925
+ const fetch = createOpenAITwinFetch({ root });
926
+ const speak = (b) => fetch(new Request('https://api.openai.com/v1/audio/speech', { method: 'POST', headers: { authorization: 'Bearer sk-twin', 'content-type': 'application/json' }, body: JSON.stringify(b) }));
927
+ const mp3 = await speak({ model: 'tts-1', input: 'hello world', voice: 'alloy' });
928
+ if (mp3.status !== 200 || mp3.headers.get('content-type') !== 'audio/mpeg' || audioFormat(new Uint8Array(await mp3.arrayBuffer())) !== 'mp3')
929
+ return false;
930
+ const wav = await speak({ model: 'tts-1', input: 'hello world', voice: 'alloy', response_format: 'wav' });
931
+ if (wav.headers.get('content-type') !== 'audio/wav' || wavSeconds(new Uint8Array(await wav.arrayBuffer())) !== 1)
932
+ return false;
933
+ const noVoice = await speak({ model: 'tts-1', input: 'x' });
934
+ const noInput = await speak({ model: 'tts-1', voice: 'alloy' });
935
+ return noVoice.status === 400 && (await noVoice.json()).error?.param === 'voice' && noInput.status === 400;
936
+ });
937
+ }
938
+ finally {
939
+ rmSync(root, { recursive: true, force: true });
940
+ }
941
+ }),
942
+ todo('openai.audio.realtime', 'audio', 'Realtime API: the stateful bidirectional websocket session protocol with deterministic labeled stub audio/text events (the repo serves other non-HTTP wire protocols over the kernel)', 'api', 'niche'),
943
+ // ── Assistants (beta) ─────────────────────────────────────────────────────────────────
944
+ done('openai.assistants.crud', 'assistants', 'Assistants: create/retrieve/update/list/delete (+ model required, 404)', 'api', 'niche', () => withRoot(async (h) => {
945
+ const noModel = await h({ m: 'POST', p: '/v1/assistants', b: { name: 'a' } });
946
+ if (noModel.status !== 400)
947
+ return false;
948
+ const c = await h({ m: 'POST', p: '/v1/assistants', b: { model: 'gpt-4o', name: 'Helper', instructions: 'be helpful', tools: [{ type: 'code_interpreter' }] } });
949
+ if (!ok(c) || !String(id(c)).startsWith('asst-') || c.body.object !== 'assistant' || c.body.name !== 'Helper')
950
+ return false;
951
+ const g = await h({ m: 'GET', p: `/v1/assistants/${id(c)}` });
952
+ if (!ok(g) || id(g) !== id(c))
953
+ return false;
954
+ const upd = await h({ m: 'POST', p: `/v1/assistants/${id(c)}`, b: { name: 'Renamed' } });
955
+ if (!ok(upd) || upd.body.name !== 'Renamed' || upd.body.instructions !== 'be helpful')
956
+ return false;
957
+ const l = await h({ m: 'GET', p: '/v1/assistants' });
958
+ if (!ok(l) || l.body.data.length !== 1)
959
+ return false;
960
+ const del = await h({ m: 'DELETE', p: `/v1/assistants/${id(c)}` });
961
+ if (!ok(del) || del.body.deleted !== true)
962
+ return false;
963
+ const after = await h({ m: 'GET', p: `/v1/assistants/${id(c)}` });
964
+ const missing = await h({ m: 'GET', p: '/v1/assistants/asst-nope' });
965
+ return after.status === 404 && missing.status === 404;
966
+ })),
967
+ done('openai.threads.crud', 'assistants', 'Threads + messages: create/retrieve/update/delete + add/list/retrieve messages', 'api', 'niche', () => withRoot(async (h) => {
968
+ // create a thread with an inline seed message
969
+ const t = await h({ m: 'POST', p: '/v1/threads', b: { messages: [{ role: 'user', content: 'seed turn' }], metadata: { k: 'v' } } });
970
+ if (!ok(t) || !String(id(t)).startsWith('thread-') || t.body.object !== 'thread')
971
+ return false;
972
+ const g = await h({ m: 'GET', p: `/v1/threads/${id(t)}` });
973
+ if (!ok(g) || id(g) !== id(t))
974
+ return false;
975
+ // the seed message is present
976
+ const seeded = await h({ m: 'GET', p: `/v1/threads/${id(t)}/messages` });
977
+ if (seeded.body.data.length !== 1)
978
+ return false;
979
+ // add a message
980
+ const m = await h({ m: 'POST', p: `/v1/threads/${id(t)}/messages`, b: { role: 'user', content: 'another turn' } });
981
+ if (!ok(m) || m.body.object !== 'thread.message' || m.body.content[0].text.value !== 'another turn')
982
+ return false;
983
+ const mg = await h({ m: 'GET', p: `/v1/threads/${id(t)}/messages/${id(m)}` });
984
+ if (!ok(mg) || id(mg) !== id(m))
985
+ return false;
986
+ const list = await h({ m: 'GET', p: `/v1/threads/${id(t)}/messages` });
987
+ if (list.body.data.length !== 2)
988
+ return false;
989
+ // content required
990
+ const noContent = await h({ m: 'POST', p: `/v1/threads/${id(t)}/messages`, b: { role: 'user' } });
991
+ if (noContent.status !== 400)
992
+ return false;
993
+ // update thread metadata
994
+ const upd = await h({ m: 'POST', p: `/v1/threads/${id(t)}`, b: { metadata: { k: 'v2' } } });
995
+ if (upd.body.metadata?.k !== 'v2')
996
+ return false;
997
+ // delete
998
+ const del = await h({ m: 'DELETE', p: `/v1/threads/${id(t)}` });
999
+ if (!ok(del) || del.body.deleted !== true)
1000
+ return false;
1001
+ const after = await h({ m: 'GET', p: `/v1/threads/${id(t)}` });
1002
+ const missingMsg = await h({ m: 'POST', p: '/v1/threads/thread-nope/messages', b: { role: 'user', content: 'x' } });
1003
+ return after.status === 404 && missingMsg.status === 404;
1004
+ })),
1005
+ done('openai.runs.crud', 'assistants', 'Runs + run steps: create (stub completion) / retrieve / list / steps / cancel', 'api', 'niche', () => withRoot(async (h) => {
1006
+ const a = await h({ m: 'POST', p: '/v1/assistants', b: { model: 'gpt-4o', name: 'Runner' } });
1007
+ const t = await h({ m: 'POST', p: '/v1/threads', b: { messages: [{ role: 'user', content: 'do a thing' }] } });
1008
+ // assistant_id required + 404 unknown assistant
1009
+ const noAsst = await h({ m: 'POST', p: `/v1/threads/${id(t)}/runs`, b: {} });
1010
+ const badAsst = await h({ m: 'POST', p: `/v1/threads/${id(t)}/runs`, b: { assistant_id: 'asst-nope' } });
1011
+ if (noAsst.status !== 400 || badAsst.status !== 404)
1012
+ return false;
1013
+ // create a run → queued, as OpenAI queues it
1014
+ const run = await h({ m: 'POST', p: `/v1/threads/${id(t)}/runs`, b: { assistant_id: id(a) } });
1015
+ if (!ok(run) || !String(id(run)).startsWith('run-') || run.body.object !== 'thread.run' || run.body.status !== 'queued')
1016
+ return false;
1017
+ if (run.body.assistant_id !== id(a) || run.body.thread_id !== id(t))
1018
+ return false;
1019
+ // retrieving it observes the run completing
1020
+ const gr = await h({ m: 'GET', p: `/v1/threads/${id(t)}/runs/${id(run)}` });
1021
+ if (!ok(gr) || id(gr) !== id(run) || field(gr, 'status') !== 'completed')
1022
+ return false;
1023
+ // the thread now has the user turn + the stub assistant reply
1024
+ const msgs = (await h({ m: 'GET', p: `/v1/threads/${id(t)}/messages` })).body.data;
1025
+ const assistantMsg = msgs.find((mm) => mm.role === 'assistant');
1026
+ if (!assistantMsg || !String(assistantMsg.content[0].text.value).includes('[twin-stub'))
1027
+ return false;
1028
+ const lr = await h({ m: 'GET', p: `/v1/threads/${id(t)}/runs` });
1029
+ if (lr.body.data.length !== 1)
1030
+ return false;
1031
+ // run steps: a message_creation step pointing at the reply message
1032
+ const steps = await h({ m: 'GET', p: `/v1/threads/${id(t)}/runs/${id(run)}/steps` });
1033
+ const sd = steps.body.data;
1034
+ if (sd.length !== 1 || sd[0].object !== 'thread.run.step' || sd[0].type !== 'message_creation')
1035
+ return false;
1036
+ if (sd[0].step_details.message_creation.message_id !== assistantMsg.id)
1037
+ return false;
1038
+ const gs = await h({ m: 'GET', p: `/v1/threads/${id(t)}/runs/${id(run)}/steps/${sd[0].id}` });
1039
+ if (!ok(gs) || gs.body.id !== sd[0].id)
1040
+ return false;
1041
+ // cancel a fresh run while it is queued → cancelling, then cancelled; a completed run refuses
1042
+ const run2 = await h({ m: 'POST', p: `/v1/threads/${id(t)}/runs`, b: { assistant_id: id(a) } });
1043
+ const cancel = await h({ m: 'POST', p: `/v1/threads/${id(t)}/runs/${id(run2)}/cancel` });
1044
+ if (!ok(cancel) || field(cancel, 'status') !== 'cancelling')
1045
+ return false;
1046
+ if (field(await h({ m: 'GET', p: `/v1/threads/${id(t)}/runs/${id(run2)}` }), 'status') !== 'cancelled')
1047
+ return false;
1048
+ if ((await h({ m: 'POST', p: `/v1/threads/${id(t)}/runs/${id(run)}/cancel` })).status !== 400)
1049
+ return false;
1050
+ const badRun = await h({ m: 'GET', p: `/v1/threads/${id(t)}/runs/run-nope` });
1051
+ return badRun.status === 404;
1052
+ })),
1053
+ // ── Admin / org ───────────────────────────────────────────────────────────────────────
1054
+ done('openai.uploads.multipart', 'uploads', 'Uploads API: create → add parts → complete assembles a File (+ cancel, validation)', 'api', 'niche', () => withRoot(async (h) => {
1055
+ // create requires filename/purpose/bytes/mime_type
1056
+ const noFn = await h({ m: 'POST', p: '/v1/uploads', b: { purpose: 'fine-tune', bytes: 10, mime_type: 'text/plain' } });
1057
+ if (noFn.status !== 400)
1058
+ return false;
1059
+ const up = await h({ m: 'POST', p: '/v1/uploads', b: { filename: 'big.jsonl', purpose: 'fine-tune', bytes: 6, mime_type: 'text/plain' } });
1060
+ if (!ok(up) || up.body.object !== 'upload' || up.body.status !== 'pending' || !String(id(up)).startsWith('upload-'))
1061
+ return false;
1062
+ const p1 = await h({ m: 'POST', p: `/v1/uploads/${id(up)}/parts`, b: { data: 'foo' } });
1063
+ const p2 = await h({ m: 'POST', p: `/v1/uploads/${id(up)}/parts`, b: { data: 'bar' } });
1064
+ if (!ok(p1) || !ok(p2) || p1.body.object !== 'upload.part')
1065
+ return false;
1066
+ // complete in the given part order assembles 'foobar' into a real File
1067
+ const done = await h({ m: 'POST', p: `/v1/uploads/${id(up)}/complete`, b: { part_ids: [p1.body.id, p2.body.id] } });
1068
+ if (!ok(done) || done.body.status !== 'completed' || !done.body.file)
1069
+ return false;
1070
+ const fileId = done.body.file.id;
1071
+ const content = await h({ m: 'GET', p: `/v1/files/${fileId}/content` });
1072
+ if (content.body !== 'foobar')
1073
+ return false;
1074
+ // adding a part after completion fails; cancel a fresh upload
1075
+ const lateP = await h({ m: 'POST', p: `/v1/uploads/${id(up)}/parts`, b: { data: 'x' } });
1076
+ const up2 = await h({ m: 'POST', p: '/v1/uploads', b: { filename: 'c.jsonl', purpose: 'fine-tune', bytes: 1, mime_type: 'text/plain' } });
1077
+ const cancel = await h({ m: 'POST', p: `/v1/uploads/${id(up2)}/cancel` });
1078
+ return lateP.status === 404 && ok(cancel) && cancel.body.status === 'cancelled';
1079
+ })),
1080
+ done('openai.admin.projects', 'admin', 'Admin: projects create/retrieve/modify/list/archive (+ archived hidden by default)', 'connector', 'niche', () => withRoot(async (h) => {
1081
+ const noName = await h({ m: 'POST', p: '/v1/organization/projects', b: {} });
1082
+ if (noName.status !== 400)
1083
+ return false;
1084
+ const c = await h({ m: 'POST', p: '/v1/organization/projects', b: { name: 'Alpha' } });
1085
+ if (!ok(c) || c.body.object !== 'organization.project' || !String(id(c)).startsWith('proj-') || c.body.status !== 'active')
1086
+ return false;
1087
+ const g = await h({ m: 'GET', p: `/v1/organization/projects/${id(c)}` });
1088
+ if (!ok(g) || id(g) !== id(c))
1089
+ return false;
1090
+ const upd = await h({ m: 'POST', p: `/v1/organization/projects/${id(c)}`, b: { name: 'Alpha2' } });
1091
+ if (upd.body.name !== 'Alpha2')
1092
+ return false;
1093
+ const arch = await h({ m: 'POST', p: `/v1/organization/projects/${id(c)}/archive` });
1094
+ if (!ok(arch) || arch.body.status !== 'archived')
1095
+ return false;
1096
+ // archived project hidden by default, visible with include_archived; the organization's Default project stays
1097
+ const def = (await h({ m: 'GET', p: '/v1/organization/projects' })).body.data;
1098
+ const withArch = (await h({ m: 'GET', p: '/v1/organization/projects?include_archived=true' })).body.data;
1099
+ const missing = await h({ m: 'GET', p: '/v1/organization/projects/proj-nope' });
1100
+ return def.length === 1 && def[0].name === 'Default project' && withArch.length === 2 && missing.status === 404;
1101
+ })),
1102
+ done('openai.admin.api_keys', 'admin', 'Admin: project API keys create (one-time secret) / list / retrieve (redacted) / delete', 'connector', 'niche', () => withRoot(async (h) => {
1103
+ const proj = await h({ m: 'POST', p: '/v1/organization/projects', b: { name: 'Keys' } });
1104
+ const pid = id(proj);
1105
+ const key = await h({ m: 'POST', p: `/v1/organization/projects/${pid}/api_keys`, b: { name: 'CI key' } });
1106
+ if (!ok(key))
1107
+ return false;
1108
+ const kb = key.body;
1109
+ // creation returns the one-time secret value (a synthetic twin key, not a real OpenAI key)
1110
+ if (kb.object !== 'organization.project.api_key' || typeof kb.value !== 'string' || !String(kb.value).startsWith('sk-twin-'))
1111
+ return false;
1112
+ // retrieve only shows the redacted value, never the secret
1113
+ const g = await h({ m: 'GET', p: `/v1/organization/projects/${pid}/api_keys/${kb.id}` });
1114
+ if (!ok(g) || g.body.value !== undefined || typeof g.body.redacted_value !== 'string')
1115
+ return false;
1116
+ const list = await h({ m: 'GET', p: `/v1/organization/projects/${pid}/api_keys` });
1117
+ if (list.body.data.length !== 1)
1118
+ return false;
1119
+ const del = await h({ m: 'DELETE', p: `/v1/organization/projects/${pid}/api_keys/${kb.id}` });
1120
+ if (!ok(del) || del.body.deleted !== true)
1121
+ return false;
1122
+ const after = await h({ m: 'GET', p: `/v1/organization/projects/${pid}/api_keys/${kb.id}` });
1123
+ return after.status === 404;
1124
+ })),
1125
+ done('openai.usage.costs', 'admin', 'Usage + costs reporting endpoints computed over REAL recorded usage', 'api', 'niche', () => withRoot(async (h) => {
1126
+ // today, one daily bucket; with no traffic its results are genuinely EMPTY (nothing hardcoded)
1127
+ const day = Date.parse(CHECKS_AT) / 1000;
1128
+ const TODAY = `start_time=${day}&end_time=${day + 86_400}`;
1129
+ const empty = await h({ m: 'GET', p: `/v1/organization/usage/completions?${TODAY}&group_by=model` });
1130
+ if (!ok(empty) || empty.body.object !== 'page' || empty.body.data.length !== 1 || empty.body.data[0].results.length !== 0)
1131
+ return false;
1132
+ const emptyCosts = await h({ m: 'GET', p: `/v1/organization/costs?${TODAY}&group_by=line_item` });
1133
+ if (emptyCosts.body.data[0].results.length !== 0)
1134
+ return false;
1135
+ // generate real usage: two chat calls (gpt-4o) + one embeddings call
1136
+ const c1 = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT() });
1137
+ const c2 = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT({ messages: [{ role: 'user', content: 'another prompt entirely' }] }) });
1138
+ await h({ m: 'POST', p: '/v1/embeddings', b: { model: 'text-embedding-3-small', input: 'embed me' } });
1139
+ // completions usage aggregates the two chat calls under gpt-4o
1140
+ const usage = await h({ m: 'GET', p: `/v1/organization/usage/completions?${TODAY}&group_by=model` });
1141
+ const buckets = usage.body.data;
1142
+ if (buckets.length !== 1)
1143
+ return false;
1144
+ const results = buckets[0].results;
1145
+ const gpt4o = results.find((r) => r.model === 'gpt-4o');
1146
+ if (!gpt4o || gpt4o.object !== 'organization.usage.completions.result' || gpt4o.num_model_requests !== 2)
1147
+ return false;
1148
+ // the recorded input/output tokens equal the sum of the two completions' real usage
1149
+ const expectedIn = c1.body.usage.prompt_tokens + c2.body.usage.prompt_tokens;
1150
+ const expectedOut = c1.body.usage.completion_tokens + c2.body.usage.completion_tokens;
1151
+ if (gpt4o.input_tokens !== expectedIn || gpt4o.output_tokens !== expectedOut)
1152
+ return false;
1153
+ // embeddings usage is filtered out of the completions view, present under embeddings
1154
+ if (results.some((r) => r.model.startsWith('text-embedding')))
1155
+ return false;
1156
+ const emb = await h({ m: 'GET', p: `/v1/organization/usage/embeddings?${TODAY}&group_by=model` });
1157
+ const embResults = emb.body.data[0].results;
1158
+ if (!embResults.some((r) => r.model === 'text-embedding-3-small' && r.object === 'organization.usage.embeddings.result'))
1159
+ return false;
1160
+ // costs: a non-zero amount computed from the recorded usage (not hardcoded)
1161
+ const costs = await h({ m: 'GET', p: `/v1/organization/costs?${TODAY}&group_by=line_item` });
1162
+ const costResults = costs.body.data[0].results;
1163
+ const completionsCost = costResults.find((r) => r.line_item === 'completions');
1164
+ if (!completionsCost || completionsCost.object !== 'organization.costs.result' || completionsCost.amount.currency !== 'usd' || !(completionsCost.amount.value > 0))
1165
+ return false;
1166
+ // the cost matches the price-table computation over the recorded tokens (gpt-4o: $2.5/$10 per 1M)
1167
+ const expectedCost = (expectedIn / 1e6) * 2.5 + (expectedOut / 1e6) * 10;
1168
+ return Math.abs(completionsCost.amount.value - expectedCost) < 1e-9;
1169
+ })),
1170
+ // ── Evals API (config → runs → output items) ──────────────────────────────────────────
1171
+ done('openai.evals.api', 'evals', 'Evals API — create/run/list evals + eval runs + output items', 'api', 'niche', () => withRoot(async (h) => {
1172
+ // create an eval, read it back, assert values
1173
+ const ev = await h({ m: 'POST', p: '/v1/evals', b: { name: 'qa-eval', data_source_config: { type: 'custom', item_schema: { type: 'object' } }, testing_criteria: [{ type: 'string_check', name: 'match', input: '{{item.x}}', reference: 'y', operation: 'eq' }] } });
1174
+ if (!ok(ev) || field(ev, 'object') !== 'eval' || field(ev, 'name') !== 'qa-eval')
1175
+ return false;
1176
+ const evalId = id(ev);
1177
+ const got = await h({ m: 'GET', p: `/v1/evals/${evalId}` });
1178
+ // the testing_criteria round-trips through the kernel byte-for-byte (not fabricated)
1179
+ const tc = field(got, 'testing_criteria');
1180
+ if (!ok(got) || id(got) !== evalId || !Array.isArray(tc) || tc[0]?.name !== 'match' || tc[0]?.operation !== 'eq')
1181
+ return false;
1182
+ const dsc = field(got, 'data_source_config');
1183
+ if (dsc?.type !== 'custom')
1184
+ return false;
1185
+ const list = await h({ m: 'GET', p: '/v1/evals' });
1186
+ if (!ok(list) || !list.body.data.some((e) => e.id === evalId))
1187
+ return false;
1188
+ // kick off a run (queued, as OpenAI queues it); a read observes it graded: completed +
1189
+ // result_counts + per_model_usage reflects the run model
1190
+ const created = await h({ m: 'POST', p: `/v1/evals/${evalId}/runs`, b: { name: 'run-1', data_source: { type: 'completions', model: 'gpt-4.1', source: { type: 'file_content', content: [] } } } });
1191
+ if (!ok(created) || field(created, 'object') !== 'eval.run' || field(created, 'status') !== 'queued')
1192
+ return false;
1193
+ const run = await h({ m: 'GET', p: `/v1/evals/${evalId}/runs/${id(created)}` });
1194
+ if (!ok(run) || field(run, 'status') !== 'completed')
1195
+ return false;
1196
+ const rc = field(run, 'result_counts');
1197
+ if (rc.passed !== 1 || rc.total !== 1 || field(run, 'eval_id') !== evalId)
1198
+ return false;
1199
+ // the run echoes the model from the request data_source (not a hardcoded default)
1200
+ if (field(run, 'model') !== 'gpt-4.1')
1201
+ return false;
1202
+ const pmu = field(run, 'per_model_usage');
1203
+ if (!Array.isArray(pmu) || pmu[0]?.model_name !== 'gpt-4.1')
1204
+ return false;
1205
+ const runId = id(run);
1206
+ const runGet = await h({ m: 'GET', p: `/v1/evals/${evalId}/runs/${runId}` });
1207
+ if (!ok(runGet) || id(runGet) !== runId)
1208
+ return false;
1209
+ const runList = await h({ m: 'GET', p: `/v1/evals/${evalId}/runs` });
1210
+ if (!ok(runList) || !runList.body.data.some((r) => r.id === runId))
1211
+ return false;
1212
+ // output items
1213
+ const items = await h({ m: 'GET', p: `/v1/evals/${evalId}/runs/${runId}/output_items` });
1214
+ if (!ok(items))
1215
+ return false;
1216
+ const it = items.body.data?.[0];
1217
+ if (!it || it.object !== 'eval.run.output_item' || it.status !== 'pass' || it.run_id !== runId)
1218
+ return false;
1219
+ // negative: missing testing_criteria → 400; missing data_source_config → 400; unknown eval id → 404
1220
+ const noCriteria = await h({ m: 'POST', p: '/v1/evals', b: { data_source_config: { type: 'custom' } } });
1221
+ const noConfig = await h({ m: 'POST', p: '/v1/evals', b: { testing_criteria: [] } });
1222
+ const missing = await h({ m: 'GET', p: '/v1/evals/eval-nope' });
1223
+ const missingRun = await h({ m: 'POST', p: '/v1/evals/eval-nope/runs', b: { data_source: {} } });
1224
+ return noCriteria.status === 400 && noConfig.status === 400 && missing.status === 404 && missingRun.status === 404;
1225
+ })),
1226
+ // ── Containers API (code-interpreter sandboxes → container files) ──────────────────────
1227
+ done('openai.containers.api', 'containers', 'Containers API — create/list/delete containers + container files (code-interpreter sandboxes)', 'api', 'niche', () => withRoot(async (h) => {
1228
+ const c = await h({ m: 'POST', p: '/v1/containers', b: { name: 'sandbox-1' } });
1229
+ if (!ok(c) || field(c, 'object') !== 'container' || field(c, 'status') !== 'running' || field(c, 'name') !== 'sandbox-1')
1230
+ return false;
1231
+ const cid = id(c);
1232
+ const got = await h({ m: 'GET', p: `/v1/containers/${cid}` });
1233
+ if (!ok(got) || id(got) !== cid)
1234
+ return false;
1235
+ const list = await h({ m: 'GET', p: '/v1/containers' });
1236
+ if (!ok(list) || !list.body.data.some((x) => x.id === cid))
1237
+ return false;
1238
+ // container file from a stored File (the JSON form OpenAI takes) → read back metadata + content
1239
+ const src = await h({ m: 'POST', p: '/v1/files', b: { purpose: 'user_data', filename: 'a.txt', content: 'hello sandbox' } });
1240
+ const cf = await h({ m: 'POST', p: `/v1/containers/${cid}/files`, b: { file_id: id(src) } });
1241
+ if (!ok(cf) || field(cf, 'object') !== 'container.file' || field(cf, 'container_id') !== cid || field(cf, 'bytes') !== 'hello sandbox'.length || field(cf, 'source') !== 'file_id')
1242
+ return false;
1243
+ const fid = id(cf);
1244
+ const content = await h({ m: 'GET', p: `/v1/containers/${cid}/files/${fid}/content` });
1245
+ if (!ok(content) || content.body !== 'hello sandbox')
1246
+ return false;
1247
+ const fileGet = await h({ m: 'GET', p: `/v1/containers/${cid}/files/${fid}` });
1248
+ if (!ok(fileGet) || id(fileGet) !== fid)
1249
+ return false;
1250
+ const del = await h({ m: 'DELETE', p: `/v1/containers/${cid}/files/${fid}` });
1251
+ if (!ok(del) || del.body.deleted !== true)
1252
+ return false;
1253
+ const afterDel = await h({ m: 'GET', p: `/v1/containers/${cid}/files/${fid}` });
1254
+ if (afterDel.status !== 404)
1255
+ return false;
1256
+ // delete the container; subsequent GET → 404
1257
+ const delC = await h({ m: 'DELETE', p: `/v1/containers/${cid}` });
1258
+ const gone = await h({ m: 'GET', p: `/v1/containers/${cid}` });
1259
+ // negative: create without name → 400; unknown container → 404
1260
+ const bad = await h({ m: 'POST', p: '/v1/containers', b: {} });
1261
+ const missing = await h({ m: 'GET', p: '/v1/containers/cntr-nope' });
1262
+ return ok(delC) && gone.status === 404 && bad.status === 400 && missing.status === 404;
1263
+ })),
1264
+ // ── Background responses (queued → poll → completed | cancel) ──────────────────────────
1265
+ done('openai.responses.background', 'responses', 'Background responses (background:true → queued, then completed over time on the World clock + poll + cancel)', 'api', 'niche', () => withRoot(async (h) => {
1266
+ // the World instants of the calls: created, then polled an hour later, when OpenAI's work is done
1267
+ const T = '2026-03-01T00:00:00.000Z';
1268
+ const LATER = '2026-03-01T01:00:00.000Z';
1269
+ // background:true → immediate queued response with no output yet
1270
+ const bg = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'background task A', background: true }, at: T });
1271
+ if (!ok(bg) || field(bg, 'status') !== 'queued' || field(bg, 'background') !== true)
1272
+ return false;
1273
+ if ((field(bg, 'output') ?? []).length !== 0)
1274
+ return false;
1275
+ const rid = id(bg);
1276
+ // polled at the instant it was made, it is still queued; an hour later it has completed with the stub output + usage
1277
+ const early = await h({ m: 'GET', p: `/v1/responses/${rid}`, at: T });
1278
+ if (!ok(early) || field(early, 'status') !== 'queued')
1279
+ return false;
1280
+ const poll = await h({ m: 'GET', p: `/v1/responses/${rid}`, at: LATER });
1281
+ if (!ok(poll) || field(poll, 'status') !== 'completed')
1282
+ return false;
1283
+ const text = outputText(poll);
1284
+ if (!text.includes('[twin-stub'))
1285
+ return false;
1286
+ // usage must be coherent: total = input + output, and output > 0 (the stub produced text)
1287
+ const usage = field(poll, 'usage');
1288
+ if (!usage || typeof usage.output_tokens !== 'number' || usage.output_tokens <= 0)
1289
+ return false;
1290
+ if (usage.total_tokens !== usage.input_tokens + usage.output_tokens)
1291
+ return false;
1292
+ // the completion is billable: a usage_record was recorded → the usage report is non-empty
1293
+ const day = Date.parse(T) / 1000;
1294
+ const report = await h({ m: 'GET', p: `/v1/organization/usage/responses?start_time=${day}&end_time=${day + 86_400}`, at: LATER });
1295
+ const buckets = report.body.data;
1296
+ if (!ok(report) || buckets.length === 0 || buckets[0].results.length === 0)
1297
+ return false;
1298
+ // a second poll stays completed (idempotent) with the same output text (no re-billing visible)
1299
+ const poll2 = await h({ m: 'GET', p: `/v1/responses/${rid}`, at: LATER });
1300
+ if (field(poll2, 'status') !== 'completed' || outputText(poll2) !== text)
1301
+ return false;
1302
+ // cancel a still-queued background response → cancelled
1303
+ const bg2 = await h({ m: 'POST', p: '/v1/responses', b: { model: 'gpt-4o', input: 'background task B', background: true }, at: LATER });
1304
+ const rid2 = id(bg2);
1305
+ const cancel = await h({ m: 'POST', p: `/v1/responses/${rid2}/cancel`, at: LATER });
1306
+ if (!ok(cancel) || field(cancel, 'status') !== 'cancelled')
1307
+ return false;
1308
+ // negative: cancel an already-completed response → 400; cancel unknown id → 404
1309
+ const reCancel = await h({ m: 'POST', p: `/v1/responses/${rid}/cancel`, at: LATER });
1310
+ const missing = await h({ m: 'POST', p: '/v1/responses/resp-nope/cancel' });
1311
+ return reCancel.status === 400 && missing.status === 404;
1312
+ })),
1313
+ // ── Fine-tuning pause / resume ─────────────────────────────────────────────────────────
1314
+ done('openai.fine_tuning.pause_resume', 'fine_tuning', 'Fine-tuning job pause / resume — status transitions', 'api', 'niche', () => withRoot(async (h) => {
1315
+ const job = await h({ m: 'POST', p: '/v1/fine_tuning/jobs', b: { model: 'gpt-4o-mini', training_file: 'file-twin-1' } });
1316
+ if (!ok(job) || field(job, 'status') !== 'validating_files')
1317
+ return false;
1318
+ const jid = id(job);
1319
+ // pause while in flight → status 'paused' (a paused job does not progress on reads)
1320
+ const paused = await h({ m: 'POST', p: `/v1/fine_tuning/jobs/${jid}/pause` });
1321
+ if (!ok(paused) || field(paused, 'status') !== 'paused')
1322
+ return false;
1323
+ const readPaused = await h({ m: 'GET', p: `/v1/fine_tuning/jobs/${jid}` });
1324
+ if (field(readPaused, 'status') !== 'paused')
1325
+ return false;
1326
+ // resume → queued again; the next read observes it finishing
1327
+ const resumed = await h({ m: 'POST', p: `/v1/fine_tuning/jobs/${jid}/resume` });
1328
+ if (!ok(resumed) || field(resumed, 'status') !== 'queued')
1329
+ return false;
1330
+ const readResumed = await h({ m: 'GET', p: `/v1/fine_tuning/jobs/${jid}` });
1331
+ if (field(readResumed, 'status') !== 'succeeded')
1332
+ return false;
1333
+ // negative: resuming the now-succeeded (non-paused) job → 400; pause unknown job → 404
1334
+ const badResume = await h({ m: 'POST', p: `/v1/fine_tuning/jobs/${jid}/resume` });
1335
+ const missing = await h({ m: 'POST', p: '/v1/fine_tuning/jobs/ftjob-nope/pause' });
1336
+ return badResume.status === 400 && missing.status === 404;
1337
+ })),
1338
+ // ── Webhook signature verification (REAL HMAC, Standard-Webhooks scheme) ────────────────
1339
+ done('openai.webhooks.verify', 'webhooks', 'Webhook event signature verification (REAL signing-secret HMAC, Standard-Webhooks)', 'api', 'niche', async () => {
1340
+ // a 32-byte base64 signing secret (whsec_<base64>), like the vendor mints.
1341
+ const secret = 'whsec_' + Buffer.from('0123456789abcdef0123456789abcdef').toString('base64');
1342
+ const signed = buildSignedOpenAIWebhook({ type: 'response.completed', resourceId: 'resp-twin-abc', secret, occurredAt: '2026-01-01T00:00:00Z', webhookId: 'wh-1' });
1343
+ // a good delivery verifies and returns the parsed event with faithful shape
1344
+ const ev = verifyOpenAIWebhook(signed.body, signed.headers, secret);
1345
+ if (ev.object !== 'event' || ev.type !== 'response.completed' || ev.data.id !== 'resp-twin-abc')
1346
+ return false;
1347
+ // the signature header is the Standard-Webhooks `v1,<base64>` form computed over id.ts.payload
1348
+ const recomputed = computeOpenAIWebhookSignature('wh-1', Math.floor(Date.parse('2026-01-01T00:00:00Z') / 1000), signed.body, secret);
1349
+ if (signed.headers['webhook-signature'] !== recomputed)
1350
+ return false;
1351
+ // a TAMPERED payload must be rejected (the real verifier throws)
1352
+ let tamperedRejected = false;
1353
+ try {
1354
+ verifyOpenAIWebhook(signed.body + ' ', signed.headers, secret);
1355
+ }
1356
+ catch (e) {
1357
+ tamperedRejected = e instanceof OpenAIWebhookVerificationError;
1358
+ }
1359
+ if (!tamperedRejected)
1360
+ return false;
1361
+ // a wrong secret must be rejected
1362
+ let wrongSecretRejected = false;
1363
+ try {
1364
+ verifyOpenAIWebhook(signed.body, signed.headers, 'whsec_' + Buffer.from('ffffffffffffffffffffffffffffffff').toString('base64'));
1365
+ }
1366
+ catch (e) {
1367
+ wrongSecretRejected = e instanceof OpenAIWebhookVerificationError;
1368
+ }
1369
+ if (!wrongSecretRejected)
1370
+ return false;
1371
+ // missing headers must be rejected
1372
+ let missingRejected = false;
1373
+ try {
1374
+ verifyOpenAIWebhook(signed.body, {}, secret);
1375
+ }
1376
+ catch (e) {
1377
+ missingRejected = e instanceof OpenAIWebhookVerificationError;
1378
+ }
1379
+ return missingRejected;
1380
+ }),
1381
+ // ── Genuinely-unmodeled real surfaces (honest todos — the API has these; the twin does not yet) ──
1382
+ todo('openai.evals.run_cancel_delete', 'evals', 'Evals — cancel a run + delete an eval/run', 'api', 'niche'),
1383
+ todo('openai.evals.update', 'evals', 'Evals — update an eval (name/metadata) via POST /v1/evals/:id', 'api', 'niche'),
1384
+ todo('openai.responses.background_stream_resume', 'responses', 'Background responses — resume an in-progress stream via starting_after cursor', 'api', 'niche'),
1385
+ todo('openai.fine_tuning.checkpoint_permissions', 'fine_tuning', 'Fine-tuning checkpoint permissions (per-project grant/list/revoke)', 'api', 'niche'),
1386
+ todo('openai.admin.invites_and_users', 'admin', 'Admin API — organization invites + users (list/retrieve/modify/delete)', 'api', 'niche'),
1387
+ // ── Errors / protocol ─────────────────────────────────────────────────────────────────
1388
+ done('openai.errors.not_found', 'errors', 'Vendor-faithful 404 on unknown route', 'api', 'core', () => withRoot(async (h) => {
1389
+ const r = await h({ m: 'GET', p: '/v1/nonexistent' });
1390
+ return r.status === 404 && typeof r.body.error === 'object' && r.body.error.type === 'invalid_request_error';
1391
+ })),
1392
+ done('openai.errors.invalid_request', 'errors', 'Vendor-faithful 400 invalid_request_error envelope (error.message/type/param/code)', 'api', 'core', () => withRoot(async (h) => {
1393
+ const r = await h({ m: 'POST', p: '/v1/chat/completions', b: '{bad json' });
1394
+ const e = r.body.error;
1395
+ return r.status === 400 && e && typeof e.message === 'string' && e.type === 'invalid_request_error' && 'param' in e && 'code' in e;
1396
+ })),
1397
+ done('openai.read_only_guard', 'errors', 'Read-only mode rejects mutations (405 vendor-shaped error)', 'api', 'common', () => {
1398
+ const root = mkdtempSync(join(tmpdir(), 'openai-cap-'));
1399
+ return handleOpenAITwinRequest({ method: 'POST', path: '/v1/chat/completions', body: JSON.stringify(CHAT()), root, readOnly: true })
1400
+ .then((r) => r.status === 405 && r.body.error?.type === 'invalid_request_error')
1401
+ .finally(() => rmSync(root, { recursive: true, force: true }));
1402
+ }),
1403
+ done('openai.auth.faked', 'errors', 'Auth is modeled but permissive — any non-empty key is accepted; trusted in-process calls pass', 'api', 'common', () => withRoot(async (h) => {
1404
+ // trusted in-process calls (no headers/apiKey) are NOT auth-gated → succeed with a REAL model catalog.
1405
+ const trusted = await h({ m: 'GET', p: '/v1/models' });
1406
+ if (!ok(trusted))
1407
+ return false;
1408
+ const tb = trusted.body;
1409
+ if (tb.object !== 'list' || !Array.isArray(tb.data) || tb.data.length === 0)
1410
+ return false;
1411
+ // a request that carries a valid bearer key is accepted (the twin can't validate real keys),
1412
+ // and it must return the SAME real, non-empty catalog content — not just any 2xx status.
1413
+ const withKey = await handleOpenAITwinRequest({ method: 'GET', path: '/v1/models', headers: { authorization: 'Bearer sk-anything' } });
1414
+ if (!ok(withKey))
1415
+ return false;
1416
+ const wb = withKey.body;
1417
+ return wb.object === 'list' && Array.isArray(wb.data) && wb.data.length === tb.data.length && JSON.stringify(wb) === JSON.stringify(tb);
1418
+ })),
1419
+ done('openai.errors.auth_401', 'errors', 'Authentication error (401) — modeled auth: missing/invalid credential on a request carrying an auth surface', 'api', 'niche', () => withRootH(async (h) => {
1420
+ // a request carrying an auth surface (headers present) but no credential → 401
1421
+ const missing = await h({ m: 'GET', p: '/v1/models', headers: {} });
1422
+ if (missing.status !== 401)
1423
+ return false;
1424
+ const me = missing.body.error;
1425
+ if (me?.type !== 'invalid_request_error' || me?.code !== 'invalid_api_key')
1426
+ return false;
1427
+ // an empty bearer is also missing
1428
+ const emptyBearer = await h({ m: 'GET', p: '/v1/models', headers: { authorization: 'Bearer ' } });
1429
+ if (emptyBearer.status !== 401)
1430
+ return false;
1431
+ // the reserved sentinel exercises the invalid-key path deterministically → 401
1432
+ const invalid = await h({ m: 'GET', p: '/v1/models', headers: { authorization: 'Bearer sk-invalid' } });
1433
+ if (invalid.status !== 401)
1434
+ return false;
1435
+ // a valid bearer passes; and a trusted call WITHOUT any auth surface is NOT gated
1436
+ const valid = await h({ m: 'GET', p: '/v1/models', headers: { authorization: 'Bearer sk-twin-good' } });
1437
+ const trusted = await handleOpenAITwinRequest({ method: 'GET', path: '/v1/models' });
1438
+ return ok(valid) && ok(trusted);
1439
+ })),
1440
+ done('openai.errors.rate_limit_429', 'errors', 'Rate limit (429) — deterministic opt-in trigger → vendor envelope + Retry-After + x-ratelimit-* headers', 'api', 'niche', () => withRootH(async (h) => {
1441
+ const auth = { authorization: 'Bearer sk-twin-good' };
1442
+ // no trigger → normal success
1443
+ const normal = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT(), headers: auth });
1444
+ if (!ok(normal))
1445
+ return false;
1446
+ // the deterministic trigger header → 429 with the vendor error envelope + rate-limit headers
1447
+ const limited = await h({ m: 'POST', p: '/v1/chat/completions', b: CHAT(), headers: { ...auth, 'x-twin-force-rate-limit': '1' } });
1448
+ if (limited.status !== 429)
1449
+ return false;
1450
+ const e = limited.body.error;
1451
+ if (e?.type !== 'rate_limit_exceeded' || e?.code !== 'rate_limit_exceeded')
1452
+ return false;
1453
+ const hdrs = limited.headers;
1454
+ if (!hdrs || hdrs['retry-after'] !== '1')
1455
+ return false;
1456
+ return hdrs['x-ratelimit-remaining-requests'] === '0' && typeof hdrs['x-ratelimit-limit-tokens'] === 'string';
1457
+ })),
1458
+ done('openai.protocol.pagination', 'errors', 'Cursor pagination: after/limit + has_more/first_id/last_id across list endpoints', 'api', 'common', () => withRoot(async (h) => {
1459
+ // create 3 files; page with limit=2 → has_more + last_id; then after=last_id → final page
1460
+ for (const f of ['a', 'b', 'c'])
1461
+ await h({ m: 'POST', p: '/v1/files', b: { purpose: 'batch', filename: `${f}.jsonl`, content: '{}' } });
1462
+ const p1 = await h({ m: 'GET', p: '/v1/files?limit=2' });
1463
+ const b1 = p1.body;
1464
+ if (!ok(p1) || b1.data.length !== 2 || b1.has_more !== true || b1.last_id !== b1.data[1].id || b1.first_id !== b1.data[0].id)
1465
+ return false;
1466
+ const p2 = await h({ m: 'GET', p: `/v1/files?limit=2&after=${b1.last_id}` });
1467
+ const b2 = p2.body;
1468
+ if (b2.data.length !== 1 || b2.has_more !== false)
1469
+ return false;
1470
+ // the two pages partition the set (no overlap)
1471
+ if (b2.data[0].id === b1.data[0].id || b2.data[0].id === b1.data[1].id)
1472
+ return false;
1473
+ // an unknown cursor → empty page (vendor returns nothing past the end)
1474
+ const empty = await h({ m: 'GET', p: '/v1/files?after=file-nope' });
1475
+ return empty.body.data.length === 0;
1476
+ })),
1477
+ // ── Connector (pull/push over an injected client) ─────────────────────────────────────
1478
+ done('openai.connector.read_surface', 'connector', 'Connector read surface (models + files + batches in one pass)', 'connector', 'core', () => withRoot(async (h) => {
1479
+ const models = await h({ m: 'GET', p: '/v1/models' });
1480
+ const files = await h({ m: 'GET', p: '/v1/files' });
1481
+ const batches = await h({ m: 'GET', p: '/v1/batches' });
1482
+ return ok(models) && Array.isArray(models.body.data) && ok(files) && ok(batches);
1483
+ })),
1484
+ done('openai.connector.pull_map', 'connector', 'Connector pulls + maps real state into the twin (offline, injected client)', 'connector', 'core', async () => {
1485
+ const { mapBatch, mapFile, syncOpenAIFromReal } = await import("./openai-connector.js");
1486
+ const { projectResources } = await import('@volter/world-core');
1487
+ const root = mkdtempSync(join(tmpdir(), 'openai-cap-'));
1488
+ try {
1489
+ const mappedFile = mapFile({ id: 'file-real', purpose: 'batch', bytes: 12, status: 'processed', created_at: 1 });
1490
+ const mappedBatch = mapBatch({ id: 'batch-real', status: 'completed', created_at: 1, request_counts: { total: 1 } });
1491
+ if (mappedFile.type !== 'file' || mappedBatch.fields.status !== 'completed')
1492
+ return false;
1493
+ const fake = async (_m, p) => {
1494
+ if (p.startsWith('/v1/files'))
1495
+ return { data: [{ id: 'file-real', purpose: 'batch', bytes: 12, status: 'processed', created_at: 1 }] };
1496
+ if (p.startsWith('/v1/batches'))
1497
+ return { data: [{ id: 'batch-real', status: 'completed', created_at: 1, request_counts: { total: 1 } }] };
1498
+ return { data: [] };
1499
+ };
1500
+ const res = await syncOpenAIFromReal(fake, { root, occurredAt: '2026-06-15T00:00:00Z' });
1501
+ if (res.deltasAppended < 1)
1502
+ return false;
1503
+ const got = projectResources('openai', root).find((r) => r.id === 'file-real');
1504
+ const again = await syncOpenAIFromReal(fake, { root, occurredAt: '2026-06-15T00:00:00Z' });
1505
+ return !!got && got.status === 'processed' && again.deltasAppended === 0;
1506
+ }
1507
+ catch (err) {
1508
+ if (isInfrastructureError(err))
1509
+ throw harnessError('openai.connector.pull_map', err);
1510
+ return false;
1511
+ }
1512
+ finally {
1513
+ rmSync(root, { recursive: true, force: true });
1514
+ }
1515
+ }),
1516
+ done('openai.connector.unsupported_op_fails', 'connector', 'Connector refuses to silently drop an unsupported push op', 'connector', 'common', async () => {
1517
+ const { pushOpenAIAction } = await import("./openai-connector.js");
1518
+ try {
1519
+ const fake = async () => ({ id: 'x' });
1520
+ let threw = false;
1521
+ try {
1522
+ await pushOpenAIAction(fake, { operation: 'batch.frobnicate', subject: { type: 'batch', id: 'batch-1' }, fields: {} });
1523
+ }
1524
+ catch {
1525
+ threw = true;
1526
+ }
1527
+ return threw;
1528
+ }
1529
+ catch (err) {
1530
+ if (isInfrastructureError(err))
1531
+ throw harnessError('openai.connector.unsupported_op_fails', err);
1532
+ return false;
1533
+ }
1534
+ }),
1535
+ // ── Conformance harness ───────────────────────────────────────────────────────────────
1536
+ done('openai.conformance.envelopes', 'conformance', 'Offline conformance harness passes (envelope shapes across the surface)', 'connector', 'core', async () => {
1537
+ const { checkOpenAIConformance } = await import("./openai-conformance.js");
1538
+ const report = await checkOpenAIConformance();
1539
+ return report.ok && report.checksRun >= 6;
1540
+ }),
1541
+ // ── Pull-surface coverage audit gaps (TWIN-46 / G2) — filed as manifest todos, which is
1542
+ // what the demand-ordered build list is drawn from. See pull-audit.json (repo root) for the
1543
+ // per-pack pull-vs-read-surface evidence.
1544
+ todo('openai.connector.pull_model', 'connector', 'Connector: pull the real model catalog (GET /v1/models) into the twin', 'connector', 'core'),
1545
+ todo('openai.connector.pull_assistant', 'connector', 'Connector: pull assistants from the real account', 'connector', 'niche'),
1546
+ // ── Missing-area sweep (TWIN-87 / F1) ────────────────────────────────────────────────────
1547
+ // The legacy completions endpoint and org audit logs had NO manifest entry of any status —
1548
+ // neither the twin nor the manifest modeled them, so they never surfaced as gaps. Filed as
1549
+ // honest todos (both genuinely buildable). See OPENAI_AREAS below + openai-capabilities.
1550
+ // test.ts's area-census meta-test, gate-wired so a future whole-area omission fails here.
1551
+ todo('openai.completions_legacy.create', 'completions_legacy', 'Legacy POST /v1/completions (pre-chat text completions endpoint)', 'api', 'niche'),
1552
+ todo('openai.completions_legacy.streaming', 'completions_legacy', 'Legacy completions streaming (text/event-stream deltas)', 'api', 'niche'),
1553
+ todo('openai.admin.audit_logs.list', 'admin', 'Admin API — organization audit logs (GET /v1/organization/audit_logs, filterable)', 'api', 'niche'),
1554
+ todo('openai.admin.audit_logs.get', 'admin', 'Admin API — retrieve a single audit log event by id', 'api', 'niche'),
1555
+ ];
1556
+ // TWIN-87 committed area census — the vendor's top-level API product areas (docs nav /
1557
+ // OpenAPI tags), authored top-down independent of what a manifest entry happens to already
1558
+ // exist for. openai-capabilities.test.ts's area-census meta-test (assertAreaCensus) fails the
1559
+ // gate if a declared area has zero manifest entries and no named exclusion, OR if a manifest
1560
+ // entry's `area` drifts outside this list — so a whole missing area (legacy completions, org
1561
+ // audit logs were the concrete F1 leak) can never again hide invisibly.
1562
+ export const OPENAI_AREAS = [
1563
+ 'admin', 'assistants', 'audio', 'batches', 'chat', 'completions_legacy', 'conformance',
1564
+ 'connector', 'containers', 'embeddings', 'errors', 'evals', 'files', 'fine_tuning', 'images',
1565
+ 'models', 'moderations', 'responses', 'streaming', 'tools', 'uploads', 'vector_stores', 'webhooks',
1566
+ ];
1567
+ export function openaiCapabilities() {
1568
+ return checkCapabilities('openai', OPENAI_CAPABILITIES);
1569
+ }