@struktur/http 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +299 -0
- package/LICENSE +110 -0
- package/package.json +30 -0
- package/src/app.ts +52 -0
- package/src/config.ts +9 -0
- package/src/index.test.ts +938 -0
- package/src/index.ts +24 -0
- package/src/middleware/auth.ts +40 -0
- package/src/routes/client.ts +15 -0
- package/src/routes/debug.ts +190 -0
- package/src/routes/extract-stream.ts +175 -0
- package/src/routes/extract.ts +223 -0
- package/src/routes/info.ts +41 -0
- package/src/routes/parse.ts +115 -0
- package/src/schemas.ts +72 -0
- package/src/utils/extraction.ts +406 -0
- package/src/utils/serialize.ts +29 -0
- package/tsconfig.json +12 -0
|
@@ -0,0 +1,938 @@
|
|
|
1
|
+
import { test, expect, describe, beforeAll, afterAll } from "bun:test";
|
|
2
|
+
import { spawn, type Subprocess } from "bun";
|
|
3
|
+
|
|
4
|
+
async function startServer(port: string, apiKey = ""): Promise<Subprocess> {
|
|
5
|
+
const server = spawn({
|
|
6
|
+
cmd: ["bun", "src/index.ts"],
|
|
7
|
+
cwd: import.meta.dir + "/..",
|
|
8
|
+
env: { ...process.env, PORT: port, API_KEY: apiKey },
|
|
9
|
+
stdout: "pipe",
|
|
10
|
+
stderr: "pipe",
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
// The server runs as a separate subprocess; a fixed sleep races its startup
|
|
14
|
+
// under CI load. Poll until it answers (openapi.json is auth-free) or time out.
|
|
15
|
+
const url = `http://localhost:${port}/openapi.json`;
|
|
16
|
+
for (let attempt = 0; attempt < 50; attempt++) {
|
|
17
|
+
try {
|
|
18
|
+
const response = await fetch(url);
|
|
19
|
+
if (response.status === 200) return server;
|
|
20
|
+
} catch {
|
|
21
|
+
// Server not bound yet — retry.
|
|
22
|
+
}
|
|
23
|
+
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
server.kill();
|
|
27
|
+
throw new Error(`Server did not become ready on port ${port}`);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async function safeJson(response: Response) {
|
|
31
|
+
const contentType = response.headers.get("content-type") || "";
|
|
32
|
+
if (contentType.includes("application/json")) {
|
|
33
|
+
return response.json();
|
|
34
|
+
}
|
|
35
|
+
return { message: await response.text() };
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
describe("HTTP API - OpenAPI Documentation", () => {
|
|
39
|
+
let server: Subprocess | null = null;
|
|
40
|
+
const PORT = "3041";
|
|
41
|
+
const baseUrl = `http://localhost:${PORT}`;
|
|
42
|
+
|
|
43
|
+
beforeAll(async () => {
|
|
44
|
+
server = await startServer(PORT);
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
afterAll(() => {
|
|
48
|
+
server?.kill();
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test("GET /openapi.json returns valid OpenAPI spec", async () => {
|
|
52
|
+
const response = await fetch(`${baseUrl}/openapi.json`);
|
|
53
|
+
expect(response.status).toBe(200);
|
|
54
|
+
|
|
55
|
+
const data = await response.json();
|
|
56
|
+
expect(data).toHaveProperty("openapi");
|
|
57
|
+
expect(data).toHaveProperty("info");
|
|
58
|
+
expect(data).toHaveProperty("paths");
|
|
59
|
+
expect(data.openapi).toBe("3.1.0");
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test("OpenAPI spec has correct API info", async () => {
|
|
63
|
+
const response = await fetch(`${baseUrl}/openapi.json`);
|
|
64
|
+
const data = await response.json();
|
|
65
|
+
|
|
66
|
+
expect(data.info.title).toBe("Struktur HTTP API");
|
|
67
|
+
expect(data.info.version).toBe("1.2.1");
|
|
68
|
+
expect(data.info.description).toContain("Struktur");
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test("OpenAPI spec has all endpoints documented", async () => {
|
|
72
|
+
const response = await fetch(`${baseUrl}/openapi.json`);
|
|
73
|
+
const data = await response.json();
|
|
74
|
+
|
|
75
|
+
expect(data.paths).toHaveProperty("/");
|
|
76
|
+
expect(data.paths).toHaveProperty("/parse");
|
|
77
|
+
expect(data.paths).toHaveProperty("/extract");
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
test("OpenAPI spec has correct tags", async () => {
|
|
81
|
+
const response = await fetch(`${baseUrl}/openapi.json`);
|
|
82
|
+
const data = await response.json();
|
|
83
|
+
|
|
84
|
+
expect(data.tags).toBeDefined();
|
|
85
|
+
const tagNames = data.tags.map((t: any) => t.name);
|
|
86
|
+
expect(tagNames).toContain("Info");
|
|
87
|
+
expect(tagNames).toContain("Parse");
|
|
88
|
+
expect(tagNames).toContain("Extract");
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
test("OpenAPI spec documents GET / endpoint", async () => {
|
|
92
|
+
const response = await fetch(`${baseUrl}/openapi.json`);
|
|
93
|
+
const data = await response.json();
|
|
94
|
+
|
|
95
|
+
const getEndpoint = data.paths["/"].get;
|
|
96
|
+
expect(getEndpoint).toBeDefined();
|
|
97
|
+
expect(getEndpoint.tags).toContain("Info");
|
|
98
|
+
expect(getEndpoint.responses[200]).toBeDefined();
|
|
99
|
+
expect(getEndpoint.responses[200].content["application/json"]).toBeDefined();
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test("OpenAPI spec documents POST /parse endpoint", async () => {
|
|
103
|
+
const response = await fetch(`${baseUrl}/openapi.json`);
|
|
104
|
+
const data = await response.json();
|
|
105
|
+
|
|
106
|
+
const parseEndpoint = data.paths["/parse"].post;
|
|
107
|
+
expect(parseEndpoint).toBeDefined();
|
|
108
|
+
expect(parseEndpoint.tags).toContain("Parse");
|
|
109
|
+
expect(parseEndpoint.responses[200]).toBeDefined();
|
|
110
|
+
expect(parseEndpoint.responses[400]).toBeDefined();
|
|
111
|
+
expect(parseEndpoint.responses[500]).toBeDefined();
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test("OpenAPI spec documents POST /extract endpoint", async () => {
|
|
115
|
+
const response = await fetch(`${baseUrl}/openapi.json`);
|
|
116
|
+
const data = await response.json();
|
|
117
|
+
|
|
118
|
+
const extractEndpoint = data.paths["/extract"].post;
|
|
119
|
+
expect(extractEndpoint).toBeDefined();
|
|
120
|
+
expect(extractEndpoint.tags).toContain("Extract");
|
|
121
|
+
expect(extractEndpoint.responses[200]).toBeDefined();
|
|
122
|
+
expect(extractEndpoint.responses[400]).toBeDefined();
|
|
123
|
+
expect(extractEndpoint.responses[500]).toBeDefined();
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
test("OpenAPI spec includes inline schemas in responses", async () => {
|
|
127
|
+
const response = await fetch(`${baseUrl}/openapi.json`);
|
|
128
|
+
const data = await response.json();
|
|
129
|
+
|
|
130
|
+
const parseResponse =
|
|
131
|
+
data.paths["/parse"].post.responses[200].content["application/json"].schema;
|
|
132
|
+
expect(parseResponse).toBeDefined();
|
|
133
|
+
expect(parseResponse.type).toBe("object");
|
|
134
|
+
expect(parseResponse.properties).toHaveProperty("artifacts");
|
|
135
|
+
|
|
136
|
+
const extractResponse =
|
|
137
|
+
data.paths["/extract"].post.responses[200].content["application/json"].schema;
|
|
138
|
+
expect(extractResponse).toBeDefined();
|
|
139
|
+
expect(extractResponse.type).toBe("object");
|
|
140
|
+
expect(extractResponse.properties).toHaveProperty("data");
|
|
141
|
+
expect(extractResponse.properties).toHaveProperty("usage");
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
test("OpenAPI spec has servers defined", async () => {
|
|
145
|
+
const response = await fetch(`${baseUrl}/openapi.json`);
|
|
146
|
+
const data = await response.json();
|
|
147
|
+
|
|
148
|
+
expect(data.servers).toBeDefined();
|
|
149
|
+
expect(data.servers.length).toBeGreaterThan(0);
|
|
150
|
+
expect(data.servers[0].url).toContain("localhost");
|
|
151
|
+
});
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
describe("HTTP API - Core Functionality", () => {
|
|
155
|
+
let server: Subprocess | null = null;
|
|
156
|
+
const PORT = "3042";
|
|
157
|
+
const baseUrl = `http://localhost:${PORT}`;
|
|
158
|
+
|
|
159
|
+
beforeAll(async () => {
|
|
160
|
+
server = await startServer(PORT);
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
afterAll(() => {
|
|
164
|
+
server?.kill();
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
test("GET / returns API info with correct structure", async () => {
|
|
168
|
+
const response = await fetch(`${baseUrl}/`);
|
|
169
|
+
const data = await response.json();
|
|
170
|
+
|
|
171
|
+
expect(response.status).toBe(200);
|
|
172
|
+
expect(data).toHaveProperty("name", "struktur-http");
|
|
173
|
+
expect(data).toHaveProperty("version");
|
|
174
|
+
expect(data).toHaveProperty("endpoints");
|
|
175
|
+
expect(data.endpoints).toHaveProperty("POST /parse");
|
|
176
|
+
expect(data.endpoints).toHaveProperty("POST /extract");
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
test("CORS headers are present", async () => {
|
|
180
|
+
const response = await fetch(`${baseUrl}/`, {
|
|
181
|
+
method: "OPTIONS",
|
|
182
|
+
headers: {
|
|
183
|
+
Origin: "http://example.com",
|
|
184
|
+
"Access-Control-Request-Method": "POST",
|
|
185
|
+
},
|
|
186
|
+
});
|
|
187
|
+
|
|
188
|
+
expect(response.headers.get("access-control-allow-origin")).toBeTruthy();
|
|
189
|
+
});
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
describe("HTTP API - Authentication", () => {
|
|
193
|
+
let server: Subprocess | null = null;
|
|
194
|
+
const PORT = "3043";
|
|
195
|
+
const API_KEY = "test-secret-key-123";
|
|
196
|
+
const baseUrl = `http://localhost:${PORT}`;
|
|
197
|
+
|
|
198
|
+
beforeAll(async () => {
|
|
199
|
+
server = await startServer(PORT, API_KEY);
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
afterAll(() => {
|
|
203
|
+
server?.kill();
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
test("Request without auth header returns 401", async () => {
|
|
207
|
+
const response = await fetch(`${baseUrl}/`);
|
|
208
|
+
expect(response.status).toBe(401);
|
|
209
|
+
const data = await safeJson(response);
|
|
210
|
+
expect(data.message).toContain("Missing Authorization");
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
test("Request with invalid auth format returns 401", async () => {
|
|
214
|
+
const response = await fetch(`${baseUrl}/`, {
|
|
215
|
+
headers: { Authorization: "Basic dGVzdA==" },
|
|
216
|
+
});
|
|
217
|
+
expect(response.status).toBe(401);
|
|
218
|
+
const data = await safeJson(response);
|
|
219
|
+
expect(data.message).toContain("Invalid Authorization");
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
test("Request with wrong API key returns 401", async () => {
|
|
223
|
+
const response = await fetch(`${baseUrl}/`, {
|
|
224
|
+
headers: { Authorization: "Bearer wrong-key" },
|
|
225
|
+
});
|
|
226
|
+
expect(response.status).toBe(401);
|
|
227
|
+
const data = await safeJson(response);
|
|
228
|
+
expect(data.message).toContain("Invalid API key");
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
test("Request with valid API key succeeds", async () => {
|
|
232
|
+
const response = await fetch(`${baseUrl}/`, {
|
|
233
|
+
headers: { Authorization: `Bearer ${API_KEY}` },
|
|
234
|
+
});
|
|
235
|
+
expect(response.status).toBe(200);
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
test("OpenAPI endpoint is accessible without auth", async () => {
|
|
239
|
+
const response = await fetch(`${baseUrl}/openapi.json`);
|
|
240
|
+
expect(response.status).toBe(200);
|
|
241
|
+
});
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
describe("HTTP API - Debug Endpoint", () => {
|
|
245
|
+
let server: Subprocess | null = null;
|
|
246
|
+
const PORT = "3051";
|
|
247
|
+
const baseUrl = `http://localhost:${PORT}`;
|
|
248
|
+
|
|
249
|
+
beforeAll(async () => {
|
|
250
|
+
server = await startServer(PORT);
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
afterAll(() => {
|
|
254
|
+
server?.kill();
|
|
255
|
+
});
|
|
256
|
+
|
|
257
|
+
test("GET /debug returns HTML", async () => {
|
|
258
|
+
const response = await fetch(`${baseUrl}/debug`);
|
|
259
|
+
expect(response.status).toBe(200);
|
|
260
|
+
expect(response.headers.get("content-type")).toContain("text/html");
|
|
261
|
+
const html = await response.text();
|
|
262
|
+
expect(html).toContain("Struktur Debug");
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
test("GET /debug is accessible without auth", async () => {
|
|
266
|
+
const authServer = await startServer("3052", "auth-secret-key");
|
|
267
|
+
try {
|
|
268
|
+
const response = await fetch(`http://localhost:3052/debug`);
|
|
269
|
+
expect(response.status).toBe(200);
|
|
270
|
+
} finally {
|
|
271
|
+
authServer.kill();
|
|
272
|
+
}
|
|
273
|
+
});
|
|
274
|
+
});
|
|
275
|
+
|
|
276
|
+
describe("HTTP API - Parse Endpoint", () => {
|
|
277
|
+
let server: Subprocess | null = null;
|
|
278
|
+
const PORT = "3044";
|
|
279
|
+
const baseUrl = `http://localhost:${PORT}`;
|
|
280
|
+
|
|
281
|
+
beforeAll(async () => {
|
|
282
|
+
server = await startServer(PORT);
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
afterAll(() => {
|
|
286
|
+
server?.kill();
|
|
287
|
+
});
|
|
288
|
+
|
|
289
|
+
test("POST /parse with text file returns valid artifacts", async () => {
|
|
290
|
+
const formData = new FormData();
|
|
291
|
+
const content = "Hello, this is a test file content!";
|
|
292
|
+
const file = new File([content], "test.txt", { type: "text/plain" });
|
|
293
|
+
formData.append("file", file);
|
|
294
|
+
|
|
295
|
+
const response = await fetch(`${baseUrl}/parse`, {
|
|
296
|
+
method: "POST",
|
|
297
|
+
body: formData,
|
|
298
|
+
});
|
|
299
|
+
|
|
300
|
+
expect(response.status).toBe(200);
|
|
301
|
+
const data = await response.json();
|
|
302
|
+
expect(data).toHaveProperty("artifacts");
|
|
303
|
+
expect(Array.isArray(data.artifacts)).toBe(true);
|
|
304
|
+
expect(data.artifacts.length).toBeGreaterThan(0);
|
|
305
|
+
|
|
306
|
+
const artifact = data.artifacts[0];
|
|
307
|
+
expect(artifact).toHaveProperty("id");
|
|
308
|
+
expect(artifact).toHaveProperty("type");
|
|
309
|
+
expect(artifact).toHaveProperty("contents");
|
|
310
|
+
expect(Array.isArray(artifact.contents)).toBe(true);
|
|
311
|
+
expect(artifact.contents[0]).toHaveProperty("text");
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
test("POST /parse with images option", async () => {
|
|
315
|
+
const formData = new FormData();
|
|
316
|
+
const file = new File(["test content"], "test.txt", { type: "text/plain" });
|
|
317
|
+
formData.append("file", file);
|
|
318
|
+
formData.append("images", "true");
|
|
319
|
+
|
|
320
|
+
const response = await fetch(`${baseUrl}/parse`, {
|
|
321
|
+
method: "POST",
|
|
322
|
+
body: formData,
|
|
323
|
+
});
|
|
324
|
+
|
|
325
|
+
expect(response.status).toBe(200);
|
|
326
|
+
const data = await response.json();
|
|
327
|
+
expect(data).toHaveProperty("artifacts");
|
|
328
|
+
});
|
|
329
|
+
|
|
330
|
+
test("POST /parse with screenshots and scale options", async () => {
|
|
331
|
+
const formData = new FormData();
|
|
332
|
+
const file = new File(["test content"], "test.txt", { type: "text/plain" });
|
|
333
|
+
formData.append("file", file);
|
|
334
|
+
formData.append("screenshots", "true");
|
|
335
|
+
formData.append("screenshotScale", "2.0");
|
|
336
|
+
formData.append("screenshotWidth", "1920");
|
|
337
|
+
|
|
338
|
+
const response = await fetch(`${baseUrl}/parse`, {
|
|
339
|
+
method: "POST",
|
|
340
|
+
body: formData,
|
|
341
|
+
});
|
|
342
|
+
|
|
343
|
+
expect(response.status).toBe(200);
|
|
344
|
+
const data = await response.json();
|
|
345
|
+
expect(data).toHaveProperty("artifacts");
|
|
346
|
+
});
|
|
347
|
+
|
|
348
|
+
test("POST /parse with HTML file", async () => {
|
|
349
|
+
const formData = new FormData();
|
|
350
|
+
const html = "<!DOCTYPE html><html><body><h1>Test</h1></body></html>";
|
|
351
|
+
const file = new File([html], "test.html", { type: "text/html" });
|
|
352
|
+
formData.append("file", file);
|
|
353
|
+
|
|
354
|
+
const response = await fetch(`${baseUrl}/parse`, {
|
|
355
|
+
method: "POST",
|
|
356
|
+
body: formData,
|
|
357
|
+
});
|
|
358
|
+
|
|
359
|
+
expect(response.status).toBe(200);
|
|
360
|
+
const data = await response.json();
|
|
361
|
+
expect(data).toHaveProperty("artifacts");
|
|
362
|
+
expect(data.artifacts.length).toBeGreaterThan(0);
|
|
363
|
+
});
|
|
364
|
+
|
|
365
|
+
test("POST /parse without file returns error", async () => {
|
|
366
|
+
const formData = new FormData();
|
|
367
|
+
formData.append("wrongField", "value");
|
|
368
|
+
|
|
369
|
+
const response = await fetch(`${baseUrl}/parse`, {
|
|
370
|
+
method: "POST",
|
|
371
|
+
body: formData,
|
|
372
|
+
});
|
|
373
|
+
|
|
374
|
+
expect(response.status).toBe(400);
|
|
375
|
+
});
|
|
376
|
+
|
|
377
|
+
test("POST /parse with non-multipart content returns error", async () => {
|
|
378
|
+
const response = await fetch(`${baseUrl}/parse`, {
|
|
379
|
+
method: "POST",
|
|
380
|
+
headers: { "Content-Type": "application/json" },
|
|
381
|
+
body: JSON.stringify({ file: "test" }),
|
|
382
|
+
});
|
|
383
|
+
|
|
384
|
+
expect(response.status).toBe(400);
|
|
385
|
+
});
|
|
386
|
+
});
|
|
387
|
+
|
|
388
|
+
describe("HTTP API - Extract Endpoint (JSON)", () => {
|
|
389
|
+
let server: Subprocess | null = null;
|
|
390
|
+
const PORT = "3045";
|
|
391
|
+
const baseUrl = `http://localhost:${PORT}`;
|
|
392
|
+
|
|
393
|
+
beforeAll(async () => {
|
|
394
|
+
server = await startServer(PORT);
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
afterAll(() => {
|
|
398
|
+
server?.kill();
|
|
399
|
+
});
|
|
400
|
+
|
|
401
|
+
test("POST /extract with missing model returns 400", async () => {
|
|
402
|
+
const response = await fetch(`${baseUrl}/extract`, {
|
|
403
|
+
method: "POST",
|
|
404
|
+
headers: { "Content-Type": "application/json" },
|
|
405
|
+
body: JSON.stringify({
|
|
406
|
+
artifacts: [{ id: "test", type: "text", contents: [{ text: "test" }] }],
|
|
407
|
+
schema: { type: "object" },
|
|
408
|
+
}),
|
|
409
|
+
});
|
|
410
|
+
|
|
411
|
+
expect(response.status).toBe(400);
|
|
412
|
+
});
|
|
413
|
+
|
|
414
|
+
test("POST /extract with missing schema/fields returns 400", async () => {
|
|
415
|
+
const response = await fetch(`${baseUrl}/extract`, {
|
|
416
|
+
method: "POST",
|
|
417
|
+
headers: { "Content-Type": "application/json" },
|
|
418
|
+
body: JSON.stringify({
|
|
419
|
+
artifacts: [{ id: "test", type: "text", contents: [{ text: "test" }] }],
|
|
420
|
+
model: "openai/gpt-4",
|
|
421
|
+
}),
|
|
422
|
+
});
|
|
423
|
+
|
|
424
|
+
expect(response.status).toBe(400);
|
|
425
|
+
});
|
|
426
|
+
|
|
427
|
+
test("POST /extract with invalid model format returns error", async () => {
|
|
428
|
+
const response = await fetch(`${baseUrl}/extract?sse=false`, {
|
|
429
|
+
method: "POST",
|
|
430
|
+
headers: { "Content-Type": "application/json" },
|
|
431
|
+
body: JSON.stringify({
|
|
432
|
+
artifacts: [{ id: "test", type: "text", contents: [{ text: "test" }] }],
|
|
433
|
+
schema: { type: "object" },
|
|
434
|
+
model: "invalid-model-format",
|
|
435
|
+
}),
|
|
436
|
+
});
|
|
437
|
+
|
|
438
|
+
expect(response.status).toBe(500);
|
|
439
|
+
});
|
|
440
|
+
|
|
441
|
+
test("POST /extract with invalid provider returns error", async () => {
|
|
442
|
+
const response = await fetch(`${baseUrl}/extract?sse=false`, {
|
|
443
|
+
method: "POST",
|
|
444
|
+
headers: { "Content-Type": "application/json" },
|
|
445
|
+
body: JSON.stringify({
|
|
446
|
+
artifacts: [{ id: "test", type: "text", contents: [{ text: "test" }] }],
|
|
447
|
+
schema: { type: "object" },
|
|
448
|
+
model: "unknown-provider/model",
|
|
449
|
+
}),
|
|
450
|
+
});
|
|
451
|
+
|
|
452
|
+
expect(response.status).toBe(500);
|
|
453
|
+
});
|
|
454
|
+
|
|
455
|
+
test("POST /extract with unsupported strategy returns error", async () => {
|
|
456
|
+
const response = await fetch(`${baseUrl}/extract?sse=false`, {
|
|
457
|
+
method: "POST",
|
|
458
|
+
headers: { "Content-Type": "application/json" },
|
|
459
|
+
body: JSON.stringify({
|
|
460
|
+
artifacts: [{ id: "test", type: "text", contents: [{ text: "test" }] }],
|
|
461
|
+
schema: { type: "object" },
|
|
462
|
+
model: "openai/gpt-4",
|
|
463
|
+
strategy: "invalid-strategy",
|
|
464
|
+
}),
|
|
465
|
+
});
|
|
466
|
+
|
|
467
|
+
expect(response.status).toBe(500);
|
|
468
|
+
});
|
|
469
|
+
|
|
470
|
+
test("POST /extract accepts valid strategy names", async () => {
|
|
471
|
+
const strategies = [
|
|
472
|
+
"simple",
|
|
473
|
+
"parallel",
|
|
474
|
+
"sequential",
|
|
475
|
+
"parallelAutoMerge",
|
|
476
|
+
"sequentialAutoMerge",
|
|
477
|
+
"doublePass",
|
|
478
|
+
"doublePassAutoMerge",
|
|
479
|
+
];
|
|
480
|
+
|
|
481
|
+
for (const strategy of strategies) {
|
|
482
|
+
const response = await fetch(`${baseUrl}/extract?sse=false`, {
|
|
483
|
+
method: "POST",
|
|
484
|
+
headers: { "Content-Type": "application/json" },
|
|
485
|
+
body: JSON.stringify({
|
|
486
|
+
artifacts: [{ id: "test", type: "text", contents: [{ text: "test" }] }],
|
|
487
|
+
schema: { type: "object", properties: { name: { type: "string" } } },
|
|
488
|
+
model: "openai/gpt-4",
|
|
489
|
+
strategy,
|
|
490
|
+
}),
|
|
491
|
+
});
|
|
492
|
+
|
|
493
|
+
// All should accept the request, but fail due to missing API key
|
|
494
|
+
expect([200, 500]).toContain(response.status);
|
|
495
|
+
}
|
|
496
|
+
});
|
|
497
|
+
|
|
498
|
+
test("POST /extract with invalid artifacts JSON returns 400", async () => {
|
|
499
|
+
const formData = new FormData();
|
|
500
|
+
formData.append("artifacts", "invalid json");
|
|
501
|
+
formData.append("model", "openai/gpt-4");
|
|
502
|
+
formData.append("schema", JSON.stringify({ type: "object" }));
|
|
503
|
+
|
|
504
|
+
const response = await fetch(`${baseUrl}/extract`, {
|
|
505
|
+
method: "POST",
|
|
506
|
+
body: formData,
|
|
507
|
+
});
|
|
508
|
+
|
|
509
|
+
expect(response.status).toBe(400);
|
|
510
|
+
});
|
|
511
|
+
|
|
512
|
+
test("POST /extract with invalid schema JSON returns 400", async () => {
|
|
513
|
+
const formData = new FormData();
|
|
514
|
+
formData.append(
|
|
515
|
+
"artifacts",
|
|
516
|
+
JSON.stringify([{ id: "test", type: "text", contents: [{ text: "test" }] }]),
|
|
517
|
+
);
|
|
518
|
+
formData.append("model", "openai/gpt-4");
|
|
519
|
+
formData.append("schema", "invalid json");
|
|
520
|
+
|
|
521
|
+
const response = await fetch(`${baseUrl}/extract`, {
|
|
522
|
+
method: "POST",
|
|
523
|
+
body: formData,
|
|
524
|
+
});
|
|
525
|
+
|
|
526
|
+
expect(response.status).toBe(400);
|
|
527
|
+
});
|
|
528
|
+
|
|
529
|
+
test("POST /extract with fields shorthand instead of schema", async () => {
|
|
530
|
+
const formData = new FormData();
|
|
531
|
+
formData.append(
|
|
532
|
+
"artifacts",
|
|
533
|
+
JSON.stringify([{ id: "test", type: "text", contents: [{ text: "test" }] }]),
|
|
534
|
+
);
|
|
535
|
+
formData.append("model", "openai/gpt-4");
|
|
536
|
+
formData.append("fields", "name,email,phone");
|
|
537
|
+
|
|
538
|
+
const response = await fetch(`${baseUrl}/extract?sse=false`, {
|
|
539
|
+
method: "POST",
|
|
540
|
+
body: formData,
|
|
541
|
+
});
|
|
542
|
+
|
|
543
|
+
// Should either succeed or fail due to missing API key
|
|
544
|
+
expect([200, 500]).toContain(response.status);
|
|
545
|
+
});
|
|
546
|
+
});
|
|
547
|
+
|
|
548
|
+
describe("HTTP API - Extract Endpoint SSE Default", () => {
|
|
549
|
+
let server: Subprocess | null = null;
|
|
550
|
+
const PORT = "3053";
|
|
551
|
+
const baseUrl = `http://localhost:${PORT}`;
|
|
552
|
+
|
|
553
|
+
beforeAll(async () => {
|
|
554
|
+
server = await startServer(PORT);
|
|
555
|
+
});
|
|
556
|
+
|
|
557
|
+
afterAll(() => {
|
|
558
|
+
server?.kill();
|
|
559
|
+
});
|
|
560
|
+
|
|
561
|
+
test("POST /extract returns SSE by default", async () => {
|
|
562
|
+
const response = await fetch(`${baseUrl}/extract`, {
|
|
563
|
+
method: "POST",
|
|
564
|
+
headers: { "Content-Type": "application/json" },
|
|
565
|
+
body: JSON.stringify({
|
|
566
|
+
artifacts: [{ id: "test", type: "text", contents: [{ text: "test" }] }],
|
|
567
|
+
schema: { type: "object" },
|
|
568
|
+
model: "openai/gpt-4",
|
|
569
|
+
}),
|
|
570
|
+
});
|
|
571
|
+
|
|
572
|
+
expect(response.status).toBe(200);
|
|
573
|
+
expect(response.headers.get("content-type")).toContain("text/event-stream");
|
|
574
|
+
});
|
|
575
|
+
|
|
576
|
+
test("POST /extract?sse=false returns JSON", async () => {
|
|
577
|
+
const response = await fetch(`${baseUrl}/extract?sse=false`, {
|
|
578
|
+
method: "POST",
|
|
579
|
+
headers: { "Content-Type": "application/json" },
|
|
580
|
+
body: JSON.stringify({
|
|
581
|
+
artifacts: [{ id: "test", type: "text", contents: [{ text: "test" }] }],
|
|
582
|
+
schema: { type: "object" },
|
|
583
|
+
model: "openai/gpt-4",
|
|
584
|
+
}),
|
|
585
|
+
});
|
|
586
|
+
|
|
587
|
+
expect(response.status).toBe(200);
|
|
588
|
+
expect(response.headers.get("content-type")).toContain("application/json");
|
|
589
|
+
});
|
|
590
|
+
|
|
591
|
+
test("POST /extract SSE stream contains error event for invalid model", async () => {
|
|
592
|
+
const response = await fetch(`${baseUrl}/extract`, {
|
|
593
|
+
method: "POST",
|
|
594
|
+
headers: { "Content-Type": "application/json" },
|
|
595
|
+
body: JSON.stringify({
|
|
596
|
+
artifacts: [{ id: "test", type: "text", contents: [{ text: "test" }] }],
|
|
597
|
+
schema: { type: "object" },
|
|
598
|
+
model: "invalid-model-format",
|
|
599
|
+
}),
|
|
600
|
+
});
|
|
601
|
+
|
|
602
|
+
expect(response.status).toBe(200);
|
|
603
|
+
expect(response.headers.get("content-type")).toContain("text/event-stream");
|
|
604
|
+
|
|
605
|
+
const body = await response.text();
|
|
606
|
+
const lines = body.split("\n").filter((line) => line.startsWith("data:"));
|
|
607
|
+
const events = lines.map((line) => JSON.parse(line.replace("data: ", "")));
|
|
608
|
+
const errorEvent = events.find((e) => e.type === "error");
|
|
609
|
+
expect(errorEvent).toBeDefined();
|
|
610
|
+
expect(errorEvent.data.message).toContain("Invalid model format");
|
|
611
|
+
});
|
|
612
|
+
});
|
|
613
|
+
|
|
614
|
+
describe("HTTP API - Extract Stream Endpoint", () => {
|
|
615
|
+
let server: Subprocess | null = null;
|
|
616
|
+
const PORT = "3050";
|
|
617
|
+
const baseUrl = `http://localhost:${PORT}`;
|
|
618
|
+
|
|
619
|
+
beforeAll(async () => {
|
|
620
|
+
server = await startServer(PORT);
|
|
621
|
+
});
|
|
622
|
+
|
|
623
|
+
afterAll(() => {
|
|
624
|
+
server?.kill();
|
|
625
|
+
});
|
|
626
|
+
|
|
627
|
+
test("POST /extract/stream returns SSE content type", async () => {
|
|
628
|
+
const response = await fetch(`${baseUrl}/extract/stream`, {
|
|
629
|
+
method: "POST",
|
|
630
|
+
headers: { "Content-Type": "application/json" },
|
|
631
|
+
body: JSON.stringify({
|
|
632
|
+
artifacts: [{ id: "test", type: "text", contents: [{ text: "test" }] }],
|
|
633
|
+
schema: { type: "object" },
|
|
634
|
+
model: "openai/gpt-4",
|
|
635
|
+
}),
|
|
636
|
+
});
|
|
637
|
+
|
|
638
|
+
expect(response.status).toBe(200);
|
|
639
|
+
expect(response.headers.get("content-type")).toContain("text/event-stream");
|
|
640
|
+
});
|
|
641
|
+
|
|
642
|
+
test("POST /extract/stream with missing model returns 400", async () => {
|
|
643
|
+
const response = await fetch(`${baseUrl}/extract/stream`, {
|
|
644
|
+
method: "POST",
|
|
645
|
+
headers: { "Content-Type": "application/json" },
|
|
646
|
+
body: JSON.stringify({
|
|
647
|
+
artifacts: [{ id: "test", type: "text", contents: [{ text: "test" }] }],
|
|
648
|
+
schema: { type: "object" },
|
|
649
|
+
}),
|
|
650
|
+
});
|
|
651
|
+
|
|
652
|
+
expect(response.status).toBe(400);
|
|
653
|
+
});
|
|
654
|
+
|
|
655
|
+
test("POST /extract/stream with multipart form data returns SSE", async () => {
|
|
656
|
+
const formData = new FormData();
|
|
657
|
+
formData.append(
|
|
658
|
+
"artifacts",
|
|
659
|
+
JSON.stringify([{ id: "test", type: "text", contents: [{ text: "test" }] }]),
|
|
660
|
+
);
|
|
661
|
+
formData.append("model", "openai/gpt-4");
|
|
662
|
+
formData.append("fields", "content");
|
|
663
|
+
|
|
664
|
+
const response = await fetch(`${baseUrl}/extract/stream`, {
|
|
665
|
+
method: "POST",
|
|
666
|
+
body: formData,
|
|
667
|
+
});
|
|
668
|
+
|
|
669
|
+
expect(response.status).toBe(200);
|
|
670
|
+
expect(response.headers.get("content-type")).toContain("text/event-stream");
|
|
671
|
+
});
|
|
672
|
+
|
|
673
|
+
test("POST /extract/stream with invalid content type returns 400", async () => {
|
|
674
|
+
const response = await fetch(`${baseUrl}/extract/stream`, {
|
|
675
|
+
method: "POST",
|
|
676
|
+
headers: { "Content-Type": "text/plain" },
|
|
677
|
+
body: "not valid",
|
|
678
|
+
});
|
|
679
|
+
|
|
680
|
+
expect(response.status).toBe(400);
|
|
681
|
+
});
|
|
682
|
+
});
|
|
683
|
+
|
|
684
|
+
describe("HTTP API - Extract with File Upload", () => {
|
|
685
|
+
let server: Subprocess | null = null;
|
|
686
|
+
const PORT = "3046";
|
|
687
|
+
const baseUrl = `http://localhost:${PORT}`;
|
|
688
|
+
|
|
689
|
+
beforeAll(async () => {
|
|
690
|
+
server = await startServer(PORT);
|
|
691
|
+
});
|
|
692
|
+
|
|
693
|
+
afterAll(() => {
|
|
694
|
+
server?.kill();
|
|
695
|
+
});
|
|
696
|
+
|
|
697
|
+
test("POST /extract with file instead of artifacts", async () => {
|
|
698
|
+
const formData = new FormData();
|
|
699
|
+
const file = new File(["John Doe works at Acme Corp"], "resume.txt", { type: "text/plain" });
|
|
700
|
+
formData.append("file", file);
|
|
701
|
+
formData.append("model", "openai/gpt-4");
|
|
702
|
+
formData.append(
|
|
703
|
+
"schema",
|
|
704
|
+
JSON.stringify({
|
|
705
|
+
type: "object",
|
|
706
|
+
properties: {
|
|
707
|
+
name: { type: "string" },
|
|
708
|
+
company: { type: "string" },
|
|
709
|
+
},
|
|
710
|
+
}),
|
|
711
|
+
);
|
|
712
|
+
formData.append("images", "false");
|
|
713
|
+
formData.append("screenshots", "false");
|
|
714
|
+
|
|
715
|
+
const response = await fetch(`${baseUrl}/extract?sse=false`, {
|
|
716
|
+
method: "POST",
|
|
717
|
+
body: formData,
|
|
718
|
+
});
|
|
719
|
+
|
|
720
|
+
if (response.status === 200) {
|
|
721
|
+
const data = await response.json();
|
|
722
|
+
expect(data).toHaveProperty("data");
|
|
723
|
+
expect(data).toHaveProperty("usage");
|
|
724
|
+
} else {
|
|
725
|
+
expect(response.status).toBe(500);
|
|
726
|
+
}
|
|
727
|
+
});
|
|
728
|
+
|
|
729
|
+
test("POST /extract with file and images option", async () => {
|
|
730
|
+
const formData = new FormData();
|
|
731
|
+
const file = new File(["test content"], "test.txt", { type: "text/plain" });
|
|
732
|
+
formData.append("file", file);
|
|
733
|
+
formData.append("model", "openai/gpt-4");
|
|
734
|
+
formData.append("fields", "content");
|
|
735
|
+
formData.append("images", "true");
|
|
736
|
+
|
|
737
|
+
const response = await fetch(`${baseUrl}/extract?sse=false`, {
|
|
738
|
+
method: "POST",
|
|
739
|
+
body: formData,
|
|
740
|
+
});
|
|
741
|
+
|
|
742
|
+
expect([200, 500]).toContain(response.status);
|
|
743
|
+
});
|
|
744
|
+
});
|
|
745
|
+
|
|
746
|
+
describe("HTTP API - Error Handling", () => {
|
|
747
|
+
let server: Subprocess | null = null;
|
|
748
|
+
const PORT = "3047";
|
|
749
|
+
const baseUrl = `http://localhost:${PORT}`;
|
|
750
|
+
|
|
751
|
+
beforeAll(async () => {
|
|
752
|
+
server = await startServer(PORT);
|
|
753
|
+
});
|
|
754
|
+
|
|
755
|
+
afterAll(() => {
|
|
756
|
+
server?.kill();
|
|
757
|
+
});
|
|
758
|
+
|
|
759
|
+
test("404 for unknown endpoint", async () => {
|
|
760
|
+
const response = await fetch(`${baseUrl}/unknown-endpoint`);
|
|
761
|
+
expect(response.status).toBe(404);
|
|
762
|
+
});
|
|
763
|
+
|
|
764
|
+
test("POST to GET endpoint returns 404", async () => {
|
|
765
|
+
const response = await fetch(`${baseUrl}/`, { method: "POST" });
|
|
766
|
+
expect(response.status).toBe(404);
|
|
767
|
+
});
|
|
768
|
+
|
|
769
|
+
test("Malformed JSON in extract request returns error", async () => {
|
|
770
|
+
const response = await fetch(`${baseUrl}/extract`, {
|
|
771
|
+
method: "POST",
|
|
772
|
+
headers: { "Content-Type": "application/json" },
|
|
773
|
+
body: "not valid json",
|
|
774
|
+
});
|
|
775
|
+
|
|
776
|
+
expect([400, 500]).toContain(response.status);
|
|
777
|
+
});
|
|
778
|
+
});
|
|
779
|
+
|
|
780
|
+
describe("HTTP API - Artifact Serialization", () => {
|
|
781
|
+
let server: Subprocess | null = null;
|
|
782
|
+
const PORT = "3048";
|
|
783
|
+
const baseUrl = `http://localhost:${PORT}`;
|
|
784
|
+
|
|
785
|
+
beforeAll(async () => {
|
|
786
|
+
server = await startServer(PORT);
|
|
787
|
+
});
|
|
788
|
+
|
|
789
|
+
afterAll(() => {
|
|
790
|
+
server?.kill();
|
|
791
|
+
});
|
|
792
|
+
|
|
793
|
+
test("Parsed artifacts have correct structure", async () => {
|
|
794
|
+
const formData = new FormData();
|
|
795
|
+
const html = "<!DOCTYPE html><html><body><h1>Test</h1></body></html>";
|
|
796
|
+
const file = new File([html], "test.html", { type: "text/html" });
|
|
797
|
+
formData.append("file", file);
|
|
798
|
+
|
|
799
|
+
const response = await fetch(`${baseUrl}/parse`, {
|
|
800
|
+
method: "POST",
|
|
801
|
+
body: formData,
|
|
802
|
+
});
|
|
803
|
+
|
|
804
|
+
expect(response.status).toBe(200);
|
|
805
|
+
const data = await response.json();
|
|
806
|
+
expect(data).toHaveProperty("artifacts");
|
|
807
|
+
expect(Array.isArray(data.artifacts)).toBe(true);
|
|
808
|
+
|
|
809
|
+
for (const artifact of data.artifacts) {
|
|
810
|
+
expect(artifact).toHaveProperty("id");
|
|
811
|
+
expect(artifact).toHaveProperty("type");
|
|
812
|
+
expect(artifact).toHaveProperty("contents");
|
|
813
|
+
expect(Array.isArray(artifact.contents)).toBe(true);
|
|
814
|
+
|
|
815
|
+
for (const content of artifact.contents) {
|
|
816
|
+
if (content.text !== undefined) {
|
|
817
|
+
expect(typeof content.text).toBe("string");
|
|
818
|
+
}
|
|
819
|
+
if (content.page !== undefined) {
|
|
820
|
+
expect(typeof content.page).toBe("number");
|
|
821
|
+
}
|
|
822
|
+
if (content.media) {
|
|
823
|
+
expect(Array.isArray(content.media)).toBe(true);
|
|
824
|
+
for (const media of content.media) {
|
|
825
|
+
expect(media).toHaveProperty("type");
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
});
|
|
831
|
+
|
|
832
|
+
test("Artifacts with metadata are preserved", async () => {
|
|
833
|
+
const formData = new FormData();
|
|
834
|
+
const file = new File(["test"], "test.txt", { type: "text/plain" });
|
|
835
|
+
formData.append("file", file);
|
|
836
|
+
|
|
837
|
+
const response = await fetch(`${baseUrl}/parse`, {
|
|
838
|
+
method: "POST",
|
|
839
|
+
body: formData,
|
|
840
|
+
});
|
|
841
|
+
|
|
842
|
+
expect(response.status).toBe(200);
|
|
843
|
+
const data = await response.json();
|
|
844
|
+
|
|
845
|
+
for (const artifact of data.artifacts) {
|
|
846
|
+
if (artifact.metadata) {
|
|
847
|
+
expect(typeof artifact.metadata).toBe("object");
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
});
|
|
851
|
+
|
|
852
|
+
test("Large text files are parsed correctly", async () => {
|
|
853
|
+
const formData = new FormData();
|
|
854
|
+
const largeContent = "Line 1\n".repeat(1000);
|
|
855
|
+
const file = new File([largeContent], "large.txt", { type: "text/plain" });
|
|
856
|
+
formData.append("file", file);
|
|
857
|
+
|
|
858
|
+
const response = await fetch(`${baseUrl}/parse`, {
|
|
859
|
+
method: "POST",
|
|
860
|
+
body: formData,
|
|
861
|
+
});
|
|
862
|
+
|
|
863
|
+
expect(response.status).toBe(200);
|
|
864
|
+
const data = await response.json();
|
|
865
|
+
expect(data).toHaveProperty("artifacts");
|
|
866
|
+
expect(data.artifacts.length).toBeGreaterThan(0);
|
|
867
|
+
|
|
868
|
+
let totalText = "";
|
|
869
|
+
for (const artifact of data.artifacts) {
|
|
870
|
+
for (const content of artifact.contents) {
|
|
871
|
+
if (content.text) {
|
|
872
|
+
totalText += content.text;
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
expect(totalText.length).toBeGreaterThan(0);
|
|
877
|
+
});
|
|
878
|
+
});
|
|
879
|
+
|
|
880
|
+
describe("HTTP API - Request Validation", () => {
|
|
881
|
+
let server: Subprocess | null = null;
|
|
882
|
+
const PORT = "3049";
|
|
883
|
+
const baseUrl = `http://localhost:${PORT}`;
|
|
884
|
+
|
|
885
|
+
beforeAll(async () => {
|
|
886
|
+
server = await startServer(PORT);
|
|
887
|
+
});
|
|
888
|
+
|
|
889
|
+
afterAll(() => {
|
|
890
|
+
server?.kill();
|
|
891
|
+
});
|
|
892
|
+
|
|
893
|
+
test("POST /extract with empty artifacts array", async () => {
|
|
894
|
+
const response = await fetch(`${baseUrl}/extract`, {
|
|
895
|
+
method: "POST",
|
|
896
|
+
headers: { "Content-Type": "application/json" },
|
|
897
|
+
body: JSON.stringify({
|
|
898
|
+
artifacts: [],
|
|
899
|
+
schema: { type: "object" },
|
|
900
|
+
model: "openai/gpt-4",
|
|
901
|
+
}),
|
|
902
|
+
});
|
|
903
|
+
|
|
904
|
+
expect([200, 400, 500]).toContain(response.status);
|
|
905
|
+
});
|
|
906
|
+
|
|
907
|
+
test("POST /extract with malformed artifacts structure", async () => {
|
|
908
|
+
const response = await fetch(`${baseUrl}/extract`, {
|
|
909
|
+
method: "POST",
|
|
910
|
+
headers: { "Content-Type": "application/json" },
|
|
911
|
+
body: JSON.stringify({
|
|
912
|
+
artifacts: "not an array",
|
|
913
|
+
schema: { type: "object" },
|
|
914
|
+
model: "openai/gpt-4",
|
|
915
|
+
}),
|
|
916
|
+
});
|
|
917
|
+
|
|
918
|
+
expect([400, 500]).toContain(response.status);
|
|
919
|
+
});
|
|
920
|
+
|
|
921
|
+
test("POST /extract with valid artifacts structure", async () => {
|
|
922
|
+
const response = await fetch(`${baseUrl}/extract`, {
|
|
923
|
+
method: "POST",
|
|
924
|
+
headers: { "Content-Type": "application/json" },
|
|
925
|
+
body: JSON.stringify({
|
|
926
|
+
artifacts: [
|
|
927
|
+
{ id: "art1", type: "text", contents: [{ text: "content 1" }] },
|
|
928
|
+
{ id: "art2", type: "html", contents: [{ text: "content 2", page: 1 }] },
|
|
929
|
+
],
|
|
930
|
+
schema: { type: "object", properties: { items: { type: "array" } } },
|
|
931
|
+
model: "openai/gpt-4",
|
|
932
|
+
strategy: "simple",
|
|
933
|
+
}),
|
|
934
|
+
});
|
|
935
|
+
|
|
936
|
+
expect([200, 500]).toContain(response.status);
|
|
937
|
+
});
|
|
938
|
+
});
|