@struktur/http 2.6.0

package/AGENTS.md

@@ -0,0 +1,299 @@
+# HTTP Package
+
+HTTP API server for running Struktur headlessly.
+
+## Overview
+
+This package provides a simple HTTP API for parsing files and extracting structured data using Struktur. It runs on Bun with Hono and `hono-openapi` for auto-generated OpenAPI documentation.
+
+## File Structure
+
+```
+src/
+  index.ts              # Bun.serve() bootstrap
+  app.ts                # Hono app instance, middleware, route mounting
+  config.ts             # Environment variable loading
+  schemas.ts            # Zod schemas (Standard Schema compliant)
+  middleware/
+    auth.ts             # Bearer token auth with /openapi.json whitelist
+  routes/
+    info.ts             # GET /
+    parse.ts            # POST /parse
+    extract.ts          # POST /extract (SSE by default, JSON with `?sse=false`; dual-mode: JSON + multipart + form)
+    extract-stream.ts   # POST /extract/stream (convenience alias, always SSE)
+    client.ts           # GET /client (Scalar API client UI)
+    debug.ts            # GET /debug (simple HTML upload/debug UI)
+  utils/
+    serialize.ts        # Artifact serialization helpers
+    extraction.ts       # Shared extraction primitives (parseExtractRequest, resolveModelForEnv, createStrategy)
+```
+
+## Quick Start
+
+```bash
+# Start the server (port 3031 by default)
+bun run start
+
+# Or with hot reload for development
+bun run dev
+
+# With auth enabled
+API_KEY=secret-key bun run start
+
+# With a specific provider key
+OPENAI_API_KEY=sk-... bun run start
+```
+
+## API Client UI
+
+Open `http://localhost:3031/client` in your browser for an interactive Scalar API client. The documentation is also available at `http://localhost:3031/openapi.json`.
+
+## Endpoints
+
+### `GET /`
+
+Returns API info and available endpoints.
+
+### `GET /client`
+
+Interactive Scalar API client UI (auto-generated from the OpenAPI spec). No auth required.
+
+### `GET /debug`
+
+Simple HTML debug page for uploading files and visualizing extraction output in real-time. Submits to `/extract/stream` and displays all SSE events plus the final pretty-printed JSON result. No auth required. Uses Tailwind CSS via CDN.
+
+### `GET /openapi.json`
+
+OpenAPI 3.1.0 specification. No auth required.
+
+### `POST /parse`
+
+Parse uploaded files into artifact JSON.
+
+**Request:**
+- Content-Type: `multipart/form-data`
+- Fields:
+  - `file` (required): File to parse
+  - `images` (optional): Extract embedded images from documents (PDFs)
+  - `screenshots` (optional): Render page screenshots
+  - `screenshotScale` (optional): Scale factor for screenshots
+  - `screenshotWidth` (optional): Target width in pixels for screenshots
+
+**Response:**
+```json
+{
+  "artifacts": [...]
+}
+```
+
+### `POST /extract`
+
+Extract structured data from documents or artifact JSON.
+
+**Streaming behavior:** By default, returns SSE (`text/event-stream`). Disable streaming with `?sse=false` to get a plain JSON response.
+
+**Request (JSON):**
+```json
+{
+  "artifacts": [...],
+  "schema": {...},
+  "model": "openai/gpt-4.1-mini",
+  "strategy": "simple",
+  "chunkSize": 10000,
+  "strict": false
+}
+```
+
+**Request (multipart/form-data):**
+- `artifacts` (optional): Artifact JSON string
+- `file` (optional): File to parse (alternative to artifacts)
+- `schema` (optional): JSON schema string
+- `fields` (optional): Shorthand field list (alternative to schema)
+- `model` (required): Model identifier (e.g., `openai/gpt-4.1-mini`, `anthropic/claude-sonnet-4-6`)
+- `strategy` (optional): Extraction strategy (default: `simple`)
+- `chunkSize` (optional): Token budget per batch (default: 10000)
+- `maxSteps` (optional): Maximum agent steps for agent strategy
+- `strict` (optional): Strict schema validation
+- `images` (optional): Extract embedded images (when using file)
+- `screenshots` (optional): Render page screenshots (when using file)
+
+**Request (application/x-www-form-urlencoded):**
+Same fields as multipart, but `artifacts` and `schema` must be JSON strings. No file upload support.
+
+**Response (default SSE):** `text/event-stream` — see event types below.
+
+**Response (`?sse=false`):**
+```json
+{
+  "data": {...},
+  "usage": {
+    "inputTokens": 100,
+    "outputTokens": 50,
+    "totalTokens": 150
+  }
+}
+```
+
+### `POST /extract/stream`
+
+Convenience alias that always streams via SSE. Accepts the same request formats as `POST /extract`.
+
+**Response:** `text/event-stream`
+
+Each event is a JSON object with a `type` field:
+
+| Event type | Description |
+|-----------|-------------|
+| `step` | Extraction step started/completed |
+| `progress` | Batch progress (current/total/percent) |
+| `message` | LLM message sent/received |
+| `tokenUsage` | Token usage update |
+| `retry` | Retry attempt |
+| `agent_tool_start` | Agent tool invocation started |
+| `agent_tool_end` | Agent tool invocation completed |
+| `agent_message` | Agent message |
+| `agent_reasoning` | Agent reasoning/thought |
+| `complete` | Final result with `data` and `usage` |
+| `error` | Error message |
+
+**Example (curl):**
+```bash
+curl -N -X POST http://localhost:3031/extract \
+  -H "Content-Type: application/json" \
+  -d '{
+    "artifacts": [{"id":"1","type":"text","contents":[{"text":"test"}]}],
+    "schema": {"type":"object","properties":{"name":{"type":"string"}}},
+    "model": "openai/gpt-4o-mini"
+  }'
+```
+
+Disable SSE to get plain JSON:
+```bash
+curl -X POST "http://localhost:3031/extract?sse=false" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "artifacts": [{"id":"1","type":"text","contents":[{"text":"test"}]}],
+    "schema": {"type":"object","properties":{"name":{"type":"string"}}},
+    "model": "openai/gpt-4o-mini"
+  }'
+```
+
+## Example Requests
+
+### Parse a file
+
+```bash
+curl -X POST http://localhost:3031/parse \
+  -F "file=@document.pdf" \
+  -F "images=true"
+```
+
+### Extract with JSON body (pre-parsed artifacts)
+
+```bash
+curl -X POST http://localhost:3031/extract \
+  -H "Content-Type: application/json" \
+  -d '{
+    "artifacts": [{"id":"1","type":"text","contents":[{"text":"John Doe works at Acme Corp"}]}],
+    "schema": {
+      "type": "object",
+      "properties": {
+        "name": {"type": "string"},
+        "company": {"type": "string"}
+      }
+    },
+    "model": "openai/gpt-4o-mini",
+    "strategy": "simple"
+  }'
+```
+
+### Extract with file upload (parse + extract in one call)
+
+```bash
+curl -X POST http://localhost:3031/extract \
+  -F "file=@document.pdf" \
+  -F 'schema={"type":"object","properties":{"title":{"type":"string"}}}' \
+  -F "model=openai/gpt-4o-mini" \
+  -F "strategy=parallel" \
+  -F "chunkSize=5000"
+```
+
+### Stream extraction with SSE (default)
+
+```bash
+curl -N -X POST http://localhost:3031/extract \
+  -H "Content-Type: application/json" \
+  -d '{
+    "artifacts": [{"id":"1","type":"text","contents":[{"text":"test"}]}],
+    "schema": {"type":"object","properties":{"name":{"type":"string"}}},
+    "model": "openai/gpt-4o-mini"
+  }'
+```
+
+### Extract with JSON response (disable SSE)
+
+```bash
+curl -X POST "http://localhost:3031/extract?sse=false" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "artifacts": [{"id":"1","type":"text","contents":[{"text":"test"}]}],
+    "schema": {"type":"object","properties":{"name":{"type":"string"}}},
+    "model": "openai/gpt-4o-mini"
+  }'
+```
+
+### With authentication enabled
+
+```bash
+curl -X POST http://localhost:3031/extract \
+  -H "Authorization: Bearer secret-key" \
+  -H "Content-Type: application/json" \
+  -d '{"artifacts": [...], "schema": {...}, "model": "openai/gpt-4o-mini"}'
+```
+
+## Authentication
+
+If `API_KEY` environment variable is set, all requests must include:
+```
+Authorization: Bearer <api-key>
+```
+
+## Environment Variables
+
+- `API_KEY`: API key for authentication (optional)
+- `PORT`: Server port (default: 3031)
+- `OPENAI_API_KEY`: OpenAI API key
+- `ANTHROPIC_API_KEY`: Anthropic API key
+- `GOOGLE_API_KEY`: Google API key
+- `OPENCODE_API_KEY`: OpenCode API key
+- `OPENROUTER_API_KEY`: OpenRouter API key
+
+## Development
+
+```bash
+# Install dependencies
+bun install
+
+# Start development server with hot reload
+bun run dev
+
+# Start production server
+bun run start
+
+# Run tests
+bun test
+```
+
+## Supported Strategies
+
+- `simple`: Single-pass extraction
+- `parallel`: Parallel batch processing with merge
+- `sequential`: Sequential batch processing
+- `parallelAutoMerge`: Parallel with auto-deduplication
+- `sequentialAutoMerge`: Sequential with auto-deduplication
+- `doublePass`: Two-pass extraction with merge
+- `doublePassAutoMerge`: Two-pass with auto-deduplication
+- `agent`: Agent-based extraction (requires `maxSteps`)
+  - Uses a sandboxed emulated shell with only read/grep/glob file utilities
+  - No external HTTP calls or command execution
+  - No custom VM needed - runs safely in the same process

package/LICENSE

@@ -0,0 +1,110 @@
+# Functional Source License, Version 1.1, MIT Future License
+
+## Abbreviation
+
+FSL-1.1-MIT
+
+## Notice
+
+Copyright 2026 mateffy
+
+## Terms and Conditions
+
+### Licensor ("We")
+
+The party offering the Software under these Terms and Conditions.
+
+### The Software
+
+The "Software" is each version of the software that we make available under
+these Terms and Conditions, as indicated by our inclusion of these Terms and
+Conditions with the Software.
+
+### License Grant
+
+Subject to your compliance with this License Grant and the Patents,
+Redistribution and Trademark clauses below, we hereby grant you the right to
+use, copy, modify, create derivative works, publicly perform, publicly display
+and redistribute the Software for any Permitted Purpose identified below.
+
+### Permitted Purpose
+
+A Permitted Purpose is any purpose other than a Competing Use. A Competing Use
+means making the Software available to others in a commercial product or
+service that:
+
+1. substitutes for the Software;
+
+2. substitutes for any other product or service we offer using the Software
+   that exists as of the date we make the Software available; or
+
+3. offers the same or substantially similar functionality as the Software.
+
+Permitted Purposes specifically include using the Software:
+
+1. for your internal use and access;
+
+2. for non-commercial education;
+
+3. for non-commercial research; and
+
+4. in connection with professional services that you provide to a licensee
+   using the Software in accordance with these Terms and Conditions.
+
+### Patents
+
+To the extent your use for a Permitted Purpose would necessarily infringe our
+patents, the license grant above includes a license under our patents. If you
+make a claim against any party that the Software infringes or contributes to
+the infringement of any patent, then your patent license to the Software ends
+immediately.
+
+### Redistribution
+
+The Terms and Conditions apply to all copies, modifications and derivatives of
+the Software.
+
+If you redistribute any copies, modifications or derivatives of the Software,
+you must include a copy of or a link to these Terms and Conditions and not
+remove any copyright notices provided in or with the Software.
+
+### Disclaimer
+
+THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR
+PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT.
+
+IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE
+SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES,
+EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE.
+
+### Trademarks
+
+Except for displaying the License Details and identifying us as the origin of
+the Software, you have no right under these Terms and Conditions to use our
+trademarks, trade names, service marks or product names.
+
+## Grant of Future License
+
+We hereby irrevocably grant you an additional license to use the Software under
+the MIT license that is effective on the second anniversary of the date we make
+the Software available. On or after that date, you may use the Software under
+the MIT license, in which case the following will apply:
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@struktur/http",
+  "version": "2.6.0",
+  "license": "FSL-1.1-MIT",
+  "type": "module",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "dependencies": {
+    "@ai-sdk/anthropic": "^3.0.0",
+    "@ai-sdk/google": "^3.0.0",
+    "@ai-sdk/openai": "^3.0.0",
+    "@hono/node-server": "^2.0.0",
+    "@hono/standard-validator": "^0.2.2",
+    "@openrouter/ai-sdk-provider": "^2.0.0",
+    "@scalar/hono-api-reference": "^0.10.10",
+    "hono": "^4.6.0",
+    "hono-openapi": "^1.3.0",
+    "zod": "^4.3.6",
+    "@struktur/sdk": "2.6.0"
+  },
+  "devDependencies": {
+    "@types/bun": "latest",
+    "typescript": "^5"
+  },
+  "scripts": {
+    "dev": "bun --hot src/index.ts",
+    "start": "bun src/index.ts",
+    "test": "bun test"
+  }
+}

package/src/app.ts

@@ -0,0 +1,52 @@
+import { Hono } from "hono";
+import { cors } from "hono/cors";
+import { openAPIRouteHandler } from "hono-openapi";
+import { authMiddleware } from "./middleware/auth";
+import infoApp from "./routes/info";
+import parseApp from "./routes/parse";
+import extractApp from "./routes/extract";
+import extractStreamApp from "./routes/extract-stream";
+import clientApp from "./routes/client";
+import debugApp from "./routes/debug";
+import packageJson from "../package.json" with { type: "json" };
+
+const app = new Hono();
+
+app.use("*", cors());
+app.use("*", authMiddleware);
+
+// Mount sub-routes
+app.route("/", infoApp);
+app.route("/", parseApp);
+app.route("/", extractApp);
+app.route("/", extractStreamApp);
+app.route("/client", clientApp);
+app.route("/debug", debugApp);
+
+// OpenAPI documentation
+app.get(
+  "/openapi.json",
+  openAPIRouteHandler(app, {
+    documentation: {
+      info: {
+        title: "Struktur HTTP API",
+        version: packageJson.version,
+        description:
+          "HTTP API for running Struktur headlessly. Parse files into artifacts and extract structured data using LLMs.",
+      },
+      servers: [
+        {
+          url: "http://localhost:3031",
+          description: "Local development server",
+        },
+      ],
+      tags: [
+        { name: "Info", description: "API information" },
+        { name: "Parse", description: "File parsing operations" },
+        { name: "Extract", description: "Data extraction operations" },
+      ],
+    },
+  }),
+);
+
+export { app };

package/src/config.ts

@@ -0,0 +1,9 @@
+export const config = {
+  API_KEY: process.env.API_KEY || "",
+  PORT: parseInt(process.env.PORT || "3031"),
+  OPENAI_API_KEY: process.env.OPENAI_API_KEY,
+  ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
+  GOOGLE_API_KEY: process.env.GOOGLE_API_KEY,
+  OPENCODE_API_KEY: process.env.OPENCODE_API_KEY,
+  OPENROUTER_API_KEY: process.env.OPENROUTER_API_KEY,
+} as const;