@struktur/sdk 0.1.0

package/README.md

@@ -0,0 +1,79 @@
+<picture height="0">
+  <source media="(min-width: 769px)" srcset="https://raw.githubusercontent.com/mateffy/struktur/main/resources/pixel.png" width="0" height="0">
+  <img src="https://raw.githubusercontent.com/mateffy/struktur/main/resources/struktur-icon-padded.webp" alt="Struktur Logo" width="150">
+</picture>
+
+<div>
+    <h1>
+        <picture>
+            <source media="(max-width: 768px)" srcset="https://raw.githubusercontent.com/mateffy/struktur/main/resources/pixel.png" width="0" height="0">
+            <img src="https://raw.githubusercontent.com/mateffy/struktur/main/resources/struktur-icon-padded.webp" alt="Struktur Logo" width="225" align="left">
+        </picture>
+        Struktur SDK
+    </h1>
+    <p>
+      All-in-one tool for structured data extraction using LLMs. Feed it documents, get back validated JSON. Handles parsing files, chunking, retries, merging, and deduplication — you just define the schema and choose a strategy. <br /><br />
+        <a href="https://struktur.sh/docs/quickstart" target="_blank">Quickstart</a> |
+        <a href="https://struktur.sh/docs" target="_blank">Documentation</a>
+    </p>
+</div>
+
+<br />
+<br />
+
+## @struktur/sdk
+
+The TypeScript SDK for Struktur — structured data extraction using the Vercel AI SDK.
+
+## Installation
+
+```bash
+npm install @struktur/sdk
+# or
+bun add @struktur/sdk
+```
+
+## Quick Example
+
+```ts
+import { extract, simple, urlToArtifact } from "@struktur/sdk";
+import { openai } from "@ai-sdk/openai";
+import type { JSONSchemaType } from "ajv";
+
+type Invoice = { number: string; vendor: string; total: number };
+
+const schema: JSONSchemaType<Invoice> = {
+  type: "object",
+  properties: {
+    number: { type: "string" },
+    vendor: { type: "string" },
+    total: { type: "number" },
+  },
+  required: ["number", "vendor", "total"],
+  additionalProperties: false,
+};
+
+const artifact = await urlToArtifact("https://example.com/invoice.pdf");
+
+const result = await extract({
+  artifacts: [artifact],
+  schema,
+  strategy: simple({ model: openai("gpt-4o-mini") }),
+});
+
+console.log(result.data.number); // fully typed
+console.log(result.usage.totalTokens);
+```
+
+## Documentation
+
+Full documentation at **[struktur.sh](https://struktur.sh)**
+
+- [SDK Reference](https://struktur.sh/docs/sdk)
+- [API Documentation](https://struktur.sh/docs/sdk/extract)
+- [Strategies](https://struktur.sh/docs/explanation/strategies)
+- [Examples](https://struktur.sh/docs/examples)
+
+## Repository
+
+This package is part of the [Struktur monorepo](https://github.com/mateffy/struktur).

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@struktur/sdk",
+  "version": "0.1.0",
+  "type": "module",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "exports": {
+    ".": "./src/index.ts",
+    "./strategies": "./src/strategies/index.ts",
+    "./artifacts": "./src/artifacts/index.ts",
+    "./parsers": "./src/parsers/index.ts",
+    "./auth": "./src/auth/index.ts",
+    "./llm": "./src/llm/index.ts",
+    "./validation": "./src/validation/index.ts",
+    "./debug": "./src/debug/index.ts"
+  },
+  "dependencies": {
+    "@ai-sdk/anthropic": "^3.0.0",
+    "@ai-sdk/google": "^3.0.0",
+    "@ai-sdk/openai": "^3.0.0",
+    "@openrouter/ai-sdk-provider": "^2.0.0",
+    "ai": "^6.0.97",
+    "ajv": "^8.17.1",
+    "ajv-formats": "^3.0.1",
+    "pdf-parse": "^2.4.5"
+  },
+  "peerDependencies": {
+    "typescript": "^5"
+  },
+  "devDependencies": {
+    "@types/bun": "latest"
+  }
+}

package/src/artifacts/AGENTS.md

@@ -0,0 +1,16 @@
+Artifacts module
+
+- Purpose: convert external inputs into Artifact DTOs and manage parser configuration.
+- Key files: `fileToArtifact.ts`, `urlToArtifact.ts`, `providers.ts`, `input.ts`.
+- Design: `ParsersConfig` is the unified configuration system for all parsers (npm packages, CLI commands, and inline handlers). The `providers` registry is deprecated — use inline parsers in `ParsersConfig` instead.
+- `parse` accepts an optional `parserConfig: ParsersConfig` (from `src/parsers/types`) that takes priority over the deprecated providers registry. Custom parsers are resolved by MIME type.
+- `ParsersConfig` supports four parser types:
+  - `NpmParserDef` — npm package with `parseFile` or `parseStream` exports
+  - `CommandFileDef` — CLI command with `FILE_PATH` placeholder
+  - `CommandStdinDef` — CLI command that reads from stdin
+  - `InlineParserDef` — inline `(buffer: Buffer) => Promise<Artifact>` function (replaces the old providers registry)
+- JSON auto-detection: when MIME type is `application/json`, `fileParser` first attempts to validate the file as `SerializedArtifact[]`. If valid, it hydrates and returns them directly. If not valid, it checks `parsers` config for a custom parser; if none, throws a clear error.
+- `parseBufferInput` resolution order: (1) parsers config, (2) providers registry (deprecated), (3) JSON auto-detection, (4) built-in `application/pdf` (via `parsePdf`), (5) built-in text/*, (6) built-in image/*, (7) error.
+- `parse` accepts `includeImages?: boolean` which is forwarded to `parsePdf` for PDF inputs. Set to `false` to suppress image extraction (used by `--no-images` CLI flag).
+- `SerializedArtifactImage` schema validation accepts an optional `imageType` field with values `"embedded"` or `"screenshot"` to differentiate between embedded images extracted from PDFs and page screenshots.
+- Tests: `fileToArtifact.test.ts`, `urlToArtifact.test.ts`, `input.test.ts`, `providers.test.ts`.

package/src/artifacts/fileToArtifact.test.ts

@@ -0,0 +1,37 @@
+import { test, expect } from "bun:test";
+import { fileToArtifact } from "./fileToArtifact";
+import type { ArtifactType } from "../types";
+
+test("fileToArtifact uses custom providers", async () => {
+  const providers = {
+    "text/plain": async (buffer: Buffer): Promise<{ id: string; type: ArtifactType; raw: () => Promise<Buffer>; contents: { text: string }[] }> => ({
+      id: "a1",
+      type: "text",
+      raw: async () => buffer,
+      contents: [{ text: buffer.toString() }],
+    }),
+  };
+
+  const artifact = await fileToArtifact(Buffer.from("hello"), {
+    mimeType: "text/plain",
+    providers,
+  });
+
+  expect(artifact.id).toBe("a1");
+  expect(artifact.contents[0]?.text).toBe("hello");
+});
+
+test("fileToArtifact falls back to text for text/* mime types", async () => {
+  const artifact = await fileToArtifact(Buffer.from("hello world"), {
+    mimeType: "text/plain",
+  });
+
+  expect(artifact.type).toBe("text");
+  expect(artifact.contents[0]?.text).toBe("hello world");
+});
+
+test("fileToArtifact throws for unknown mime types", async () => {
+  await expect(
+    fileToArtifact(Buffer.from("test"), { mimeType: "application/unknown" })
+  ).rejects.toThrow("No artifact provider registered");
+});

package/src/artifacts/fileToArtifact.ts

@@ -0,0 +1,44 @@
+import type { Artifact } from "../types";
+import { defaultArtifactProviders, type ArtifactProviders } from "./providers";
+
+export type FileToArtifactOptions = {
+  mimeType: string;
+  providers?: ArtifactProviders;
+};
+
+const bufferToTextArtifact = (buffer: Buffer): Artifact => ({
+  id: `artifact-${crypto.randomUUID()}`,
+  type: "text",
+  raw: async () => buffer,
+  contents: [{ text: buffer.toString() }],
+});
+
+const bufferToImageArtifact = (buffer: Buffer): Artifact => ({
+  id: `artifact-${crypto.randomUUID()}`,
+  type: "image",
+  raw: async () => buffer,
+  contents: [{ media: [{ type: "image", contents: buffer }] }],
+});
+
+export const fileToArtifact = async (
+  buffer: Buffer,
+  options: FileToArtifactOptions
+): Promise<Artifact> => {
+  const providers = options.providers ?? defaultArtifactProviders;
+  const provider = providers[options.mimeType];
+  if (provider) {
+    return provider(buffer);
+  }
+
+  if (options.mimeType.startsWith("text/")) {
+    return bufferToTextArtifact(buffer);
+  }
+
+  if (options.mimeType.startsWith("image/")) {
+    return bufferToImageArtifact(buffer);
+  }
+
+  throw new Error(
+    `No artifact provider registered for ${options.mimeType}`
+  );
+};

package/src/artifacts/input.test.ts

@@ -0,0 +1,243 @@
+import { test, expect, mock } from "bun:test";
+import {
+  clearArtifactInputParsers,
+  parse,
+  splitTextIntoContents,
+  validateSerializedArtifacts,
+} from "./input";
+
+// ---------------------------------------------------------------------------
+// Stub pdf-parse so the built-in PDF path in parseBufferInput can be tested
+// without a real PDF file.
+// ---------------------------------------------------------------------------
+mock.module("pdf-parse", () => ({
+  PDFParse: class {
+    constructor(_opts: unknown) {}
+    async getText() {
+      return { pages: [{ num: 1, text: "pdf text" }], text: "pdf text", total: 1 };
+    }
+    async getImage(_params?: unknown) {
+      return { pages: [], total: 0 };
+    }
+    async getInfo() {
+      return { Title: "Stub PDF" };
+    }
+    async destroy() {}
+  },
+}));
+
+test("validateSerializedArtifacts accepts a single artifact", () => {
+  const artifacts = validateSerializedArtifacts({
+    id: "a1",
+    type: "text",
+    contents: [{ text: "hello" }],
+  });
+
+  expect(artifacts).toHaveLength(1);
+  expect(artifacts[0]?.id).toBe("a1");
+});
+
+test("validateSerializedArtifacts rejects invalid artifacts", () => {
+  expect(() =>
+    validateSerializedArtifacts({
+      type: "text",
+      contents: [],
+    })
+  ).toThrow("Schema validation failed");
+});
+
+test("splitTextIntoContents splits paragraphs", () => {
+  const contents = splitTextIntoContents("one\n\nTwo\n\nthree");
+  expect(contents).toHaveLength(3);
+  expect(contents[1]?.text).toBe("Two");
+});
+
+test("parseInputToArtifacts builds text artifacts", async () => {
+  clearArtifactInputParsers();
+  const artifacts = await parse({
+    kind: "text",
+    text: "hello\n\nworld",
+    id: "t1",
+  });
+
+  expect(artifacts).toHaveLength(1);
+  expect(artifacts[0]?.id).toBe("t1");
+  expect(artifacts[0]?.contents).toHaveLength(2);
+});
+
+test("parse uses providers when available", async () => {
+  const providers = {
+    "application/pdf": async () => ({
+      id: "pdf-1",
+      type: "pdf" as const,
+      raw: async () => Buffer.from("pdf"),
+      contents: [{ text: "from-provider" }],
+    }),
+  };
+
+  const artifacts = await parse({
+    kind: "buffer",
+    buffer: Buffer.from("data"),
+    mimeType: "application/pdf",
+  }, { providers });
+
+  expect(artifacts[0]?.type).toBe("pdf");
+  expect(artifacts[0]?.contents[0]?.text).toBe("from-provider");
+});
+
+test("parse builds image artifacts", async () => {
+  clearArtifactInputParsers();
+  const buffer = Buffer.from([1, 2, 3]);
+  const artifacts = await parse({
+    kind: "buffer",
+    buffer,
+    mimeType: "image/png",
+    id: "img-1",
+  });
+
+  const media = artifacts[0]?.contents[0]?.media?.[0];
+  expect(artifacts[0]?.type).toBe("image");
+  expect(media?.type).toBe("image");
+  expect((media?.contents as Buffer).equals(buffer)).toBe(true);
+});
+
+test("parse uses parserConfig over providers", async () => {
+  const providers = {
+    "application/pdf": async () => ({
+      id: "from-provider",
+      type: "pdf" as const,
+      raw: async () => Buffer.from("pdf"),
+      contents: [{ text: "from-provider" }],
+    }),
+  };
+
+  const artifactJson = JSON.stringify([{
+    id: "from-parser",
+    type: "text",
+    contents: [{ text: "from-parser" }],
+  }]);
+  const parserConfig = {
+    "application/pdf": {
+      type: "command-stdin" as const,
+      command: `echo ${JSON.stringify(artifactJson)}`,
+    },
+  };
+
+  const artifacts = await parse(
+    {
+      kind: "buffer",
+      buffer: Buffer.from("data"),
+      mimeType: "application/pdf",
+    },
+    { providers, parserConfig }
+  );
+
+  expect(artifacts[0]?.id).toBe("from-parser");
+  expect(artifacts[0]?.contents[0]?.text).toBe("from-parser");
+});
+
+test("parse auto-detects JSON artifacts (buffer)", async () => {
+  clearArtifactInputParsers();
+  const serialized = [{ id: "j1", type: "text" as const, contents: [{ text: "json artifact" }] }];
+  const buffer = Buffer.from(JSON.stringify(serialized));
+
+  const artifacts = await parse({
+    kind: "buffer",
+    buffer,
+    mimeType: "application/json",
+  });
+
+  expect(artifacts).toHaveLength(1);
+  expect(artifacts[0]?.id).toBe("j1");
+  expect(artifacts[0]?.contents[0]?.text).toBe("json artifact");
+});
+
+test("parse throws for non-artifact JSON with no custom parser", async () => {
+  clearArtifactInputParsers();
+  const buffer = Buffer.from(JSON.stringify({ some: "object" }));
+
+  await expect(
+    parse({
+      kind: "buffer",
+      buffer,
+      mimeType: "application/json",
+    })
+  ).rejects.toThrow("not in SerializedArtifact format");
+});
+
+// ─── built-in PDF path ───────────────────────────────────────────────────────
+
+test("parse routes application/pdf to built-in parsePdf", async () => {
+  clearArtifactInputParsers();
+  const pdfBuffer = Buffer.from("%PDF-1.4 fake");
+
+  const artifacts = await parse({
+    kind: "buffer",
+    buffer: pdfBuffer,
+    mimeType: "application/pdf",
+  });
+
+  expect(artifacts).toHaveLength(1);
+  expect(artifacts[0]?.type).toBe("pdf");
+  expect(artifacts[0]?.contents[0]?.text).toBe("pdf text");
+});
+
+test("parse passes includeImages: false to parsePdf", async () => {
+  clearArtifactInputParsers();
+  const pdfBuffer = Buffer.from("%PDF-1.4 fake");
+
+  // With includeImages: false the result should still be a pdf artifact;
+  // we can't easily observe the absence of images here (the stub returns none
+  // anyway) but we verify the call succeeds and type is correct.
+  const artifacts = await parse(
+    {
+      kind: "buffer",
+      buffer: pdfBuffer,
+      mimeType: "application/pdf",
+    },
+    { includeImages: false }
+  );
+
+  expect(artifacts).toHaveLength(1);
+  expect(artifacts[0]?.type).toBe("pdf");
+});
+
+test("validateSerializedArtifacts accepts imageType field in media", () => {
+  const artifacts = validateSerializedArtifacts({
+    id: "a1",
+    type: "pdf",
+    contents: [
+      {
+        page: 1,
+        text: "page text",
+        media: [
+          { type: "image", base64: "abc123", imageType: "embedded" },
+          { type: "image", base64: "def456", imageType: "screenshot" },
+        ],
+      },
+    ],
+  });
+
+  expect(artifacts).toHaveLength(1);
+  expect(artifacts[0]?.contents[0]?.media).toHaveLength(2);
+  expect(artifacts[0]?.contents[0]?.media![0]?.imageType).toBe("embedded");
+  expect(artifacts[0]?.contents[0]?.media![1]?.imageType).toBe("screenshot");
+});
+
+test("validateSerializedArtifacts accepts media without imageType (optional field)", () => {
+  const artifacts = validateSerializedArtifacts({
+    id: "a1",
+    type: "pdf",
+    contents: [
+      {
+        page: 1,
+        text: "page text",
+        media: [{ type: "image", base64: "abc123" }],
+      },
+    ],
+  });
+
+  expect(artifacts).toHaveLength(1);
+  expect(artifacts[0]?.contents[0]?.media).toHaveLength(1);
+  expect(artifacts[0]?.contents[0]?.media![0]?.imageType).toBeUndefined();
+});