@struktur/sdk 2.1.2 → 2.2.0

package/src/parsers/AGENTS.md

@@ -1,58 +0,0 @@
-# Parsers module
-
-- Purpose: detect MIME types, run external/npm/command parsers, and provide built-in PDF support.
-- Key files: `types.ts`, `collect.ts`, `mime.ts`, `npm.ts`, `runner.ts`, `pdf.ts`, `index.ts`.
-
-## Types (`types.ts`)
-
-- `NpmParserDef` — npm package parser definition (`type: "npm"`, `package: string`)
-- `CommandFileDef` — command with `FILE_PATH` placeholder (`type: "command-file"`, `command: string`)
-- `CommandStdinDef` — command that reads from stdin (`type: "command-stdin"`, `command: string`)
-- `InlineParserDef` — inline handler function (`type: "inline"`, `handler: (buffer: Buffer) => Promise<Artifact>`)
-- `ParserDef` — union of the four variants
-- `ParsersConfig` — `Record<string, ParserDef>` keyed by MIME type
-- `ParserInput` — `{ kind: "file"; path: string } | { kind: "buffer"; buffer: Buffer }`
-
-## npm Contract (`npm.ts`)
-
-- `ParseStreamFn`, `ParseFileFn`, `DetectFileTypeFn`, `NpmParserModule` — interfaces that npm parser packages must implement.
-- At least one of `parseStream` or `parseFile` must be exported.
-
-## collectStream (`collect.ts`)
-
-- `collectStream(stream: ReadableStream<Uint8Array>): Promise<Buffer>` — public utility for npm parser authors to collect a stream into a Buffer.
-
-## MIME Detection (`mime.ts`)
-
-Two-layer detection + npm detectFileType callbacks:
-
-1. **Magic bytes** (authoritative): PDF, PNG, JPEG, GIF, WebP, ZIP/Office
-2. **npm `detectFileType`**: called after magic bytes with first 512 bytes
-3. **Extension database**: fallback when no magic bytes match (file inputs only)
-
-`detectMimeType({ buffer?, filePath?, mimeOverride?, npmParsers? }): Promise<string | null>`
-
-## Runner (`runner.ts`)
-
-`runParser(def: ParserDef, input: ParserInput, mimeType: string): Promise<Artifact[]>`
-
-- **npm**: Dynamic import, prefer `parseFile` for file inputs (zero-copy), prefer `parseStream` for buffer inputs. Falls back via temp-file if needed.
-- **command-file**: Interpolates `FILE_PATH` in command, writes temp file for buffer inputs.
-- **command-stdin**: Pipes input buffer to subprocess stdin; captures stdout as `SerializedArtifact[]` JSON.
-- **inline**: Calls the handler function directly with the buffer (reads file into buffer if needed).
-
-## Built-in PDF Parser (`pdf.ts`)
-
-`parsePdf(input: Buffer | ReadableStream<Uint8Array>, options?: ParsePdfOptions): Promise<Artifact>`
-
-Uses `pdf-parse` (npm package). Extracts per-page text **and** embedded images into `ArtifactContent[]`
-with `page` numbers set. Returns an `Artifact` with `type: "pdf"`.
-
-- Text extraction: per-page via `parser.getText()`; falls back to full document text when no per-page info is available.
-- Image extraction: per-page via `parser.getImage({ imageBuffer: false, imageDataUrl: true })`. Each embedded image is mapped to an `ArtifactImage` with `base64` (raw base64 string, data-URL prefix stripped), `width`, `height`, and `imageType: "embedded"`. Images are merged into the `media` array of the matching `ArtifactContent` entry. Pages that contain images but no text produce their own content entry. Image extraction failure is non-fatal — the parser continues and returns text-only content.
-- Screenshot rendering: per-page via `parser.getScreenshot()`. Each page is rendered to a PNG image and added to the `media` array with `imageType: "screenshot"`. Screenshots are appended to any embedded images for the same page. Screenshot rendering failure is non-fatal — the parser continues without screenshots.
-- `imageThreshold` defaults to 80 px (from pdf-parse), filtering out tiny decorative images.
-- `ParsePdfOptions.includeImages` (default `true`): set to `false` to skip `getImage()` entirely and return text-only content. This is used by the `--no-images` CLI flag.
-- `ParsePdfOptions.screenshots` (default `false`): set to `true` to render page screenshots and include them as images. This is used by the `--screenshots` CLI flag.
-- `ParsePdfOptions.screenshotScale` (default `1.5`): scale factor for screenshot rendering. Higher values produce larger, higher-quality images.
-- `ParsePdfOptions.screenshotWidth`: target width in pixels for screenshots. If specified, takes precedence over `screenshotScale` and height is calculated to maintain aspect ratio.

package/src/parsers/collect.test.ts

@@ -1,56 +0,0 @@
-import { test, expect } from "bun:test";
-import { collectStream } from "./collect";
-
-test("collectStream collects single chunk", async () => {
-  const data = Buffer.from("hello world");
-  const stream = new ReadableStream<Uint8Array>({
-    start(controller) {
-      controller.enqueue(data);
-      controller.close();
-    },
-  });
-
-  const result = await collectStream(stream);
-  expect(result.toString()).toBe("hello world");
-});
-
-test("collectStream collects multiple chunks", async () => {
-  const chunks = [Buffer.from("foo"), Buffer.from("bar"), Buffer.from("baz")];
-  const stream = new ReadableStream<Uint8Array>({
-    start(controller) {
-      for (const chunk of chunks) {
-        controller.enqueue(chunk);
-      }
-      controller.close();
-    },
-  });
-
-  const result = await collectStream(stream);
-  expect(result.toString()).toBe("foobarbaz");
-});
-
-test("collectStream returns Buffer", async () => {
-  const stream = new ReadableStream<Uint8Array>({
-    start(controller) {
-      controller.enqueue(new Uint8Array([1, 2, 3]));
-      controller.close();
-    },
-  });
-
-  const result = await collectStream(stream);
-  expect(Buffer.isBuffer(result)).toBe(true);
-  expect(result[0]).toBe(1);
-  expect(result[1]).toBe(2);
-  expect(result[2]).toBe(3);
-});
-
-test("collectStream handles empty stream", async () => {
-  const stream = new ReadableStream<Uint8Array>({
-    start(controller) {
-      controller.close();
-    },
-  });
-
-  const result = await collectStream(stream);
-  expect(result.length).toBe(0);
-});

package/src/parsers/collect.ts

@@ -1,31 +0,0 @@
-/**
- * Collects a ReadableStream<Uint8Array> into a Buffer.
- * Uses Web Streams API — compatible with Bun and Node 18+.
- * Exported as a public utility for npm parser authors.
- */
-export async function collectStream(stream: ReadableStream<Uint8Array>): Promise<Buffer> {
-  const reader = stream.getReader();
-  const chunks: Uint8Array[] = [];
-
-  try {
-    while (true) {
-      const { done, value } = await reader.read();
-      if (done) {
-        break;
-      }
-      chunks.push(value);
-    }
-  } finally {
-    reader.releaseLock();
-  }
-
-  const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
-  const result = new Uint8Array(totalLength);
-  let offset = 0;
-  for (const chunk of chunks) {
-    result.set(chunk, offset);
-    offset += chunk.length;
-  }
-
-  return Buffer.from(result);
-}

package/src/parsers/index.ts

@@ -1,6 +0,0 @@
-export type { ParserDef, ParsersConfig, NpmParserDef, CommandFileDef, CommandStdinDef, InlineParserDef, ParserInput } from "./types";
-export { runParser } from "./runner";
-export { detectMimeType } from "./mime";
-export { collectStream } from "./collect";
-export { parsePdf } from "./pdf";
-export type { ParsePdfOptions } from "./pdf";

package/src/parsers/mime.test.ts

@@ -1,91 +0,0 @@
-import { test, expect } from "bun:test";
-import { detectMimeType } from "./mime";
-
-test("detectMimeType returns mimeOverride when provided", async () => {
-  const result = await detectMimeType({ mimeOverride: "application/pdf" });
-  expect(result).toBe("application/pdf");
-});
-
-test("detectMimeType detects PDF from magic bytes", async () => {
-  const pdfHeader = Buffer.from([0x25, 0x50, 0x44, 0x46, 0x2d, 0x31, 0x2e, 0x34]);
-  const result = await detectMimeType({ buffer: pdfHeader });
-  expect(result).toBe("application/pdf");
-});
-
-test("detectMimeType detects PNG from magic bytes", async () => {
-  const pngHeader = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
-  const result = await detectMimeType({ buffer: pngHeader });
-  expect(result).toBe("image/png");
-});
-
-test("detectMimeType detects JPEG from magic bytes", async () => {
-  const jpegHeader = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]);
-  const result = await detectMimeType({ buffer: jpegHeader });
-  expect(result).toBe("image/jpeg");
-});
-
-test("detectMimeType detects GIF from magic bytes", async () => {
-  const gifHeader = Buffer.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61]);
-  const result = await detectMimeType({ buffer: gifHeader });
-  expect(result).toBe("image/gif");
-});
-
-test("detectMimeType detects WebP from magic bytes", async () => {
-  const webpHeader = Buffer.alloc(12);
-  // RIFF at offset 0
-  webpHeader[0] = 0x52; webpHeader[1] = 0x49; webpHeader[2] = 0x46; webpHeader[3] = 0x46;
-  // WEBP at offset 8
-  webpHeader[8] = 0x57; webpHeader[9] = 0x45; webpHeader[10] = 0x42; webpHeader[11] = 0x50;
-  const result = await detectMimeType({ buffer: webpHeader });
-  expect(result).toBe("image/webp");
-});
-
-test("detectMimeType falls back to extension lookup for .txt", async () => {
-  const result = await detectMimeType({ filePath: "/some/file.txt" });
-  expect(result).toBe("text/plain");
-});
-
-test("detectMimeType falls back to extension lookup for .md", async () => {
-  const result = await detectMimeType({ filePath: "README.md" });
-  expect(result).toBe("text/markdown");
-});
-
-test("detectMimeType falls back to extension lookup for .json", async () => {
-  const result = await detectMimeType({ filePath: "data.json" });
-  expect(result).toBe("application/json");
-});
-
-test("detectMimeType falls back to extension lookup for .docx", async () => {
-  const result = await detectMimeType({ filePath: "doc.docx" });
-  expect(result).toBe("application/vnd.openxmlformats-officedocument.wordprocessingml.document");
-});
-
-test("detectMimeType returns null for unknown extension with no magic bytes", async () => {
-  const result = await detectMimeType({ filePath: "file.xyz" });
-  expect(result).toBeNull();
-});
-
-test("detectMimeType returns null with no inputs", async () => {
-  const result = await detectMimeType({});
-  expect(result).toBeNull();
-});
-
-test("detectMimeType mimeOverride takes precedence over magic bytes", async () => {
-  const pdfHeader = Buffer.from([0x25, 0x50, 0x44, 0x46]);
-  const result = await detectMimeType({
-    buffer: pdfHeader,
-    mimeOverride: "text/plain",
-  });
-  expect(result).toBe("text/plain");
-});
-
-test("detectMimeType extension takes precedence over null buffer detection", async () => {
-  // Buffer with no magic bytes, but file has .pdf extension
-  const randomBuffer = Buffer.from([0x00, 0x01, 0x02, 0x03]);
-  const result = await detectMimeType({
-    buffer: randomBuffer,
-    filePath: "document.pdf",
-  });
-  // magic bytes don't match, falls back to extension
-  expect(result).toBe("application/pdf");
-});

package/src/parsers/mime.ts

@@ -1,137 +0,0 @@
-import path from "node:path";
-import type { NpmParserDef } from "./types";
-
-// Magic byte signatures for common file types
-const MAGIC_BYTES: Array<{ mimeType: string; bytes: number[]; offset?: number }> = [
-  // PDF: %PDF
-  { mimeType: "application/pdf", bytes: [0x25, 0x50, 0x44, 0x46] },
-  // PNG: 89 50 4E 47
-  { mimeType: "image/png", bytes: [0x89, 0x50, 0x4e, 0x47] },
-  // JPEG: FF D8 FF
-  { mimeType: "image/jpeg", bytes: [0xff, 0xd8, 0xff] },
-  // GIF: GIF8
-  { mimeType: "image/gif", bytes: [0x47, 0x49, 0x46, 0x38] },
-  // ZIP / Office Open XML (DOCX/XLSX/PPTX all start with PK\x03\x04)
-  {
-    mimeType: "application/zip",
-    bytes: [0x50, 0x4b, 0x03, 0x04],
-  },
-];
-
-// WebP has RIFF at offset 0 and WEBP at offset 8
-const isWebP = (header: Uint8Array): boolean => {
-  if (header.length < 12) return false;
-  const riff =
-    header[0] === 0x52 && header[1] === 0x49 && header[2] === 0x46 && header[3] === 0x46;
-  const webp =
-    header[8] === 0x57 && header[9] === 0x45 && header[10] === 0x42 && header[11] === 0x50;
-  return riff && webp;
-};
-
-const matchesMagicBytes = (header: Uint8Array, bytes: number[], offset = 0): boolean => {
-  if (header.length < offset + bytes.length) return false;
-  return bytes.every((b, i) => header[offset + i] === b);
-};
-
-const detectFromMagicBytes = (header: Uint8Array): string | null => {
-  if (isWebP(header)) return "image/webp";
-
-  for (const { mimeType, bytes, offset } of MAGIC_BYTES) {
-    if (matchesMagicBytes(header, bytes, offset ?? 0)) {
-      return mimeType;
-    }
-  }
-
-  return null;
-};
-
-// Extension → MIME type lookup
-const EXTENSION_MIME_MAP: Record<string, string> = {
-  ".txt": "text/plain",
-  ".md": "text/markdown",
-  ".markdown": "text/markdown",
-  ".html": "text/html",
-  ".htm": "text/html",
-  ".json": "application/json",
-  ".pdf": "application/pdf",
-  ".png": "image/png",
-  ".jpg": "image/jpeg",
-  ".jpeg": "image/jpeg",
-  ".gif": "image/gif",
-  ".webp": "image/webp",
-  ".csv": "text/csv",
-  ".xml": "application/xml",
-  ".yaml": "application/yaml",
-  ".yml": "application/yaml",
-  ".docx":
-    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-  ".xlsx":
-    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-  ".pptx":
-    "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-  ".mp4": "video/mp4",
-  ".mp3": "audio/mpeg",
-  ".wav": "audio/wav",
-  ".ogg": "audio/ogg",
-  ".svg": "image/svg+xml",
-  ".ts": "text/plain",
-  ".tsx": "text/plain",
-  ".js": "text/javascript",
-  ".jsx": "text/javascript",
-  ".css": "text/css",
-  ".toml": "application/toml",
-};
-
-export type NpmParserEntry = {
-  mimeType: string;
-  def: NpmParserDef;
-};
-
-export async function detectMimeType(options: {
-  buffer?: Buffer;
-  filePath?: string;
-  mimeOverride?: string;
-  npmParsers?: NpmParserEntry[];
-}): Promise<string | null> {
-  const { buffer, filePath, mimeOverride, npmParsers } = options;
-
-  // --mime override takes precedence
-  if (mimeOverride) {
-    return mimeOverride;
-  }
-
-  // Layer 1: magic bytes (authoritative)
-  if (buffer && buffer.length > 0) {
-    const header = buffer.subarray(0, 512);
-    const magicMime = detectFromMagicBytes(header);
-    if (magicMime) {
-      return magicMime;
-    }
-
-    // Layer 3: npm parser detectFileType callbacks (after built-ins)
-    if (npmParsers && npmParsers.length > 0) {
-      for (const entry of npmParsers) {
-        try {
-          const mod = await import(entry.def.package) as {
-            detectFileType?: (header: Uint8Array) => boolean;
-          };
-          if (typeof mod.detectFileType === "function" && mod.detectFileType(header)) {
-            return entry.mimeType;
-          }
-        } catch {
-          // If the package fails to load, skip it
-        }
-      }
-    }
-  }
-
-  // Layer 2: extension database (for file inputs)
-  if (filePath) {
-    const ext = path.extname(filePath).toLowerCase();
-    if (ext && ext in EXTENSION_MIME_MAP) {
-      return EXTENSION_MIME_MAP[ext] ?? null;
-    }
-  }
-
-  return null;
-}

package/src/parsers/npm.ts

@@ -1,26 +0,0 @@
-import type { Artifact } from "../types";
-
-/**
- * Contract for npm parser packages.
- *
- * A parser package must export at least one of `parseStream` or `parseFile`.
- * `detectFileType` is optional.
- */
-
-// Parser receives a ReadableStream — no disk I/O needed
-export type ParseStreamFn = (
-  stream: ReadableStream<Uint8Array>,
-  mimeType: string,
-) => Promise<Artifact[]>;
-
-// Parser receives a file path — useful for libraries that only work with files
-export type ParseFileFn = (filePath: string, mimeType: string) => Promise<Artifact[]>;
-
-// Magic byte detection — optional; return true if this parser handles the given bytes
-export type DetectFileTypeFn = (header: Uint8Array) => boolean;
-
-export type NpmParserModule = {
-  parseStream?: ParseStreamFn;
-  parseFile?: ParseFileFn;
-  detectFileType?: DetectFileTypeFn;
-};