@struktur/sdk 0.1.0

package/src/parsers/collect.test.ts

@@ -0,0 +1,56 @@
+import { test, expect } from "bun:test";
+import { collectStream } from "./collect";
+
+test("collectStream collects single chunk", async () => {
+  const data = Buffer.from("hello world");
+  const stream = new ReadableStream<Uint8Array>({
+    start(controller) {
+      controller.enqueue(data);
+      controller.close();
+    },
+  });
+
+  const result = await collectStream(stream);
+  expect(result.toString()).toBe("hello world");
+});
+
+test("collectStream collects multiple chunks", async () => {
+  const chunks = [Buffer.from("foo"), Buffer.from("bar"), Buffer.from("baz")];
+  const stream = new ReadableStream<Uint8Array>({
+    start(controller) {
+      for (const chunk of chunks) {
+        controller.enqueue(chunk);
+      }
+      controller.close();
+    },
+  });
+
+  const result = await collectStream(stream);
+  expect(result.toString()).toBe("foobarbaz");
+});
+
+test("collectStream returns Buffer", async () => {
+  const stream = new ReadableStream<Uint8Array>({
+    start(controller) {
+      controller.enqueue(new Uint8Array([1, 2, 3]));
+      controller.close();
+    },
+  });
+
+  const result = await collectStream(stream);
+  expect(Buffer.isBuffer(result)).toBe(true);
+  expect(result[0]).toBe(1);
+  expect(result[1]).toBe(2);
+  expect(result[2]).toBe(3);
+});
+
+test("collectStream handles empty stream", async () => {
+  const stream = new ReadableStream<Uint8Array>({
+    start(controller) {
+      controller.close();
+    },
+  });
+
+  const result = await collectStream(stream);
+  expect(result.length).toBe(0);
+});

package/src/parsers/collect.ts

@@ -0,0 +1,31 @@
+/**
+ * Collects a ReadableStream<Uint8Array> into a Buffer.
+ * Uses Web Streams API — compatible with Bun and Node 18+.
+ * Exported as a public utility for npm parser authors.
+ */
+export async function collectStream(stream: ReadableStream<Uint8Array>): Promise<Buffer> {
+  const reader = stream.getReader();
+  const chunks: Uint8Array[] = [];
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) {
+        break;
+      }
+      chunks.push(value);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+  const result = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    result.set(chunk, offset);
+    offset += chunk.length;
+  }
+
+  return Buffer.from(result);
+}

package/src/parsers/index.ts

@@ -0,0 +1,6 @@
+export type { ParserDef, ParsersConfig, NpmParserDef, CommandFileDef, CommandStdinDef, InlineParserDef, ParserInput } from "./types";
+export { runParser } from "./runner";
+export { detectMimeType } from "./mime";
+export { collectStream } from "./collect";
+export { parsePdf } from "./pdf";
+export type { ParsePdfOptions } from "./pdf";

package/src/parsers/mime.test.ts

@@ -0,0 +1,91 @@
+import { test, expect } from "bun:test";
+import { detectMimeType } from "./mime";
+
+test("detectMimeType returns mimeOverride when provided", async () => {
+  const result = await detectMimeType({ mimeOverride: "application/pdf" });
+  expect(result).toBe("application/pdf");
+});
+
+test("detectMimeType detects PDF from magic bytes", async () => {
+  const pdfHeader = Buffer.from([0x25, 0x50, 0x44, 0x46, 0x2d, 0x31, 0x2e, 0x34]);
+  const result = await detectMimeType({ buffer: pdfHeader });
+  expect(result).toBe("application/pdf");
+});
+
+test("detectMimeType detects PNG from magic bytes", async () => {
+  const pngHeader = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+  const result = await detectMimeType({ buffer: pngHeader });
+  expect(result).toBe("image/png");
+});
+
+test("detectMimeType detects JPEG from magic bytes", async () => {
+  const jpegHeader = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]);
+  const result = await detectMimeType({ buffer: jpegHeader });
+  expect(result).toBe("image/jpeg");
+});
+
+test("detectMimeType detects GIF from magic bytes", async () => {
+  const gifHeader = Buffer.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61]);
+  const result = await detectMimeType({ buffer: gifHeader });
+  expect(result).toBe("image/gif");
+});
+
+test("detectMimeType detects WebP from magic bytes", async () => {
+  const webpHeader = Buffer.alloc(12);
+  // RIFF at offset 0
+  webpHeader[0] = 0x52; webpHeader[1] = 0x49; webpHeader[2] = 0x46; webpHeader[3] = 0x46;
+  // WEBP at offset 8
+  webpHeader[8] = 0x57; webpHeader[9] = 0x45; webpHeader[10] = 0x42; webpHeader[11] = 0x50;
+  const result = await detectMimeType({ buffer: webpHeader });
+  expect(result).toBe("image/webp");
+});
+
+test("detectMimeType falls back to extension lookup for .txt", async () => {
+  const result = await detectMimeType({ filePath: "/some/file.txt" });
+  expect(result).toBe("text/plain");
+});
+
+test("detectMimeType falls back to extension lookup for .md", async () => {
+  const result = await detectMimeType({ filePath: "README.md" });
+  expect(result).toBe("text/markdown");
+});
+
+test("detectMimeType falls back to extension lookup for .json", async () => {
+  const result = await detectMimeType({ filePath: "data.json" });
+  expect(result).toBe("application/json");
+});
+
+test("detectMimeType falls back to extension lookup for .docx", async () => {
+  const result = await detectMimeType({ filePath: "doc.docx" });
+  expect(result).toBe("application/vnd.openxmlformats-officedocument.wordprocessingml.document");
+});
+
+test("detectMimeType returns null for unknown extension with no magic bytes", async () => {
+  const result = await detectMimeType({ filePath: "file.xyz" });
+  expect(result).toBeNull();
+});
+
+test("detectMimeType returns null with no inputs", async () => {
+  const result = await detectMimeType({});
+  expect(result).toBeNull();
+});
+
+test("detectMimeType mimeOverride takes precedence over magic bytes", async () => {
+  const pdfHeader = Buffer.from([0x25, 0x50, 0x44, 0x46]);
+  const result = await detectMimeType({
+    buffer: pdfHeader,
+    mimeOverride: "text/plain",
+  });
+  expect(result).toBe("text/plain");
+});
+
+test("detectMimeType extension takes precedence over null buffer detection", async () => {
+  // Buffer with no magic bytes, but file has .pdf extension
+  const randomBuffer = Buffer.from([0x00, 0x01, 0x02, 0x03]);
+  const result = await detectMimeType({
+    buffer: randomBuffer,
+    filePath: "document.pdf",
+  });
+  // magic bytes don't match, falls back to extension
+  expect(result).toBe("application/pdf");
+});

package/src/parsers/mime.ts

@@ -0,0 +1,137 @@
+import path from "node:path";
+import type { NpmParserDef } from "./types";
+
+// Magic byte signatures for common file types
+const MAGIC_BYTES: Array<{ mimeType: string; bytes: number[]; offset?: number }> = [
+  // PDF: %PDF
+  { mimeType: "application/pdf", bytes: [0x25, 0x50, 0x44, 0x46] },
+  // PNG: 89 50 4E 47
+  { mimeType: "image/png", bytes: [0x89, 0x50, 0x4e, 0x47] },
+  // JPEG: FF D8 FF
+  { mimeType: "image/jpeg", bytes: [0xff, 0xd8, 0xff] },
+  // GIF: GIF8
+  { mimeType: "image/gif", bytes: [0x47, 0x49, 0x46, 0x38] },
+  // ZIP / Office Open XML (DOCX/XLSX/PPTX all start with PK\x03\x04)
+  {
+    mimeType: "application/zip",
+    bytes: [0x50, 0x4b, 0x03, 0x04],
+  },
+];
+
+// WebP has RIFF at offset 0 and WEBP at offset 8
+const isWebP = (header: Uint8Array): boolean => {
+  if (header.length < 12) return false;
+  const riff =
+    header[0] === 0x52 && header[1] === 0x49 && header[2] === 0x46 && header[3] === 0x46;
+  const webp =
+    header[8] === 0x57 && header[9] === 0x45 && header[10] === 0x42 && header[11] === 0x50;
+  return riff && webp;
+};
+
+const matchesMagicBytes = (header: Uint8Array, bytes: number[], offset = 0): boolean => {
+  if (header.length < offset + bytes.length) return false;
+  return bytes.every((b, i) => header[offset + i] === b);
+};
+
+const detectFromMagicBytes = (header: Uint8Array): string | null => {
+  if (isWebP(header)) return "image/webp";
+
+  for (const { mimeType, bytes, offset } of MAGIC_BYTES) {
+    if (matchesMagicBytes(header, bytes, offset ?? 0)) {
+      return mimeType;
+    }
+  }
+
+  return null;
+};
+
+// Extension → MIME type lookup
+const EXTENSION_MIME_MAP: Record<string, string> = {
+  ".txt": "text/plain",
+  ".md": "text/markdown",
+  ".markdown": "text/markdown",
+  ".html": "text/html",
+  ".htm": "text/html",
+  ".json": "application/json",
+  ".pdf": "application/pdf",
+  ".png": "image/png",
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".gif": "image/gif",
+  ".webp": "image/webp",
+  ".csv": "text/csv",
+  ".xml": "application/xml",
+  ".yaml": "application/yaml",
+  ".yml": "application/yaml",
+  ".docx":
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  ".xlsx":
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+  ".pptx":
+    "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+  ".mp4": "video/mp4",
+  ".mp3": "audio/mpeg",
+  ".wav": "audio/wav",
+  ".ogg": "audio/ogg",
+  ".svg": "image/svg+xml",
+  ".ts": "text/plain",
+  ".tsx": "text/plain",
+  ".js": "text/javascript",
+  ".jsx": "text/javascript",
+  ".css": "text/css",
+  ".toml": "application/toml",
+};
+
+export type NpmParserEntry = {
+  mimeType: string;
+  def: NpmParserDef;
+};
+
+export async function detectMimeType(options: {
+  buffer?: Buffer;
+  filePath?: string;
+  mimeOverride?: string;
+  npmParsers?: NpmParserEntry[];
+}): Promise<string | null> {
+  const { buffer, filePath, mimeOverride, npmParsers } = options;
+
+  // --mime override takes precedence
+  if (mimeOverride) {
+    return mimeOverride;
+  }
+
+  // Layer 1: magic bytes (authoritative)
+  if (buffer && buffer.length > 0) {
+    const header = buffer.subarray(0, 512);
+    const magicMime = detectFromMagicBytes(header);
+    if (magicMime) {
+      return magicMime;
+    }
+
+    // Layer 3: npm parser detectFileType callbacks (after built-ins)
+    if (npmParsers && npmParsers.length > 0) {
+      for (const entry of npmParsers) {
+        try {
+          const mod = await import(entry.def.package) as {
+            detectFileType?: (header: Uint8Array) => boolean;
+          };
+          if (typeof mod.detectFileType === "function" && mod.detectFileType(header)) {
+            return entry.mimeType;
+          }
+        } catch {
+          // If the package fails to load, skip it
+        }
+      }
+    }
+  }
+
+  // Layer 2: extension database (for file inputs)
+  if (filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    if (ext && ext in EXTENSION_MIME_MAP) {
+      return EXTENSION_MIME_MAP[ext] ?? null;
+    }
+  }
+
+  return null;
+}

package/src/parsers/npm.ts

@@ -0,0 +1,26 @@
+import type { Artifact } from "../types";
+
+/**
+ * Contract for npm parser packages.
+ *
+ * A parser package must export at least one of `parseStream` or `parseFile`.
+ * `detectFileType` is optional.
+ */
+
+// Parser receives a ReadableStream — no disk I/O needed
+export type ParseStreamFn = (
+  stream: ReadableStream<Uint8Array>,
+  mimeType: string,
+) => Promise<Artifact[]>;
+
+// Parser receives a file path — useful for libraries that only work with files
+export type ParseFileFn = (filePath: string, mimeType: string) => Promise<Artifact[]>;
+
+// Magic byte detection — optional; return true if this parser handles the given bytes
+export type DetectFileTypeFn = (header: Uint8Array) => boolean;
+
+export type NpmParserModule = {
+  parseStream?: ParseStreamFn;
+  parseFile?: ParseFileFn;
+  detectFileType?: DetectFileTypeFn;
+};