@struktur/sdk 0.1.0

package/src/llm/models.ts

@@ -0,0 +1,190 @@
+import type { ProviderModelsResult } from "../types";
+import { resolveProviderEnvVar, resolveProviderToken } from "../auth/tokens";
+
+const openAiModelsUrl = "https://api.openai.com/v1/models";
+const anthropicModelsUrl = "https://api.anthropic.com/v1/models";
+const googleModelsUrl = "https://generativelanguage.googleapis.com/v1beta/models";
+const openRouterModelsUrl = "https://openrouter.ai/api/v1/models";
+
+const getTokenForProvider = async (provider: string) => {
+  const envVar = resolveProviderEnvVar(provider);
+  if (envVar && process.env[envVar]) {
+    return process.env[envVar] as string;
+  }
+  return await resolveProviderToken(provider);
+};
+
+const parseOpenAiModels = (json: unknown) => {
+  const data = (json as { data?: Array<{ id?: string }> } | undefined)?.data ?? [];
+  return data.map((item) => item.id).filter((id): id is string => typeof id === "string");
+};
+
+const parseAnthropicModels = (json: unknown) => {
+  const data = (json as { data?: Array<{ id?: string }> } | undefined)?.data ?? [];
+  return data.map((item) => item.id).filter((id): id is string => typeof id === "string");
+};
+
+const parseGoogleModels = (json: unknown) => {
+  const data = (json as { models?: Array<{ name?: string }> } | undefined)?.models ?? [];
+  return data
+    .map((item) => item.name)
+    .filter((name): name is string => typeof name === "string")
+    .map((name) => name.replace(/^models\//, ""));
+};
+
+const parseOpenRouterModels = (json: unknown) => {
+  const data = (json as { data?: Array<{ id?: string }> } | undefined)?.data ?? [];
+  return data.map((item) => item.id).filter((id): id is string => typeof id === "string");
+};
+
+const requestModels = async (provider: string, token: string): Promise<string[]> => {
+  if (provider === "openai") {
+    const response = await fetch(openAiModelsUrl, {
+      headers: { Authorization: `Bearer ${token}` },
+    });
+    if (!response.ok) {
+      throw new Error(await response.text());
+    }
+    const json = (await response.json()) as unknown;
+    return parseOpenAiModels(json);
+  }
+
+  if (provider === "anthropic") {
+    const response = await fetch(anthropicModelsUrl, {
+      headers: {
+        "x-api-key": token,
+        "anthropic-version": "2023-06-01",
+      },
+    });
+    if (!response.ok) {
+      throw new Error(await response.text());
+    }
+    const json = (await response.json()) as unknown;
+    return parseAnthropicModels(json);
+  }
+
+  if (provider === "google") {
+    const response = await fetch(`${googleModelsUrl}?key=${encodeURIComponent(token)}`);
+    if (!response.ok) {
+      throw new Error(await response.text());
+    }
+    const json = (await response.json()) as unknown;
+    return parseGoogleModels(json);
+  }
+
+  if (provider === "openrouter") {
+    const response = await fetch(openRouterModelsUrl, {
+      headers: { Authorization: `Bearer ${token}` },
+    });
+    if (!response.ok) {
+      throw new Error(await response.text());
+    }
+    const json = (await response.json()) as unknown;
+    return parseOpenRouterModels(json);
+  }
+
+  if (provider === "opencode") {
+    // OpenCode doesn't have a public models endpoint, return known models
+    return [
+      "gpt-5.2",
+      "gpt-5.2-codex",
+      "gpt-5.1",
+      "gpt-5.1-codex",
+      "gpt-5.1-codex-max",
+      "gpt-5.1-codex-mini",
+      "gpt-5",
+      "gpt-5-codex",
+      "gpt-5-nano",
+      "claude-opus-4-6",
+      "claude-opus-4-5",
+      "claude-opus-4-1",
+      "claude-sonnet-4-6",
+      "claude-sonnet-4-5",
+      "claude-sonnet-4",
+      "claude-haiku-4-5",
+      "claude-haiku-3.5",
+      "gemini-3.1-pro",
+      "gemini-3-pro",
+      "gemini-3-flash",
+      "minimax-m2.5",
+      "minimax-m2.5-free",
+      "minimax-m2.1",
+      "glm-5",
+      "glm-5-free",
+      "glm-4.7",
+      "glm-4.6",
+      "kimi-k2.5",
+      "kimi-k2.5-free",
+      "kimi-k2-thinking",
+      "kimi-k2",
+      "qwen3-coder",
+      "big-pickle",
+    ];
+  }
+
+  throw new Error(`Unsupported provider: ${provider}`);
+};
+
+const cheapestModelPreferences: Record<string, string[]> = {
+  openai: ["gpt-4.1-nano", "gpt-4.1-mini", "gpt-4o-mini", "gpt-4o"],
+  anthropic: ["claude-3-5-haiku", "claude-3-haiku"],
+  google: ["gemini-1.5-flash-8b", "gemini-1.5-flash", "gemini-2.0-flash", "gemini-1.5-pro"],
+  opencode: ["gpt-5-nano", "claude-haiku-3.5", "gemini-3-flash", "kimi-k2-free", "glm-5-free", "minimax-m2.5-free"],
+  openrouter: ["openai/gpt-4o-mini", "anthropic/claude-3.5-haiku", "google/gemini-flash-1.5"],
+};
+
+const matchesPreference = (model: string, preference: string) => {
+  return model === preference || model.startsWith(`${preference}-`);
+};
+
+export const listProviderModels = async (provider: string): Promise<ProviderModelsResult> => {
+  const token = await getTokenForProvider(provider);
+  if (!token) {
+    return { provider, ok: false, error: "No token available" };
+  }
+
+  try {
+    const models = await requestModels(provider, token);
+    return { provider, ok: true, models };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { provider, ok: false, error: message };
+  }
+};
+
+export const listAllProviderModels = async (providers: string[]) => {
+  const results = await Promise.all(providers.map((provider) => listProviderModels(provider)));
+  return results;
+};
+
+export const pickCheapestModel = (provider: string, models: string[]) => {
+  const preferences = cheapestModelPreferences[provider] ?? [];
+  for (const preference of preferences) {
+    const match = models.find((model) => matchesPreference(model, preference));
+    if (match) {
+      return match;
+    }
+  }
+  return models[0];
+};
+
+export const resolveCheapestModel = async (provider: string) => {
+  const result = await listProviderModels(provider);
+  if (!result.ok) {
+    throw new Error(result.error ?? `Unable to list models for provider: ${provider}`);
+  }
+  const models = result.models ?? [];
+  const model = pickCheapestModel(provider, models);
+  if (!model) {
+    throw new Error(`No models available for provider: ${provider}`);
+  }
+  return model;
+};
+
+export const __testing__ = {
+  parseOpenAiModels,
+  parseAnthropicModels,
+  parseGoogleModels,
+  parseOpenRouterModels,
+  pickCheapestModel,
+};

package/src/merge/AGENTS.md

@@ -0,0 +1,6 @@
+Merge module
+
+- Purpose: schema-aware merging and deduplication of extracted data.
+- Key files: `SmartDataMerger.ts`, `Deduplicator.ts`.
+- Design: arrays concatenate, objects shallow-merge, scalars prefer new values; dedupe uses CRC32 hashing.
+- Tests: `SmartDataMerger.test.ts`, `Deduplicator.test.ts`.

package/src/merge/Deduplicator.test.ts

@@ -0,0 +1,108 @@
+import { test, expect } from "bun:test";
+import { findExactDuplicatesWithHashing, deduplicateByIndices, fnv1a32 } from "./Deduplicator";
+
+test("fnv1a32: official test vectors from lcn2/fnv", () => {
+  expect(fnv1a32("")).toBe(0x811c9dc5);
+  expect(fnv1a32("a")).toBe(0xe40c292c);
+  expect(fnv1a32("b")).toBe(0xe70c2de5);
+  expect(fnv1a32("c")).toBe(0xe60c2c52);
+  expect(fnv1a32("d")).toBe(0xe10c2473);
+  expect(fnv1a32("e")).toBe(0xe00c22e0);
+  expect(fnv1a32("f")).toBe(0xe30c2799);
+  expect(fnv1a32("fo")).toBe(0x6222e842);
+  expect(fnv1a32("foo")).toBe(0xa9f37ed7);
+  expect(fnv1a32("foob")).toBe(0x3f5076ef);
+  expect(fnv1a32("fooba")).toBe(0x39aaa18a);
+  expect(fnv1a32("foobar")).toBe(0xbf9cf968);
+  expect(fnv1a32("chongo was here!\n")).toBe(0xd49930d5);
+});
+
+test("fnv1a32: consistent results", () => {
+  const str = "test string for consistency";
+  const hash1 = fnv1a32(str);
+  const hash2 = fnv1a32(str);
+  const hash3 = fnv1a32(str);
+  expect(hash1).toBe(hash2);
+  expect(hash2).toBe(hash3);
+});
+
+test("fnv1a32: different strings produce different hashes", () => {
+  const strings = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"];
+  const hashes = strings.map(fnv1a32);
+  const uniqueHashes = new Set(hashes);
+  expect(uniqueHashes.size).toBe(strings.length);
+});
+
+test("fnv1a32: handles unicode", () => {
+  const hash1 = fnv1a32("hello");
+  const hash2 = fnv1a32("héllo");
+  const hash3 = fnv1a32("你好");
+  expect(hash1).not.toBe(hash2);
+  expect(typeof hash3).toBe("number");
+  expect(hash3).toBeGreaterThan(0);
+});
+
+test("fnv1a32: handles special characters", () => {
+  const hash1 = fnv1a32('{"key":"value"}');
+  const hash2 = fnv1a32('{"key":"value2"}');
+  expect(hash1).not.toBe(hash2);
+});
+
+test("fnv1a32: returns unsigned 32-bit integer", () => {
+  const hash = fnv1a32("some test string");
+  expect(hash).toBeGreaterThanOrEqual(0);
+  expect(hash).toBeLessThan(4294967296);
+  expect(Number.isInteger(hash)).toBe(true);
+});
+
+test("fnv1a32: collision resistance for similar strings", () => {
+  const strings = [
+    "item1", "item2", "item3", "item4", "item5",
+    "item6", "item7", "item8", "item9", "item10",
+    "Item1", "ITEM1", "itemA", "itemB", "itemC",
+  ];
+  const hashes = strings.map(fnv1a32);
+  const uniqueHashes = new Set(hashes);
+  expect(uniqueHashes.size).toBe(strings.length);
+});
+
+test("findExactDuplicatesWithHashing finds duplicates", () => {
+  const duplicates = findExactDuplicatesWithHashing([
+    { id: 1, name: "A" },
+    { id: 1, name: "A" },
+    { id: 2, name: "B" },
+  ]);
+
+  expect(duplicates).toEqual([1]);
+});
+
+test("deduplicateByIndices removes by index", () => {
+  const items = ["a", "b", "c"];
+  const result = deduplicateByIndices(items, [1]);
+  expect(result).toEqual(["a", "c"]);
+});
+
+test("findExactDuplicatesWithHashing handles empty array", () => {
+  expect(findExactDuplicatesWithHashing([])).toEqual([]);
+});
+
+test("findExactDuplicatesWithHashing handles no duplicates", () => {
+  expect(findExactDuplicatesWithHashing([1, 2, 3])).toEqual([]);
+});
+
+test("findExactDuplicatesWithHashing handles all duplicates", () => {
+  expect(findExactDuplicatesWithHashing([1, 1, 1])).toEqual([1, 2]);
+});
+
+test("findExactDuplicatesWithHashing handles complex objects", () => {
+  const obj = { a: 1, b: { c: 2 } };
+  const duplicates = findExactDuplicatesWithHashing([obj, obj, { a: 1, b: { c: 3 } }]);
+  expect(duplicates).toEqual([1]);
+});
+
+test("findExactDuplicatesWithHashing: key order doesn't matter", () => {
+  const obj1 = { a: 1, b: 2 };
+  const obj2 = { b: 2, a: 1 };
+  const duplicates = findExactDuplicatesWithHashing([obj1, obj2]);
+  expect(duplicates).toEqual([1]);
+});

package/src/merge/Deduplicator.ts

@@ -0,0 +1,45 @@
+export const fnv1a32 = (str: string): number => {
+  let hash = 2166136261;
+  for (let i = 0; i < str.length; i++) {
+    hash ^= str.charCodeAt(i);
+    hash = Math.imul(hash, 16777619);
+  }
+  return hash >>> 0;
+};
+
+const stableStringify = (value: unknown): string => {
+  if (value === null || typeof value !== "object") {
+    return JSON.stringify(value);
+  }
+
+  if (Array.isArray(value)) {
+    return `[${value.map((item) => stableStringify(item)).join(",")}]`;
+  }
+
+  const entries = Object.entries(value as Record<string, unknown>)
+    .sort(([a], [b]) => a.localeCompare(b))
+    .map(([key, val]) => `"${key}":${stableStringify(val)}`);
+
+  return `{${entries.join(",")}}`;
+};
+
+export const findExactDuplicatesWithHashing = (items: unknown[]) => {
+  const seen = new Map<number, number>();
+  const duplicates: number[] = [];
+
+  items.forEach((item, index) => {
+    const hash = fnv1a32(stableStringify(item));
+    if (seen.has(hash)) {
+      duplicates.push(index);
+      return;
+    }
+    seen.set(hash, index);
+  });
+
+  return duplicates;
+};
+
+export const deduplicateByIndices = <T>(items: T[], indices: number[]) => {
+  const remove = new Set(indices);
+  return items.filter((_, index) => !remove.has(index));
+};

package/src/merge/SmartDataMerger.test.ts

@@ -0,0 +1,177 @@
+import { test, expect } from "bun:test";
+import { SmartDataMerger } from "./SmartDataMerger";
+
+test("SmartDataMerger concatenates arrays and preserves scalars", () => {
+  const schema = {
+    type: "object",
+    properties: {
+      items: { type: "array" },
+      title: { type: "string" },
+    },
+  };
+
+  const merger = new SmartDataMerger(schema);
+  const result = merger.merge(
+    { items: [1], title: "A" },
+    { items: [2], title: "" }
+  );
+
+  expect(result.items).toEqual([1, 2]);
+  expect(result.title).toBe("A");
+});
+
+test("SmartDataMerger merges nested objects", () => {
+  const schema = {
+    type: "object",
+    properties: {
+      user: {
+        type: "object",
+        properties: {
+          name: { type: "string" },
+          email: { type: "string" },
+        },
+      },
+    },
+  };
+
+  const merger = new SmartDataMerger(schema);
+  const result = merger.merge(
+    { user: { name: "Alice" } },
+    { user: { email: "alice@example.com" } }
+  );
+
+  expect(result.user).toEqual({ name: "Alice", email: "alice@example.com" });
+});
+
+test("SmartDataMerger prefers new scalar values when not empty", () => {
+  const schema = {
+    type: "object",
+    properties: {
+      title: { type: "string" },
+      count: { type: "number" },
+    },
+  };
+
+  const merger = new SmartDataMerger(schema);
+  const result = merger.merge(
+    { title: "Old", count: 1 },
+    { title: "New", count: 2 }
+  );
+
+  expect(result.title).toBe("New");
+  expect(result.count).toBe(2);
+});
+
+test("SmartDataMerger preserves old value when new is null", () => {
+  const schema = {
+    type: "object",
+    properties: {
+      title: { type: "string" },
+    },
+  };
+
+  const merger = new SmartDataMerger(schema);
+  const result = merger.merge({ title: "Old" }, { title: null });
+
+  expect(result.title).toBe("Old");
+});
+
+test("SmartDataMerger preserves old value when new is undefined", () => {
+  const schema = {
+    type: "object",
+    properties: {
+      title: { type: "string" },
+    },
+  };
+
+  const merger = new SmartDataMerger(schema);
+  const result = merger.merge({ title: "Old" }, {});
+
+  expect(result.title).toBe("Old");
+});
+
+test("SmartDataMerger handles missing current value for arrays", () => {
+  const schema = {
+    type: "object",
+    properties: {
+      items: { type: "array" },
+    },
+  };
+
+  const merger = new SmartDataMerger(schema);
+  const result = merger.merge({}, { items: [1, 2] });
+
+  expect(result.items).toEqual([1, 2]);
+});
+
+test("SmartDataMerger handles missing new value for arrays", () => {
+  const schema = {
+    type: "object",
+    properties: {
+      items: { type: "array" },
+    },
+  };
+
+  const merger = new SmartDataMerger(schema);
+  const result = merger.merge({ items: [1, 2] }, {});
+
+  expect(result.items).toEqual([1, 2]);
+});
+
+test("SmartDataMerger handles non-array values for array schema", () => {
+  const schema = {
+    type: "object",
+    properties: {
+      items: { type: "array" },
+    },
+  };
+
+  const merger = new SmartDataMerger(schema);
+  const result = merger.merge({ items: "not-an-array" }, { items: [1] });
+
+  expect(result.items).toEqual([1]);
+});
+
+test("SmartDataMerger handles non-object values for object schema", () => {
+  const schema = {
+    type: "object",
+    properties: {
+      user: { type: "object", properties: {} },
+    },
+  };
+
+  const merger = new SmartDataMerger(schema);
+  const result = merger.merge({ user: "not-an-object" }, { user: { name: "Alice" } });
+
+  expect(result.user).toEqual({ name: "Alice" });
+});
+
+test("SmartDataMerger preserves properties not in schema", () => {
+  const schema = {
+    type: "object",
+    properties: {
+      title: { type: "string" },
+    },
+  };
+
+  const merger = new SmartDataMerger(schema);
+  const result = merger.merge(
+    { title: "A", extra: "preserved" },
+    { title: "B" }
+  );
+
+  expect(result.title).toBe("B");
+  expect(result.extra).toBe("preserved");
+});
+
+test("SmartDataMerger handles empty schema properties", () => {
+  const schema = {
+    type: "object",
+    properties: {},
+  };
+
+  const merger = new SmartDataMerger(schema);
+  const result = merger.merge({ title: "A" }, { title: "B" });
+
+  expect(result.title).toBe("A");
+});

package/src/merge/SmartDataMerger.ts

@@ -0,0 +1,56 @@
+import type { AnyJSONSchema } from "../types";
+
+const isArraySchema = (schema: Record<string, unknown>) => {
+  if (schema.type === "array") {
+    return true;
+  }
+  return false;
+};
+
+const isObjectSchema = (schema: Record<string, unknown>) => {
+  return schema.type === "object" && typeof schema.properties === "object";
+};
+
+export class SmartDataMerger {
+  private schema: AnyJSONSchema;
+
+  constructor(schema: AnyJSONSchema) {
+    this.schema = schema;
+  }
+
+  merge(currentData: Record<string, unknown>, newData: Record<string, unknown>) {
+    const merged: Record<string, unknown> = { ...currentData };
+    const properties =
+      (this.schema as { properties?: Record<string, Record<string, unknown>> })
+        .properties ?? {};
+
+    for (const [key, propSchema] of Object.entries(properties)) {
+      const currentValue = currentData[key];
+      const newValue = newData[key];
+
+      if (isArraySchema(propSchema)) {
+        merged[key] = [
+          ...(Array.isArray(currentValue) ? currentValue : []),
+          ...(Array.isArray(newValue) ? newValue : []),
+        ];
+        continue;
+      }
+
+      if (isObjectSchema(propSchema)) {
+        merged[key] = {
+          ...(typeof currentValue === "object" && currentValue ? currentValue : {}),
+          ...(typeof newValue === "object" && newValue ? newValue : {}),
+        };
+        continue;
+      }
+
+      if (newValue !== undefined && newValue !== null && newValue !== "") {
+        merged[key] = newValue;
+      } else if (currentValue !== undefined) {
+        merged[key] = currentValue;
+      }
+    }
+
+    return merged;
+  }
+}

package/src/parsers/AGENTS.md

@@ -0,0 +1,58 @@
+# Parsers module
+
+- Purpose: detect MIME types, run external/npm/command parsers, and provide built-in PDF support.
+- Key files: `types.ts`, `collect.ts`, `mime.ts`, `npm.ts`, `runner.ts`, `pdf.ts`, `index.ts`.
+
+## Types (`types.ts`)
+
+- `NpmParserDef` — npm package parser definition (`type: "npm"`, `package: string`)
+- `CommandFileDef` — command with `FILE_PATH` placeholder (`type: "command-file"`, `command: string`)
+- `CommandStdinDef` — command that reads from stdin (`type: "command-stdin"`, `command: string`)
+- `InlineParserDef` — inline handler function (`type: "inline"`, `handler: (buffer: Buffer) => Promise<Artifact>`)
+- `ParserDef` — union of the four variants
+- `ParsersConfig` — `Record<string, ParserDef>` keyed by MIME type
+- `ParserInput` — `{ kind: "file"; path: string } | { kind: "buffer"; buffer: Buffer }`
+
+## npm Contract (`npm.ts`)
+
+- `ParseStreamFn`, `ParseFileFn`, `DetectFileTypeFn`, `NpmParserModule` — interfaces that npm parser packages must implement.
+- At least one of `parseStream` or `parseFile` must be exported.
+
+## collectStream (`collect.ts`)
+
+- `collectStream(stream: ReadableStream<Uint8Array>): Promise<Buffer>` — public utility for npm parser authors to collect a stream into a Buffer.
+
+## MIME Detection (`mime.ts`)
+
+Two-layer detection + npm detectFileType callbacks:
+
+1. **Magic bytes** (authoritative): PDF, PNG, JPEG, GIF, WebP, ZIP/Office
+2. **npm `detectFileType`**: called after magic bytes with first 512 bytes
+3. **Extension database**: fallback when no magic bytes match (file inputs only)
+
+`detectMimeType({ buffer?, filePath?, mimeOverride?, npmParsers? }): Promise<string | null>`
+
+## Runner (`runner.ts`)
+
+`runParser(def: ParserDef, input: ParserInput, mimeType: string): Promise<Artifact[]>`
+
+- **npm**: Dynamic import, prefer `parseFile` for file inputs (zero-copy), prefer `parseStream` for buffer inputs. Falls back via temp-file if needed.
+- **command-file**: Interpolates `FILE_PATH` in command, writes temp file for buffer inputs.
+- **command-stdin**: Pipes input buffer to subprocess stdin; captures stdout as `SerializedArtifact[]` JSON.
+- **inline**: Calls the handler function directly with the buffer (reads file into buffer if needed).
+
+## Built-in PDF Parser (`pdf.ts`)
+
+`parsePdf(input: Buffer | ReadableStream<Uint8Array>, options?: ParsePdfOptions): Promise<Artifact>`
+
+Uses `pdf-parse` (npm package). Extracts per-page text **and** embedded images into `ArtifactContent[]`
+with `page` numbers set. Returns an `Artifact` with `type: "pdf"`.
+
+- Text extraction: per-page via `parser.getText()`; falls back to full document text when no per-page info is available.
+- Image extraction: per-page via `parser.getImage({ imageBuffer: false, imageDataUrl: true })`. Each embedded image is mapped to an `ArtifactImage` with `base64` (raw base64 string, data-URL prefix stripped), `width`, `height`, and `imageType: "embedded"`. Images are merged into the `media` array of the matching `ArtifactContent` entry. Pages that contain images but no text produce their own content entry. Image extraction failure is non-fatal — the parser continues and returns text-only content.
+- Screenshot rendering: per-page via `parser.getScreenshot()`. Each page is rendered to a PNG image and added to the `media` array with `imageType: "screenshot"`. Screenshots are appended to any embedded images for the same page. Screenshot rendering failure is non-fatal — the parser continues without screenshots.
+- `imageThreshold` defaults to 80 px (from pdf-parse), filtering out tiny decorative images.
+- `ParsePdfOptions.includeImages` (default `true`): set to `false` to skip `getImage()` entirely and return text-only content. This is used by the `--no-images` CLI flag.
+- `ParsePdfOptions.screenshots` (default `false`): set to `true` to render page screenshots and include them as images. This is used by the `--screenshots` CLI flag.
+- `ParsePdfOptions.screenshotScale` (default `1.5`): scale factor for screenshot rendering. Higher values produce larger, higher-quality images.
+- `ParsePdfOptions.screenshotWidth`: target width in pixels for screenshots. If specified, takes precedence over `screenshotScale` and height is calculated to maintain aspect ratio.