@struktur/sdk 0.1.0

package/src/auth/tokens.test.ts

@@ -0,0 +1,58 @@
+import { test, expect } from "bun:test";
+import path from "node:path";
+import os from "node:os";
+import { rm } from "node:fs/promises";
+import {
+  deleteProviderToken,
+  getProviderTokenOrThrow,
+  listStoredProviders,
+  resolveProviderToken,
+  setProviderToken,
+} from "./tokens";
+
+const makeTempDir = () => {
+  const suffix = Math.random().toString(16).slice(2);
+  return path.join(os.tmpdir(), `struktur-test-${suffix}`);
+};
+
+test("setProviderToken stores token in file when keychain disabled", async () => {
+  const tempDir = makeTempDir();
+  process.env.STRUKTUR_CONFIG_DIR = tempDir;
+  process.env.STRUKTUR_DISABLE_KEYCHAIN = "1";
+
+  try {
+    const storage = await setProviderToken("openai", "sk-test", "auto");
+    expect(storage).toBe("file");
+
+    const resolved = await resolveProviderToken("openai");
+    expect(resolved).toBe("sk-test");
+
+    const listed = await listStoredProviders();
+    expect(listed).toEqual([{ provider: "openai", storage: "file" }]);
+
+    const token = await getProviderTokenOrThrow("openai");
+    expect(token).toBe("sk-test");
+  } finally {
+    delete process.env.STRUKTUR_CONFIG_DIR;
+    delete process.env.STRUKTUR_DISABLE_KEYCHAIN;
+    await rm(tempDir, { recursive: true, force: true });
+  }
+});
+
+test("deleteProviderToken removes stored token", async () => {
+  const tempDir = makeTempDir();
+  process.env.STRUKTUR_CONFIG_DIR = tempDir;
+  process.env.STRUKTUR_DISABLE_KEYCHAIN = "1";
+
+  try {
+    await setProviderToken("anthropic", "sk-test", "auto");
+    const deleted = await deleteProviderToken("anthropic");
+    expect(deleted).toBe(true);
+    const resolved = await resolveProviderToken("anthropic");
+    expect(resolved).toBeUndefined();
+  } finally {
+    delete process.env.STRUKTUR_CONFIG_DIR;
+    delete process.env.STRUKTUR_DISABLE_KEYCHAIN;
+    await rm(tempDir, { recursive: true, force: true });
+  }
+});

package/src/auth/tokens.ts

@@ -0,0 +1,229 @@
+import path from "node:path";
+import os from "node:os";
+import { chmod, mkdir } from "node:fs/promises";
+
+export type TokenStorageType = "auto" | "keychain" | "file";
+
+export type TokenEntry = {
+  storage: "keychain" | "file";
+  token?: string;
+  account?: string;
+  service?: string;
+};
+
+type TokenStore = {
+  version: 1;
+  providers: Record<string, TokenEntry>;
+};
+
+const CONFIG_DIR_ENV = "STRUKTUR_CONFIG_DIR";
+const DISABLE_KEYCHAIN_ENV = "STRUKTUR_DISABLE_KEYCHAIN";
+const SERVICE_ENV = "STRUKTUR_KEYCHAIN_SERVICE";
+const DEFAULT_SERVICE = "struktur";
+
+const resolveConfigDir = () => {
+  return process.env[CONFIG_DIR_ENV] ?? path.join(os.homedir(), ".config", "struktur");
+};
+
+const resolveTokensPath = () => path.join(resolveConfigDir(), "tokens.json");
+
+const emptyStore = (): TokenStore => ({ version: 1, providers: {} });
+
+const readTokenStore = async (): Promise<TokenStore> => {
+  const tokensPath = resolveTokensPath();
+  const exists = await Bun.file(tokensPath).exists();
+  if (!exists) {
+    return emptyStore();
+  }
+  const raw = await Bun.file(tokensPath).text();
+  const parsed = JSON.parse(raw) as TokenStore;
+  if (!parsed || parsed.version !== 1 || typeof parsed.providers !== "object") {
+    return emptyStore();
+  }
+  return parsed;
+};
+
+const writeTokenStore = async (store: TokenStore) => {
+  const configDir = resolveConfigDir();
+  const tokensPath = resolveTokensPath();
+  await mkdir(configDir, { recursive: true, mode: 0o700 });
+  await Bun.write(tokensPath, JSON.stringify(store, null, 2));
+  await chmod(configDir, 0o700);
+  await chmod(tokensPath, 0o600);
+};
+
+const isKeychainAvailable = async () => {
+  if (process.env[DISABLE_KEYCHAIN_ENV]) {
+    return false;
+  }
+  if (process.platform !== "darwin") {
+    return false;
+  }
+  return await Bun.file("/usr/bin/security").exists();
+};
+
+const keychainService = () => process.env[SERVICE_ENV] ?? DEFAULT_SERVICE;
+
+const runSecurity = async (args: string[]) => {
+  const proc = Bun.spawn({
+    cmd: ["/usr/bin/security", ...args],
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  const stdout = await new Response(proc.stdout).text();
+  const stderr = await new Response(proc.stderr).text();
+  const exitCode = await proc.exited;
+  if (exitCode !== 0) {
+    const message = stderr.trim() || `security exited with ${exitCode}`;
+    throw new Error(message);
+  }
+  return stdout;
+};
+
+const writeKeychainToken = async (provider: string, token: string) => {
+  await runSecurity([
+    "add-generic-password",
+    "-a",
+    provider,
+    "-s",
+    keychainService(),
+    "-w",
+    token,
+    "-U",
+  ]);
+};
+
+const readKeychainToken = async (provider: string) => {
+  const output = await runSecurity([
+    "find-generic-password",
+    "-a",
+    provider,
+    "-s",
+    keychainService(),
+    "-w",
+  ]);
+  return output.trim();
+};
+
+const deleteKeychainToken = async (provider: string) => {
+  await runSecurity([
+    "delete-generic-password",
+    "-a",
+    provider,
+    "-s",
+    keychainService(),
+  ]);
+};
+
+export const listStoredProviders = async () => {
+  const store = await readTokenStore();
+  return Object.entries(store.providers).map(([provider, entry]) => ({
+    provider,
+    storage: entry.storage,
+  }));
+};
+
+export const setProviderToken = async (
+  provider: string,
+  token: string,
+  storage: TokenStorageType = "auto"
+) => {
+  const store = await readTokenStore();
+  let resolvedStorage: TokenEntry["storage"] = "file";
+
+  if (storage === "keychain") {
+    if (!(await isKeychainAvailable())) {
+      throw new Error("Keychain is not available on this platform.");
+    }
+    resolvedStorage = "keychain";
+  } else if (storage === "auto") {
+    resolvedStorage = (await isKeychainAvailable()) ? "keychain" : "file";
+  }
+
+  if (resolvedStorage === "keychain") {
+    await writeKeychainToken(provider, token);
+    store.providers[provider] = {
+      storage: "keychain",
+      account: provider,
+      service: keychainService(),
+    };
+  } else {
+    store.providers[provider] = {
+      storage: "file",
+      token,
+    };
+  }
+
+  await writeTokenStore(store);
+  return resolvedStorage;
+};
+
+export const deleteProviderToken = async (provider: string) => {
+  const store = await readTokenStore();
+  const entry = store.providers[provider];
+  if (!entry) {
+    return false;
+  }
+
+  if (entry.storage === "keychain") {
+    try {
+      await deleteKeychainToken(provider);
+    } catch {
+      // ignore errors for missing keychain items
+    }
+  }
+
+  delete store.providers[provider];
+  await writeTokenStore(store);
+  return true;
+};
+
+export const resolveProviderToken = async (provider: string) => {
+  const store = await readTokenStore();
+  const entry = store.providers[provider];
+  if (!entry) {
+    return undefined;
+  }
+
+  if (entry.storage === "file") {
+    return entry.token;
+  }
+
+  try {
+    return await readKeychainToken(provider);
+  } catch {
+    return undefined;
+  }
+};
+
+export const getProviderTokenOrThrow = async (provider: string) => {
+  const token = await resolveProviderToken(provider);
+  if (!token) {
+    throw new Error(`No token stored for provider: ${provider}`);
+  }
+  return token;
+};
+
+export const resolveProviderEnvVar = (provider: string) => {
+  switch (provider) {
+    case "openai":
+      return "OPENAI_API_KEY";
+    case "anthropic":
+      return "ANTHROPIC_API_KEY";
+    case "google":
+      return "GOOGLE_GENERATIVE_AI_API_KEY";
+    case "opencode":
+      return "OPENCODE_API_KEY";
+    case "openrouter":
+      return "OPENROUTER_API_KEY";
+    default:
+      return undefined;
+  }
+};
+
+export const maskToken = (token: string) => {
+  if (token.length <= 8) {
+    return "********";
+  }
+  return `${token.slice(0, 4)}...${token.slice(-4)}`;
+};

package/src/chunking/AGENTS.md

@@ -0,0 +1,11 @@
+Chunking module
+
+- Purpose: split and batch artifacts based on token and image limits.
+- Key files: `ArtifactSplitter.ts`, `ArtifactBatcher.ts`.
+- Design: split large artifact contents into parts, then batch parts to fit limits.
+- Tests: `ArtifactSplitter.test.ts`, `ArtifactBatcher.test.ts`.
+
+IMPORTANT: When modifying chunking/batching logic, you MUST also update the client-side
+JavaScript implementation in `src/cli.ts` (`generateArtifactViewerHtml`) to keep the
+artifact viewer's chunking visualization in sync. The viewer includes a version stamp
+to help users verify they're comparing the correct algorithm version.

package/src/chunking/ArtifactBatcher.test.ts

@@ -0,0 +1,22 @@
+import { test, expect } from "bun:test";
+import type { Artifact } from "../types";
+import { batchArtifacts } from "./ArtifactBatcher";
+
+const makeArtifact = (id: string, text: string): Artifact => ({
+  id,
+  type: "text",
+  raw: async () => Buffer.from(text),
+  contents: [{ text }],
+});
+
+test("batchArtifacts respects maxTokens", () => {
+  const artifacts = [
+    makeArtifact("a1", "abcdefgh"),
+    makeArtifact("a2", "abcdefgh"),
+  ];
+
+  const batches = batchArtifacts(artifacts, { maxTokens: 2 });
+  expect(batches.length).toBe(2);
+  expect(batches[0]?.length).toBe(1);
+  expect(batches[1]?.length).toBe(1);
+});

package/src/chunking/ArtifactBatcher.ts

@@ -0,0 +1,110 @@
+import type { Artifact } from "../types";
+import type { DebugLogger } from "../debug/logger";
+import {
+  countArtifactTokens,
+  countArtifactImages,
+  type TokenCountOptions,
+} from "../tokenization";
+import { splitArtifact } from "./ArtifactSplitter";
+
+export type BatchOptions = TokenCountOptions & {
+  maxTokens: number;
+  maxImages?: number;
+  modelMaxTokens?: number;
+  debug?: DebugLogger;
+};
+
+export const batchArtifacts = (
+  artifacts: Artifact[],
+  options: BatchOptions
+): Artifact[][] => {
+  const debug = options.debug;
+  const maxTokens = options.modelMaxTokens
+    ? Math.min(options.maxTokens, options.modelMaxTokens)
+    : options.maxTokens;
+
+  // Log batching start
+  debug?.batchingStart({
+    totalArtifacts: artifacts.length,
+    maxTokens: options.maxTokens,
+    maxImages: options.maxImages,
+    modelMaxTokens: options.modelMaxTokens,
+    effectiveMaxTokens: maxTokens,
+  });
+
+  const batches: Artifact[][] = [];
+  let currentBatch: Artifact[] = [];
+  let currentTokens = 0;
+  let currentImages = 0;
+
+  for (const artifact of artifacts) {
+    const splitOptions: any = { 
+      maxTokens,
+      debug,
+    };
+    if (options.maxImages !== undefined) splitOptions.maxImages = options.maxImages;
+    if (options.textTokenRatio !== undefined) splitOptions.textTokenRatio = options.textTokenRatio;
+    if (options.defaultImageTokens !== undefined) splitOptions.defaultImageTokens = options.defaultImageTokens;
+    
+    const splits = splitArtifact(artifact, splitOptions);
+
+    for (const split of splits) {
+      const splitTokens = countArtifactTokens(split, options);
+      const splitImages = countArtifactImages(split);
+
+      const exceedsTokens =
+        currentBatch.length > 0 && currentTokens + splitTokens > maxTokens;
+      const exceedsImages =
+        options.maxImages !== undefined &&
+        currentBatch.length > 0 &&
+        currentImages + splitImages > options.maxImages;
+
+      if (exceedsTokens || exceedsImages) {
+        // Log batch creation
+        debug?.batchCreated({
+          batchIndex: batches.length,
+          artifactCount: currentBatch.length,
+          totalTokens: currentTokens,
+          totalImages: currentImages,
+          artifactIds: currentBatch.map(a => a.id),
+        });
+        
+        batches.push(currentBatch);
+        currentBatch = [];
+        currentTokens = 0;
+        currentImages = 0;
+      }
+
+      currentBatch.push(split);
+      currentTokens += splitTokens;
+      currentImages += splitImages;
+    }
+  }
+
+  if (currentBatch.length > 0) {
+    // Log final batch
+    debug?.batchCreated({
+      batchIndex: batches.length,
+      artifactCount: currentBatch.length,
+      totalTokens: currentTokens,
+      totalImages: currentImages,
+      artifactIds: currentBatch.map(a => a.id),
+    });
+    batches.push(currentBatch);
+  }
+
+  // Log batching complete
+  debug?.batchingComplete({
+    totalBatches: batches.length,
+    batches: batches.map((batch, index) => ({
+      index,
+      artifactCount: batch.length,
+      tokens: batch.reduce((sum, a) => sum + (a.tokens ?? 0), 0),
+      images: batch.reduce((sum, a) => 
+        sum + a.contents.reduce((c, content) => c + (content.media?.length ?? 0), 0), 0
+      ),
+    })),
+  });
+
+  return batches;
+};

package/src/chunking/ArtifactSplitter.test.ts

@@ -0,0 +1,38 @@
+import { test, expect } from "bun:test";
+import type { Artifact } from "../types";
+import { splitArtifact } from "./ArtifactSplitter";
+
+const baseArtifact = (text: string): Artifact => ({
+  id: "artifact-1",
+  type: "text",
+  raw: async () => Buffer.from(text),
+  contents: [{ text }],
+});
+
+test("splitArtifact splits large text into chunks", () => {
+  const artifact = baseArtifact("abcdefghijklmnopqrst");
+  const chunks = splitArtifact(artifact, { maxTokens: 2 });
+
+  expect(chunks.length).toBe(3);
+  expect(chunks[0]?.contents[0]?.text).toBe("abcdefgh");
+  expect(chunks[1]?.contents[0]?.text).toBe("ijklmnop");
+  expect(chunks[2]?.contents[0]?.text).toBe("qrst");
+});
+
+test("splitArtifact keeps media on first text chunk", () => {
+  const artifact: Artifact = {
+    id: "artifact-2",
+    type: "pdf",
+    raw: async () => Buffer.from(""),
+    contents: [
+      {
+        text: "abcdefghijklmnopqrst",
+        media: [{ type: "image", url: "https://example.com/x.png" }],
+      },
+    ],
+  };
+
+  const chunks = splitArtifact(artifact, { maxTokens: 2 });
+  expect(chunks[0]?.contents[0]?.media?.length).toBe(1);
+  expect(chunks[1]?.contents[0]?.media).toBeUndefined();
+});

package/src/chunking/ArtifactSplitter.ts

@@ -0,0 +1,151 @@
+import type { Artifact, ArtifactContent } from "../types";
+import type { DebugLogger } from "../debug/logger";
+import {
+  countContentTokens,
+  countArtifactImages,
+  countArtifactTokens,
+  estimateTextTokens,
+  type TokenCountOptions,
+} from "../tokenization";
+
+export type SplitOptions = TokenCountOptions & {
+  maxTokens: number;
+  maxImages?: number;
+  debug?: DebugLogger;
+};
+
+const splitTextIntoChunks = (
+  content: ArtifactContent,
+  maxTokens: number,
+  options?: TokenCountOptions,
+  debug?: DebugLogger,
+  artifactId?: string
+): ArtifactContent[] => {
+  if (!content.text) {
+    return [content];
+  }
+
+  const totalTokens = estimateTextTokens(content.text, options);
+  if (totalTokens <= maxTokens) {
+    return [content];
+  }
+
+  const ratio = options?.textTokenRatio ?? 4;
+  const chunkSize = Math.max(1, maxTokens * ratio);
+  const chunks: ArtifactContent[] = [];
+
+  // Log text splitting
+  if (debug && artifactId) {
+    debug.chunkingSplit({
+      artifactId,
+      originalContentCount: 1,
+      splitContentCount: Math.ceil(content.text.length / chunkSize),
+      splitReason: "text_too_long",
+      originalTokens: totalTokens,
+      chunkSize,
+    });
+  }
+
+  for (let offset = 0; offset < content.text.length; offset += chunkSize) {
+    const text = content.text.slice(offset, offset + chunkSize);
+    chunks.push({
+      page: content.page,
+      text,
+      media: offset === 0 ? content.media : undefined,
+    });
+  }
+
+  return chunks;
+};
+
+export const splitArtifact = (
+  artifact: Artifact,
+  options: SplitOptions
+): Artifact[] => {
+  const { maxTokens, maxImages, debug } = options;
+  const splitContents: ArtifactContent[] = [];
+
+  // Log chunking start
+  const totalTokens = countArtifactTokens(artifact, options);
+  debug?.chunkingStart({
+    artifactId: artifact.id,
+    totalTokens,
+    maxTokens,
+    maxImages,
+  });
+
+  for (const content of artifact.contents) {
+    splitContents.push(...splitTextIntoChunks(content, maxTokens, options, debug, artifact.id));
+  }
+
+  const chunks: Artifact[] = [];
+  let currentContents: ArtifactContent[] = [];
+  let currentTokens = 0;
+  let currentImages = 0;
+
+  for (const content of splitContents) {
+    const contentTokens = countContentTokens(content, options);
+    const contentImages = content.media?.length ?? 0;
+
+    const exceedsTokens =
+      currentContents.length > 0 && currentTokens + contentTokens > maxTokens;
+    const exceedsImages =
+      maxImages !== undefined &&
+      currentContents.length > 0 &&
+      currentImages + contentImages > maxImages;
+
+    if (exceedsTokens || exceedsImages) {
+      // Log chunk creation
+      if (debug) {
+        debug.chunkingSplit({
+          artifactId: artifact.id,
+          originalContentCount: splitContents.length,
+          splitContentCount: chunks.length + 1,
+          splitReason: exceedsTokens ? "content_limit" : "content_limit",
+          originalTokens: totalTokens,
+          chunkSize: maxTokens,
+        });
+      }
+      
+      chunks.push({
+        ...artifact,
+        id: `${artifact.id}:part:${chunks.length + 1}`,
+        contents: currentContents,
+        tokens: currentTokens,
+      });
+      currentContents = [];
+      currentTokens = 0;
+      currentImages = 0;
+    }
+
+    currentContents.push(content);
+    currentTokens += contentTokens;
+    currentImages += contentImages;
+  }
+
+  if (currentContents.length > 0) {
+    chunks.push({
+      ...artifact,
+      id: `${artifact.id}:part:${chunks.length + 1}`,
+      contents: currentContents,
+      tokens: currentTokens,
+    });
+  }
+
+  if (chunks.length === 0) {
+    chunks.push({
+      ...artifact,
+      id: `${artifact.id}:part:1`,
+      tokens: countArtifactTokens(artifact, options),
+    });
+  }
+
+  // Log chunking result
+  debug?.chunkingResult({
+    artifactId: artifact.id,
+    chunksCreated: chunks.length,
+    chunkSizes: chunks.map(c => c.tokens ?? 0),
+  });
+
+  return chunks;
+};

package/src/debug/AGENTS.md

@@ -0,0 +1,79 @@
+# Debug Module
+
+## Overview
+
+The debug module provides comprehensive JSON logging for the Struktur extraction pipeline. When `--debug` flag is enabled via CLI, every operation is logged as single-line JSON to stderr.
+
+## Key Files
+
+- `logger.ts`: Core debug logger with structured logging functions for every pipeline stage.
+
+## Debug Log Types
+
+### CLI Initialization
+- `cli_init`: CLI arguments and configuration
+- `schema_loaded`: Schema source and size
+- `artifacts_loaded`: Artifact count, types, tokens, images
+- `model_resolved`: Model specification resolution
+- `strategy_created`: Strategy selection with config
+
+### Chunking
+- `chunking_start`: Per-artifact chunking begins
+- `chunking_split`: Text or content splits due to limits
+- `chunking_result`: Final chunks created with sizes
+
+### Batching
+- `batching_start`: Batch creation parameters
+- `batch_created`: Individual batch details
+- `batching_complete`: Summary of all batches
+
+### Strategy Execution
+- `strategy_run_start`: Strategy begins with estimated steps
+- `step`: Step progression through pipeline
+- `progress`: Progress updates within steps
+
+### LLM Calls
+- `llm_call_start`: API call initiation with prompt sizes
+- `prompt_system`: Full system prompt (verbose)
+- `prompt_user`: Full user content (verbose)
+- `llm_call_complete`: Call completion with tokens/timing
+- `raw_response`: Raw LLM response data (verbose)
+
+### Validation
+- `validation_start`: Validation attempt begins
+- `validation_success`: Validation passed
+- `validation_failed`: Validation errors
+- `retry`: Retry attempt triggered
+
+### Merging
+- `merge_start`: Merge operation begins
+- `smart_merge_field`: Per-field merge operations
+- `merge_complete`: Merge success/failure
+
+### Deduplication
+- `dedupe_start`: Deduplication begins
+- `dedupe_complete`: Duplicates found and removed
+
+### Results
+- `token_usage`: Token consumption tracking
+- `extraction_complete`: Final extraction status
+
+## Usage
+
+Enable via CLI:
+```bash
+struktur extract --debug -t "text to extract" -s schema.json
+```
+
+Debug logs are written to stderr as single-line JSON:
+```json
+{"timestamp":"2026-02-24T20:00:00.000Z","type":"cli_init","args":{"strategy":"simple"}}
+```
+
+## Design Notes
+
+- All logs include ISO8601 timestamps
+- Logs are single-line JSON for easy parsing
+- Output goes to stderr to not interfere with stdout results
+- The debug logger is passed through the entire pipeline via `ExtractionOptions.debug`
+- When debug is disabled (default), all logging calls are no-ops