@struktur/sdk 2.1.1 → 2.2.0

package/src/chunking/ArtifactBatcher.ts

@@ -1,110 +0,0 @@
-import type { Artifact } from "../types";
-import type { DebugLogger } from "../debug/logger";
-import {
-  countArtifactTokens,
-  countArtifactImages,
-  type TokenCountOptions,
-} from "../tokenization";
-import { splitArtifact } from "./ArtifactSplitter";
-
-export type BatchOptions = TokenCountOptions & {
-  maxTokens: number;
-  maxImages?: number;
-  modelMaxTokens?: number;
-  debug?: DebugLogger;
-};
-
-export const batchArtifacts = (
-  artifacts: Artifact[],
-  options: BatchOptions
-): Artifact[][] => {
-  const debug = options.debug;
-  const maxTokens = options.modelMaxTokens
-    ? Math.min(options.maxTokens, options.modelMaxTokens)
-    : options.maxTokens;
-
-  // Log batching start
-  debug?.batchingStart({
-    totalArtifacts: artifacts.length,
-    maxTokens: options.maxTokens,
-    maxImages: options.maxImages,
-    modelMaxTokens: options.modelMaxTokens,
-    effectiveMaxTokens: maxTokens,
-  });
-
-  const batches: Artifact[][] = [];
-  let currentBatch: Artifact[] = [];
-  let currentTokens = 0;
-  let currentImages = 0;
-
-  for (const artifact of artifacts) {
-    const splitOptions: any = { 
-      maxTokens,
-      debug,
-    };
-    if (options.maxImages !== undefined) splitOptions.maxImages = options.maxImages;
-    if (options.textTokenRatio !== undefined) splitOptions.textTokenRatio = options.textTokenRatio;
-    if (options.defaultImageTokens !== undefined) splitOptions.defaultImageTokens = options.defaultImageTokens;
-    
-    const splits = splitArtifact(artifact, splitOptions);
-
-    for (const split of splits) {
-      const splitTokens = countArtifactTokens(split, options);
-      const splitImages = countArtifactImages(split);
-
-      const exceedsTokens =
-        currentBatch.length > 0 && currentTokens + splitTokens > maxTokens;
-      const exceedsImages =
-        options.maxImages !== undefined &&
-        currentBatch.length > 0 &&
-        currentImages + splitImages > options.maxImages;
-
-      if (exceedsTokens || exceedsImages) {
-        // Log batch creation
-        debug?.batchCreated({
-          batchIndex: batches.length,
-          artifactCount: currentBatch.length,
-          totalTokens: currentTokens,
-          totalImages: currentImages,
-          artifactIds: currentBatch.map(a => a.id),
-        });
-        
-        batches.push(currentBatch);
-        currentBatch = [];
-        currentTokens = 0;
-        currentImages = 0;
-      }
-
-      currentBatch.push(split);
-      currentTokens += splitTokens;
-      currentImages += splitImages;
-    }
-  }
-
-  if (currentBatch.length > 0) {
-    // Log final batch
-    debug?.batchCreated({
-      batchIndex: batches.length,
-      artifactCount: currentBatch.length,
-      totalTokens: currentTokens,
-      totalImages: currentImages,
-      artifactIds: currentBatch.map(a => a.id),
-    });
-    batches.push(currentBatch);
-  }
-
-  // Log batching complete
-  debug?.batchingComplete({
-    totalBatches: batches.length,
-    batches: batches.map((batch, index) => ({
-      index,
-      artifactCount: batch.length,
-      tokens: batch.reduce((sum, a) => sum + (a.tokens ?? 0), 0),
-      images: batch.reduce((sum, a) => 
-        sum + a.contents.reduce((c, content) => c + (content.media?.length ?? 0), 0), 0
-      ),
-    })),
-  });
-
-  return batches;
-};

package/src/chunking/ArtifactSplitter.test.ts

@@ -1,38 +0,0 @@
-import { test, expect } from "bun:test";
-import type { Artifact } from "../types";
-import { splitArtifact } from "./ArtifactSplitter";
-
-const baseArtifact = (text: string): Artifact => ({
-  id: "artifact-1",
-  type: "text",
-  raw: async () => Buffer.from(text),
-  contents: [{ text }],
-});
-
-test("splitArtifact splits large text into chunks", () => {
-  const artifact = baseArtifact("abcdefghijklmnopqrst");
-  const chunks = splitArtifact(artifact, { maxTokens: 2 });
-
-  expect(chunks.length).toBe(3);
-  expect(chunks[0]?.contents[0]?.text).toBe("abcdefgh");
-  expect(chunks[1]?.contents[0]?.text).toBe("ijklmnop");
-  expect(chunks[2]?.contents[0]?.text).toBe("qrst");
-});
-
-test("splitArtifact keeps media on first text chunk", () => {
-  const artifact: Artifact = {
-    id: "artifact-2",
-    type: "pdf",
-    raw: async () => Buffer.from(""),
-    contents: [
-      {
-        text: "abcdefghijklmnopqrst",
-        media: [{ type: "image", url: "https://example.com/x.png" }],
-      },
-    ],
-  };
-
-  const chunks = splitArtifact(artifact, { maxTokens: 2 });
-  expect(chunks[0]?.contents[0]?.media?.length).toBe(1);
-  expect(chunks[1]?.contents[0]?.media).toBeUndefined();
-});

package/src/chunking/ArtifactSplitter.ts

@@ -1,151 +0,0 @@
-import type { Artifact, ArtifactContent } from "../types";
-import type { DebugLogger } from "../debug/logger";
-import {
-  countContentTokens,
-  countArtifactImages,
-  countArtifactTokens,
-  estimateTextTokens,
-  type TokenCountOptions,
-} from "../tokenization";
-
-export type SplitOptions = TokenCountOptions & {
-  maxTokens: number;
-  maxImages?: number;
-  debug?: DebugLogger;
-};
-
-const splitTextIntoChunks = (
-  content: ArtifactContent,
-  maxTokens: number,
-  options?: TokenCountOptions,
-  debug?: DebugLogger,
-  artifactId?: string
-): ArtifactContent[] => {
-  if (!content.text) {
-    return [content];
-  }
-
-  const totalTokens = estimateTextTokens(content.text, options);
-  if (totalTokens <= maxTokens) {
-    return [content];
-  }
-
-  const ratio = options?.textTokenRatio ?? 4;
-  const chunkSize = Math.max(1, maxTokens * ratio);
-  const chunks: ArtifactContent[] = [];
-
-  // Log text splitting
-  if (debug && artifactId) {
-    debug.chunkingSplit({
-      artifactId,
-      originalContentCount: 1,
-      splitContentCount: Math.ceil(content.text.length / chunkSize),
-      splitReason: "text_too_long",
-      originalTokens: totalTokens,
-      chunkSize,
-    });
-  }
-
-  for (let offset = 0; offset < content.text.length; offset += chunkSize) {
-    const text = content.text.slice(offset, offset + chunkSize);
-    chunks.push({
-      page: content.page,
-      text,
-      media: offset === 0 ? content.media : undefined,
-    });
-  }
-
-  return chunks;
-};
-
-export const splitArtifact = (
-  artifact: Artifact,
-  options: SplitOptions
-): Artifact[] => {
-  const { maxTokens, maxImages, debug } = options;
-  const splitContents: ArtifactContent[] = [];
-
-  // Log chunking start
-  const totalTokens = countArtifactTokens(artifact, options);
-  debug?.chunkingStart({
-    artifactId: artifact.id,
-    totalTokens,
-    maxTokens,
-    maxImages,
-  });
-
-  for (const content of artifact.contents) {
-    splitContents.push(...splitTextIntoChunks(content, maxTokens, options, debug, artifact.id));
-  }
-
-  const chunks: Artifact[] = [];
-  let currentContents: ArtifactContent[] = [];
-  let currentTokens = 0;
-  let currentImages = 0;
-
-  for (const content of splitContents) {
-    const contentTokens = countContentTokens(content, options);
-    const contentImages = content.media?.length ?? 0;
-
-    const exceedsTokens =
-      currentContents.length > 0 && currentTokens + contentTokens > maxTokens;
-    const exceedsImages =
-      maxImages !== undefined &&
-      currentContents.length > 0 &&
-      currentImages + contentImages > maxImages;
-
-    if (exceedsTokens || exceedsImages) {
-      // Log chunk creation
-      if (debug) {
-        debug.chunkingSplit({
-          artifactId: artifact.id,
-          originalContentCount: splitContents.length,
-          splitContentCount: chunks.length + 1,
-          splitReason: exceedsTokens ? "content_limit" : "content_limit",
-          originalTokens: totalTokens,
-          chunkSize: maxTokens,
-        });
-      }
-      
-      chunks.push({
-        ...artifact,
-        id: `${artifact.id}:part:${chunks.length + 1}`,
-        contents: currentContents,
-        tokens: currentTokens,
-      });
-      currentContents = [];
-      currentTokens = 0;
-      currentImages = 0;
-    }
-
-    currentContents.push(content);
-    currentTokens += contentTokens;
-    currentImages += contentImages;
-  }
-
-  if (currentContents.length > 0) {
-    chunks.push({
-      ...artifact,
-      id: `${artifact.id}:part:${chunks.length + 1}`,
-      contents: currentContents,
-      tokens: currentTokens,
-    });
-  }
-
-  if (chunks.length === 0) {
-    chunks.push({
-      ...artifact,
-      id: `${artifact.id}:part:1`,
-      tokens: countArtifactTokens(artifact, options),
-    });
-  }
-
-  // Log chunking result
-  debug?.chunkingResult({
-    artifactId: artifact.id,
-    chunksCreated: chunks.length,
-    chunkSizes: chunks.map(c => c.tokens ?? 0),
-  });
-
-  return chunks;
-};

package/src/debug/AGENTS.md

@@ -1,79 +0,0 @@
-# Debug Module
-
-## Overview
-
-The debug module provides comprehensive JSON logging for the Struktur extraction pipeline. When `--debug` flag is enabled via CLI, every operation is logged as single-line JSON to stderr.
-
-## Key Files
-
-- `logger.ts`: Core debug logger with structured logging functions for every pipeline stage.
-
-## Debug Log Types
-
-### CLI Initialization
-- `cli_init`: CLI arguments and configuration
-- `schema_loaded`: Schema source and size
-- `artifacts_loaded`: Artifact count, types, tokens, images
-- `model_resolved`: Model specification resolution
-- `strategy_created`: Strategy selection with config
-
-### Chunking
-- `chunking_start`: Per-artifact chunking begins
-- `chunking_split`: Text or content splits due to limits
-- `chunking_result`: Final chunks created with sizes
-
-### Batching
-- `batching_start`: Batch creation parameters
-- `batch_created`: Individual batch details
-- `batching_complete`: Summary of all batches
-
-### Strategy Execution
-- `strategy_run_start`: Strategy begins with estimated steps
-- `step`: Step progression through pipeline
-- `progress`: Progress updates within steps
-
-### LLM Calls
-- `llm_call_start`: API call initiation with prompt sizes
-- `prompt_system`: Full system prompt (verbose)
-- `prompt_user`: Full user content (verbose)
-- `llm_call_complete`: Call completion with tokens/timing
-- `raw_response`: Raw LLM response data (verbose)
-
-### Validation
-- `validation_start`: Validation attempt begins
-- `validation_success`: Validation passed
-- `validation_failed`: Validation errors
-- `retry`: Retry attempt triggered
-
-### Merging
-- `merge_start`: Merge operation begins
-- `smart_merge_field`: Per-field merge operations
-- `merge_complete`: Merge success/failure
-
-### Deduplication
-- `dedupe_start`: Deduplication begins
-- `dedupe_complete`: Duplicates found and removed
-
-### Results
-- `token_usage`: Token consumption tracking
-- `extraction_complete`: Final extraction status
-
-## Usage
-
-Enable via CLI:
-```bash
-struktur extract --debug -t "text to extract" -s schema.json
-```
-
-Debug logs are written to stderr as single-line JSON:
-```json
-{"timestamp":"2026-02-24T20:00:00.000Z","type":"cli_init","args":{"strategy":"simple"}}
-```
-
-## Design Notes
-
-- All logs include ISO8601 timestamps
-- Logs are single-line JSON for easy parsing
-- Output goes to stderr to not interfere with stdout results
-- The debug logger is passed through the entire pipeline via `ExtractionOptions.debug`
-- When debug is disabled (default), all logging calls are no-ops

package/src/debug/logger.test.ts

@@ -1,244 +0,0 @@
-import { test, expect, beforeEach, afterEach } from "bun:test";
-import { createDebugLogger } from "./logger";
-
-let stderrOutput: string[];
-const originalStderrWrite = process.stderr.write;
-
-beforeEach(() => {
-  stderrOutput = [];
-  process.stderr.write = (chunk: unknown) => {
-    if (typeof chunk === "string") {
-      stderrOutput.push(chunk);
-    }
-    return true;
-  };
-});
-
-afterEach(() => {
-  process.stderr.write = originalStderrWrite;
-});
-
-test("createDebugLogger with enabled=false is a no-op", () => {
-  const logger = createDebugLogger(false);
-  logger.cliInit({ args: { test: true } });
-  expect(stderrOutput.length).toBe(0);
-});
-
-test("createDebugLogger with enabled=true logs to stderr", () => {
-  const logger = createDebugLogger(true);
-  logger.cliInit({ args: { test: true } });
-  expect(stderrOutput.length).toBe(1);
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.type).toBe("cli_init");
-  expect(parsed.args).toEqual({ test: true });
-  expect(parsed.timestamp).toBeDefined();
-});
-
-test("cliInit logs correct type", () => {
-  const logger = createDebugLogger(true);
-  logger.cliInit({ args: { strategy: "simple" } });
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.type).toBe("cli_init");
-});
-
-test("schemaLoaded logs source and size", () => {
-  const logger = createDebugLogger(true);
-  logger.schemaLoaded({ source: "file.json", schemaSize: 100 });
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.type).toBe("schema_loaded");
-  expect(parsed.source).toBe("file.json");
-  expect(parsed.schemaSize).toBe(100);
-});
-
-test("artifactsLoaded logs artifact details", () => {
-  const logger = createDebugLogger(true);
-  logger.artifactsLoaded({
-    count: 2,
-    artifacts: [
-      { id: "a1", type: "text", contentCount: 1, tokens: 10 },
-      { id: "a2", type: "pdf", contentCount: 3 },
-    ],
-    totalTokens: 1010,
-    totalImages: 2,
-  });
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.type).toBe("artifacts_loaded");
-  expect(parsed.count).toBe(2);
-  expect(parsed.totalTokens).toBe(1010);
-  expect(parsed.totalImages).toBe(2);
-});
-
-test("chunkingStart logs chunking parameters", () => {
-  const logger = createDebugLogger(true);
-  logger.chunkingStart({
-    artifactId: "a1",
-    totalTokens: 100,
-    maxTokens: 50,
-    maxImages: 5,
-  });
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.type).toBe("chunking_start");
-  expect(parsed.artifactId).toBe("a1");
-  expect(parsed.maxTokens).toBe(50);
-});
-
-test("llmCallStart logs call details", () => {
-  const logger = createDebugLogger(true);
-  logger.llmCallStart({
-    callId: "call-1",
-    model: "gpt-4",
-    schemaName: "extract",
-    systemLength: 100,
-    userLength: 200,
-    artifactCount: 3,
-  });
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.type).toBe("llm_call_start");
-  expect(parsed.callId).toBe("call-1");
-  expect(parsed.artifactCount).toBe(3);
-});
-
-test("llmCallComplete logs success with duration", () => {
-  const logger = createDebugLogger(true);
-  logger.llmCallComplete({
-    callId: "call-1",
-    success: true,
-    inputTokens: 100,
-    outputTokens: 50,
-    totalTokens: 150,
-    durationMs: 1234,
-  });
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.type).toBe("llm_call_complete");
-  expect(parsed.success).toBe(true);
-  expect(parsed.durationMs).toBe(1234);
-});
-
-test("llmCallComplete logs failure with error", () => {
-  const logger = createDebugLogger(true);
-  logger.llmCallComplete({
-    callId: "call-1",
-    success: false,
-    inputTokens: 100,
-    outputTokens: 0,
-    totalTokens: 100,
-    error: "API error",
-  });
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.success).toBe(false);
-  expect(parsed.error).toBe("API error");
-});
-
-test("retry logs retry attempt", () => {
-  const logger = createDebugLogger(true);
-  logger.retry({
-    callId: "call-1",
-    attempt: 2,
-    maxAttempts: 3,
-    reason: "schema_validation_failed",
-  });
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.type).toBe("retry");
-  expect(parsed.attempt).toBe(2);
-  expect(parsed.reason).toBe("schema_validation_failed");
-});
-
-test("validationStart logs validation attempt", () => {
-  const logger = createDebugLogger(true);
-  logger.validationStart({
-    callId: "call-1",
-    attempt: 1,
-    maxAttempts: 3,
-    strict: false,
-  });
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.type).toBe("validation_start");
-  expect(parsed.strict).toBe(false);
-});
-
-test("validationSuccess logs successful validation", () => {
-  const logger = createDebugLogger(true);
-  logger.validationSuccess({ callId: "call-1", attempt: 1 });
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.type).toBe("validation_success");
-});
-
-test("validationFailed logs validation errors", () => {
-  const logger = createDebugLogger(true);
-  logger.validationFailed({
-    callId: "call-1",
-    attempt: 1,
-    errors: [{ keyword: "required", message: "missing field" }],
-  });
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.type).toBe("validation_failed");
-  expect(parsed.errors).toBeDefined();
-});
-
-test("mergeStart logs merge operation", () => {
-  const logger = createDebugLogger(true);
-  logger.mergeStart({
-    mergeId: "merge-1",
-    inputCount: 3,
-    strategy: "parallel",
-  });
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.type).toBe("merge_start");
-  expect(parsed.inputCount).toBe(3);
-});
-
-test("mergeComplete logs merge result", () => {
-  const logger = createDebugLogger(true);
-  logger.mergeComplete({ mergeId: "merge-1", success: true });
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.type).toBe("merge_complete");
-  expect(parsed.success).toBe(true);
-});
-
-test("dedupeStart logs deduplication start", () => {
-  const logger = createDebugLogger(true);
-  logger.dedupeStart({ dedupeId: "dedupe-1", itemCount: 10 });
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.type).toBe("dedupe_start");
-  expect(parsed.itemCount).toBe(10);
-});
-
-test("dedupeComplete logs deduplication result", () => {
-  const logger = createDebugLogger(true);
-  logger.dedupeComplete({
-    dedupeId: "dedupe-1",
-    duplicatesFound: 3,
-    itemsRemoved: 3,
-  });
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.type).toBe("dedupe_complete");
-  expect(parsed.duplicatesFound).toBe(3);
-});
-
-test("extractionComplete logs final result", () => {
-  const logger = createDebugLogger(true);
-  logger.extractionComplete({
-    success: true,
-    totalInputTokens: 100,
-    totalOutputTokens: 50,
-    totalTokens: 150,
-  });
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.type).toBe("extraction_complete");
-  expect(parsed.success).toBe(true);
-});
-
-test("smartMergeField logs field merge operation", () => {
-  const logger = createDebugLogger(true);
-  logger.smartMergeField({
-    mergeId: "merge-1",
-    field: "items",
-    operation: "merge_arrays",
-    leftCount: 5,
-    rightCount: 3,
-    resultCount: 8,
-  });
-  const parsed = JSON.parse(stderrOutput[0]!);
-  expect(parsed.type).toBe("smart_merge_field");
-  expect(parsed.operation).toBe("merge_arrays");
-});