@struktur/sdk 2.1.2 → 2.3.0

package/src/strategies/index.test.ts

@@ -1,20 +0,0 @@
-import { test, expect } from "bun:test";
-import * as strategies from "./index";
-
-test("strategies index re-exports constructors and helpers", () => {
-  expect(typeof strategies.SimpleStrategy).toBe("function");
-  expect(typeof strategies.ParallelStrategy).toBe("function");
-  expect(typeof strategies.SequentialStrategy).toBe("function");
-  expect(typeof strategies.ParallelAutoMergeStrategy).toBe("function");
-  expect(typeof strategies.SequentialAutoMergeStrategy).toBe("function");
-  expect(typeof strategies.DoublePassStrategy).toBe("function");
-  expect(typeof strategies.DoublePassAutoMergeStrategy).toBe("function");
-
-  expect(typeof strategies.simple).toBe("function");
-  expect(typeof strategies.parallel).toBe("function");
-  expect(typeof strategies.sequential).toBe("function");
-  expect(typeof strategies.parallelAutoMerge).toBe("function");
-  expect(typeof strategies.sequentialAutoMerge).toBe("function");
-  expect(typeof strategies.doublePass).toBe("function");
-  expect(typeof strategies.doublePassAutoMerge).toBe("function");
-});

package/src/strategies/utils.test.ts

@@ -1,76 +0,0 @@
-import { test, expect } from "bun:test";
-import type { Artifact } from "../types";
-import type { JSONSchemaType } from "ajv";
-import { batchArtifacts } from "../chunking/ArtifactBatcher";
-import { serializeSchema, mergeUsage, getBatches, extractWithPrompt } from "./utils";
-
-type Output = { title: string };
-
-const schema: JSONSchemaType<Output> = {
-  type: "object",
-  properties: { title: { type: "string" } },
-  required: ["title"],
-  additionalProperties: false,
-};
-
-const makeArtifact = (id: string, text: string): Artifact => ({
-  id,
-  type: "text",
-  raw: async () => Buffer.from(text),
-  contents: [{ text }],
-});
-
-test("serializeSchema returns JSON", () => {
-  expect(serializeSchema({ ok: true })).toBe("{\"ok\":true}");
-});
-
-test("mergeUsage sums token usage", () => {
-  const usage = mergeUsage([
-    { inputTokens: 1, outputTokens: 2, totalTokens: 3 },
-    { inputTokens: 4, outputTokens: 5, totalTokens: 9 },
-  ]);
-
-  expect(usage).toEqual({ inputTokens: 5, outputTokens: 7, totalTokens: 12 });
-});
-
-test("getBatches delegates to batchArtifacts", () => {
-  const artifacts = [makeArtifact("a1", "hello"), makeArtifact("a2", "world")];
-  const options = { maxTokens: 1 };
-
-  expect(getBatches(artifacts, options)).toEqual(batchArtifacts(artifacts, options));
-});
-
-test("extractWithPrompt builds user content and returns result", async () => {
-  const artifacts: Artifact[] = [
-    {
-      id: "a1",
-      type: "image",
-      raw: async () => Buffer.from(""),
-      contents: [
-        {
-          text: "hello",
-          media: [{ type: "image", base64: "abc" }],
-        },
-      ],
-    },
-  ];
-
-  let receivedUser: unknown;
-  const result = await extractWithPrompt<Output>({
-    model: {},
-    schema,
-    system: "sys",
-    user: "prompt",
-    artifacts,
-    execute: async (request) => {
-      receivedUser = request.user;
-      return {
-        data: { title: "ok" },
-        usage: { inputTokens: 1, outputTokens: 2, totalTokens: 3 },
-      };
-    },
-  });
-
-  expect(Array.isArray(receivedUser)).toBe(true);
-  expect(result.data.title).toBe("ok");
-});

package/src/strategies/utils.ts

@@ -1,95 +0,0 @@
-import type { Artifact, ExtractionEvents, Usage, TelemetryAdapter } from "../types";
-import type { DebugLogger } from "../debug/logger";
-import { batchArtifacts, type BatchOptions } from "../chunking/ArtifactBatcher";
-import { buildUserContent } from "../llm/message";
-import { runWithRetries } from "../llm/RetryingRunner";
-
-export const serializeSchema = (schema: unknown) => {
-  return JSON.stringify(schema);
-};
-
-export const mergeUsage = (usages: Usage[]) => {
-  return usages.reduce(
-    (acc, usage) => ({
-      inputTokens: acc.inputTokens + usage.inputTokens,
-      outputTokens: acc.outputTokens + usage.outputTokens,
-      totalTokens: acc.totalTokens + usage.totalTokens,
-    }),
-    { inputTokens: 0, outputTokens: 0, totalTokens: 0 }
-  );
-};
-
-export const getBatches = (
-  artifacts: Artifact[],
-  options: BatchOptions,
-  debug?: DebugLogger,
-  telemetry?: TelemetryAdapter,
-  parentSpan?: { id: string; traceId: string; name: string; kind: string; startTime: number; parentId?: string }
-) => {
-  // Create chunking span if telemetry is enabled
-  const chunkingSpan = telemetry?.startSpan({
-    name: "struktur.chunking",
-    kind: "RETRIEVER",
-    parentSpan,
-    attributes: {
-      "chunking.artifact_count": artifacts.length,
-      "chunking.max_tokens": options.maxTokens,
-      "chunking.max_images": options.maxImages,
-    },
-  });
-  
-  const batches = batchArtifacts(artifacts, { ...options, debug });
-  
-  // Record chunking results
-  if (chunkingSpan && telemetry) {
-    batches.forEach((batch, index) => {
-      telemetry.recordEvent(chunkingSpan, {
-        type: "chunk",
-        chunkIndex: index,
-        totalChunks: batches.length,
-        tokens: batch.reduce((sum, a) => sum + (a.tokens || 0), 0),
-        images: batch.reduce((sum, a) => 
-          sum + (a.contents?.flatMap((c) => c.media || []).length || 0), 0),
-      });
-    });
-    
-    telemetry.endSpan(chunkingSpan, {
-      status: "ok",
-      output: { batchCount: batches.length },
-    });
-  }
-  
-  return batches;
-};
-
-export const extractWithPrompt = async <T>(options: {
-  model: unknown;
-  schema: unknown;
-  system: string;
-  user: string;
-  artifacts: Artifact[];
-  events?: ExtractionEvents;
-  execute?: typeof runWithRetries<T>;
-  strict?: boolean;
-  debug?: DebugLogger;
-  callId?: string;
-  telemetry?: TelemetryAdapter;
-  parentSpan?: { id: string; traceId: string; name: string; kind: string; startTime: number; parentId?: string };
-}) => {
-  const userContent = buildUserContent(options.user, options.artifacts);
-  const result = await runWithRetries<T>({
-    model: options.model,
-    schema: options.schema,
-    system: options.system,
-    user: userContent,
-    events: options.events,
-    execute: options.execute,
-    strict: options.strict,
-    debug: options.debug,
-    callId: options.callId,
-    telemetry: options.telemetry,
-    parentSpan: options.parentSpan,
-  });
-
-  return result;
-};

package/src/tokenization.test.ts

@@ -1,119 +0,0 @@
-import { test, expect } from "bun:test";
-import type { Artifact } from "./types";
-import {
-  countArtifactTokens,
-  estimateTextTokens,
-  estimateImageTokens,
-  countContentTokens,
-} from "./tokenization";
-
-test("estimateTextTokens uses default ratio", () => {
-  expect(estimateTextTokens("abcd")).toBe(1);
-  expect(estimateTextTokens("abcdefgh")).toBe(2);
-});
-
-test("estimateTextTokens uses custom ratio", () => {
-  expect(estimateTextTokens("abcd", { textTokenRatio: 2 })).toBe(2);
-  expect(estimateTextTokens("abcdefgh", { textTokenRatio: 2 })).toBe(4);
-});
-
-test("estimateImageTokens uses default image tokens", () => {
-  expect(estimateImageTokens({ type: "image" })).toBe(1000);
-});
-
-test("estimateImageTokens uses custom image tokens", () => {
-  expect(estimateImageTokens({ type: "image" }, { defaultImageTokens: 500 })).toBe(500);
-});
-
-test("countArtifactTokens honors artifact.tokens override", () => {
-  const artifact: Artifact = {
-    id: "a1",
-    type: "text",
-    raw: async () => Buffer.from(""),
-    contents: [{ text: "hello" }],
-    tokens: 42,
-  };
-
-  expect(countArtifactTokens(artifact)).toBe(42);
-});
-
-test("countArtifactTokens sums text and media tokens", () => {
-  const artifact: Artifact = {
-    id: "a2",
-    type: "text",
-    raw: async () => Buffer.from(""),
-    contents: [
-      {
-        text: "abcdefgh",
-        media: [{ type: "image" }],
-      },
-    ],
-  };
-
-  const tokens = countArtifactTokens(artifact);
-  expect(tokens).toBe(1002);
-});
-
-test("countContentTokens counts text tokens", () => {
-  const tokens = countContentTokens({ text: "abcdefgh" });
-  expect(tokens).toBe(2);
-});
-
-test("countContentTokens counts image tokens", () => {
-  const tokens = countContentTokens({ media: [{ type: "image" }] });
-  expect(tokens).toBe(1000);
-});
-
-test("countContentTokens counts image text tokens", () => {
-  const tokens = countContentTokens({
-    media: [{ type: "image", text: "abcd" }],
-  });
-  expect(tokens).toBe(1001);
-});
-
-test("countContentTokens sums multiple images", () => {
-  const tokens = countContentTokens({
-    media: [{ type: "image" }, { type: "image" }],
-  });
-  expect(tokens).toBe(2000);
-});
-
-test("countContentTokens handles empty content", () => {
-  const tokens = countContentTokens({});
-  expect(tokens).toBe(0);
-});
-
-test("countArtifactTokens sums multiple contents", () => {
-  const artifact: Artifact = {
-    id: "a1",
-    type: "text",
-    raw: async () => Buffer.from(""),
-    contents: [
-      { text: "abcd" },
-      { text: "efgh" },
-      { media: [{ type: "image" }] },
-    ],
-  };
-
-  const tokens = countArtifactTokens(artifact);
-  expect(tokens).toBe(1002);
-});
-
-test("countArtifactTokens with custom options", () => {
-  const artifact: Artifact = {
-    id: "a1",
-    type: "text",
-    raw: async () => Buffer.from(""),
-    contents: [
-      { text: "abcdefgh" },
-      { media: [{ type: "image" }] },
-    ],
-  };
-
-  const tokens = countArtifactTokens(artifact, {
-    textTokenRatio: 2,
-    defaultImageTokens: 500,
-  });
-
-  expect(tokens).toBe(504);
-});

package/src/tokenization.ts

@@ -1,71 +0,0 @@
-import type { Artifact, ArtifactContent, ArtifactImage } from "./types";
-
-export type TokenCountOptions = {
-  textTokenRatio?: number;
-  defaultImageTokens?: number;
-};
-
-const defaultOptions: Required<TokenCountOptions> = {
-  textTokenRatio: 4,
-  defaultImageTokens: 1000,
-};
-
-const mergeOptions = (options?: TokenCountOptions) => ({
-  ...defaultOptions,
-  ...(options ?? {}),
-});
-
-export const estimateTextTokens = (text: string, options?: TokenCountOptions) => {
-  const { textTokenRatio } = mergeOptions(options);
-  return Math.ceil(text.length / textTokenRatio);
-};
-
-export const estimateImageTokens = (
-  _image: ArtifactImage,
-  options?: TokenCountOptions
-) => {
-  const { defaultImageTokens } = mergeOptions(options);
-  return defaultImageTokens;
-};
-
-export const countContentTokens = (
-  content: ArtifactContent,
-  options?: TokenCountOptions
-) => {
-  let tokens = 0;
-
-  if (content.text) {
-    tokens += estimateTextTokens(content.text, options);
-  }
-
-  if (content.media?.length) {
-    for (const media of content.media) {
-      tokens += estimateImageTokens(media, options);
-      if (media.text) {
-        tokens += estimateTextTokens(media.text, options);
-      }
-    }
-  }
-
-  return tokens;
-};
-
-export const countArtifactTokens = (
-  artifact: Artifact,
-  options?: TokenCountOptions
-) => {
-  if (typeof artifact.tokens === "number") {
-    return artifact.tokens;
-  }
-
-  return artifact.contents.reduce(
-    (total, content) => total + countContentTokens(content, options),
-    0
-  );
-};
-
-export const countArtifactImages = (artifact: Artifact) => {
-  return artifact.contents.reduce((count, content) => {
-    return count + (content.media?.length ?? 0);
-  }, 0);
-};

package/src/types.test.ts

@@ -1,25 +0,0 @@
-import { test, expect } from "bun:test";
-import type { Artifact, ExtractionResult, ExtractionStrategy, Usage } from "./types";
-
-test("types can be used to build core DTOs", async () => {
-  const usage: Usage = { inputTokens: 1, outputTokens: 2, totalTokens: 3 };
-  const artifact: Artifact = {
-    id: "a1",
-    type: "text",
-    raw: async () => Buffer.from(""),
-    contents: [{ text: "hello" }],
-  };
-
-  const strategy: ExtractionStrategy<{ title: string }> = {
-    name: "test",
-    run: async () => ({ data: { title: "ok" }, usage }),
-  };
-
-  const result: ExtractionResult<{ title: string }> = await strategy.run({
-    artifacts: [artifact],
-    schema: { type: "object" },
-    strategy,
-  });
-
-  expect(result.data.title).toBe("ok");
-});

package/src/types.ts

@@ -1,174 +0,0 @@
-import type { JSONSchemaType } from "ajv";
-import type { DebugLogger } from "./debug/logger";
-
-export type ArtifactType = "text" | "image" | "pdf" | "file";
-
-export type ImageType = "embedded" | "screenshot";
-
-export type ArtifactImage = {
-  type: "image";
-  url?: string;
-  base64?: string;
-  contents?: Buffer;
-  text?: string;
-  x?: number;
-  y?: number;
-  width?: number;
-  height?: number;
-  imageType?: ImageType;
-};
-
-export type ArtifactContent = {
-  page?: number;
-  text?: string;
-  media?: ArtifactImage[];
-};
-
-export interface Artifact {
-  id: string;
-  type: ArtifactType;
-  raw: () => Promise<Buffer>;
-  contents: ArtifactContent[];
-  metadata?: Record<string, unknown>;
-  tokens?: number;
-}
-
-export type Usage = {
-  inputTokens: number;
-  outputTokens: number;
-  totalTokens: number;
-};
-
-export type ExtractionResult<T> = {
-  data: T;
-  usage: Usage;
-  error?: Error;
-};
-
-/**
- * Telemetry adapter interface for tracing extraction operations.
- * This is a minimal interface that matches the full TelemetryAdapter from @struktur/telemetry.
- * SDK users should import adapters from @struktur/telemetry package.
- */
-export interface TelemetryAdapter {
-  readonly name: string;
-  readonly version: string;
-  initialize(): Promise<void>;
-  shutdown(): Promise<void>;
-  startSpan(context: {
-    name: string;
-    kind: "CHAIN" | "LLM" | "TOOL" | "AGENT" | "RETRIEVER" | "EMBEDDING" | "RERANKER";
-    parentSpan?: { id: string; traceId: string };
-    attributes?: Record<string, unknown>;
-    startTime?: number;
-  }): { id: string; traceId: string; name: string; kind: string; startTime: number; parentId?: string };
-  endSpan(span: { id: string }, result?: { status: "ok" | "error"; error?: Error; output?: unknown; latencyMs?: number }): void;
-  recordEvent(span: { id: string }, event: unknown): void;
-  setAttributes(span: { id: string }, attributes: Record<string, unknown>): void;
-  setContext(context: { sessionId?: string; userId?: string; metadata?: Record<string, unknown>; tags?: string[] }): void;
-}
-
-export type StepInfo = {
-  step: number;
-  total?: number;
-  label?: string;
-  detail?: string;
-};
-
-export type ProgressInfo = {
-  current: number;
-  total: number;
-  percent?: number;
-};
-
-export type MessageInfo = {
-  role: "system" | "user" | "assistant" | "tool";
-  content: unknown;
-};
-
-export type TokenUsageInfo = Usage & {
-  model?: string;
-};
-
-export type RetryInfo = {
-	attempt: number;
-	maxAttempts: number;
-	reason?: string;
-};
-
-export type AgentToolStartInfo = {
-	toolName: string;
-	toolCallId: string;
-	args: Record<string, unknown>;
-};
-
-export type AgentToolEndInfo = {
-	toolCallId: string;
-	result?: Record<string, unknown>;
-	error?: string;
-};
-
-export type AgentMessageInfo = {
-	content: string;
-	role?: "assistant" | "user";
-};
-
-export type AgentReasoningInfo = {
-	thought: string;
-};
-
-export type AgentEvents = {
-	onAgentToolStart?: (info: AgentToolStartInfo) => void | Promise<void>;
-	onAgentToolEnd?: (info: AgentToolEndInfo) => void | Promise<void>;
-	onAgentMessage?: (info: AgentMessageInfo) => void | Promise<void>;
-	onAgentReasoning?: (info: AgentReasoningInfo) => void | Promise<void>;
-};
-
-export type ExtractionEvents = {
-	onStep?: (info: StepInfo) => void | Promise<void>;
-	onMessage?: (info: MessageInfo) => void | Promise<void>;
-	onProgress?: (info: ProgressInfo) => void | Promise<void>;
-	onTokenUsage?: (info: TokenUsageInfo) => void | Promise<void>;
-	onRetry?: (info: RetryInfo) => void | Promise<void>;
-} & AgentEvents;
-
-export type AnyJSONSchema = Record<string, unknown>;
-export type TypedJSONSchema<T> = JSONSchemaType<T>;
-
-export type ProviderModelsResult = {
-  provider: string;
-  ok: boolean;
-  models?: string[];
-  error?: string;
-};
-
-export type ExtractionOptions<T> = {
-  artifacts: Artifact[];
-  /**
-   * JSON Schema for the extracted output.
-   * Exactly one of `schema`, `fields`, or an inline schema via the CLI must be provided.
-   */
-  schema?: TypedJSONSchema<T> | AnyJSONSchema;
-  /**
-   * Shorthand schema definition as a comma-separated string of `name` or `name:type` tokens.
-   * E.g. `"title, price:number"`. Defaults to `string` when no type is specified.
-   * Mutually exclusive with `schema`.
-   */
-  fields?: string;
-  strategy: ExtractionStrategy<T>;
-  events?: ExtractionEvents;
-  debug?: DebugLogger;
-  strict?: boolean;
-  /**
-   * Telemetry adapter for tracing extraction operations.
-   * Supports Phoenix (Arize), Langfuse, and other OpenTelemetry-compatible providers.
-   * Import from `@struktur/telemetry` package and pass the adapter here.
-   */
-  telemetry?: TelemetryAdapter | null;
-}
-
-export interface ExtractionStrategy<T> {
-  name: string;
-  run(options: ExtractionOptions<T>): Promise<ExtractionResult<T>>;
-  getEstimatedSteps?: (artifacts: Artifact[]) => number;
-}

package/src/validation/AGENTS.md

@@ -1,7 +0,0 @@
-Validation module
-
-- Purpose: Schema validation and error shaping.
-- Key files: `validator.ts`.
-- Design: `validateOrThrow` compiles schemas and throws `SchemaValidationError` on failure; `createAjv` registers `ajv-formats` for common schema formats and adds custom `artifact-id` format for referencing images in artifacts.
-- Custom formats: `artifact-id` validates strings matching pattern `artifact:ID/images/imageNUM.EXT` (e.g., `artifact:123456/images/image1.jpg`).
-- Tests: `validator.test.ts`.