@struktur/sdk 2.1.2 → 2.2.0

package/src/llm/RetryingRunner.test.ts

@@ -1,174 +0,0 @@
-import { test, expect } from "bun:test";
-import type { JSONSchemaType } from "ajv";
-import { runWithRetries } from "./RetryingRunner";
-
-type Output = { title: string };
-
-const schema: JSONSchemaType<Output> = {
-  type: "object",
-  properties: { title: { type: "string" } },
-  required: ["title"],
-  additionalProperties: false,
-};
-
-test("runWithRetries emits onRetry event when retrying", async () => {
-  let calls = 0;
-  const retryEvents: Array<{ attempt: number; maxAttempts: number; reason?: string }> = [];
-  
-  const result = await runWithRetries<Output>({
-    model: {},
-    schema,
-    system: "sys",
-    user: "user",
-    execute: async () => {
-      calls += 1;
-      if (calls === 1) {
-        return {
-          data: { title: 123 } as unknown as Output,
-          usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
-        };
-      }
-      return {
-        data: { title: "ok" },
-        usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
-      };
-    },
-    events: {
-      onRetry: (info) => {
-        retryEvents.push(info);
-      },
-    },
-  });
-
-  expect(result.data.title).toBe("ok");
-  expect(calls).toBe(2);
-  expect(retryEvents).toHaveLength(1);
-  expect(retryEvents[0]?.attempt).toBe(2);
-  expect(retryEvents[0]?.maxAttempts).toBe(3);
-  expect(retryEvents[0]?.reason).toBe("schema_validation_failed");
-});
-
-test("runWithRetries retries on validation error", async () => {
-  let calls = 0;
-  const result = await runWithRetries<Output>({
-    model: {},
-    schema,
-    system: "sys",
-    user: "user",
-    execute: async () => {
-      calls += 1;
-      if (calls === 1) {
-        return {
-          data: { title: 123 } as unknown as Output,
-          usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
-        };
-      }
-      return {
-        data: { title: "ok" },
-        usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
-      };
-    },
-  });
-
-  expect(result.data.title).toBe("ok");
-  expect(calls).toBe(2);
-});
-
-test("runWithRetries with strict=false retries on missing required fields until max attempts", async () => {
-  let calls = 0;
-  
-  await expect(
-    runWithRetries<Output>({
-      model: {},
-      schema,
-      system: "sys",
-      user: "user",
-      strict: false,
-      maxAttempts: 2,
-      execute: async () => {
-        calls += 1;
-        return {
-          data: {} as Output,
-          usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
-        };
-      },
-    })
-  ).rejects.toThrow();
-  
-  expect(calls).toBe(2);
-});
-
-test("runWithRetries with strict=true validates required fields on every attempt", async () => {
-  let calls = 0;
-  
-  await expect(
-    runWithRetries<Output>({
-      model: {},
-      schema,
-      system: "sys",
-      user: "user",
-      strict: true,
-      maxAttempts: 2,
-      execute: async () => {
-        calls += 1;
-        return {
-          data: {} as Output,
-          usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
-        };
-      },
-    })
-  ).rejects.toThrow();
-  
-  expect(calls).toBe(2);
-});
-
-test("runWithRetries with strict=false still validates type errors", async () => {
-  let calls = 0;
-  const result = await runWithRetries<Output>({
-    model: {},
-    schema,
-    system: "sys",
-    user: "user",
-    strict: false,
-    execute: async () => {
-      calls += 1;
-      if (calls === 1) {
-        return {
-          data: { title: 123 } as unknown as Output,
-          usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
-        };
-      }
-      return {
-        data: { title: "ok" },
-        usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
-      };
-    },
-  });
-
-  expect(result.data.title).toBe("ok");
-  expect(calls).toBe(2);
-});
-
-test("runWithRetries enforces strict validation on final attempt even with strict=false", async () => {
-  let calls = 0;
-  
-  await expect(
-    runWithRetries<Output>({
-      model: {},
-      schema,
-      system: "sys",
-      user: "user",
-      strict: false,
-      maxAttempts: 2,
-      execute: async () => {
-        calls += 1;
-        return {
-          data: {} as Output,
-          usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
-        };
-      },
-    })
-  ).rejects.toThrow();
-  
-  expect(calls).toBe(2);
-});

package/src/llm/RetryingRunner.ts

@@ -1,270 +0,0 @@
-import {
-  createAjv,
-  validateOrThrow,
-  SchemaValidationError,
-  validateAllowingMissingRequired,
-} from "../validation/validator";
-import type { ModelMessage } from "ai";
-import type { ExtractionEvents, Usage, TelemetryAdapter } from "../types";
-import type { DebugLogger } from "../debug/logger";
-import { generateStructured } from "./LLMClient";
-import type { UserContent } from "./message";
-
-export type RetryOptions<T> = {
-  model: unknown;
-  schema: unknown;
-  system: string;
-  user: UserContent;
-  events?: ExtractionEvents;
-  maxAttempts?: number;
-  schemaName?: string;
-  execute?: typeof generateStructured<T>;
-  strict?: boolean;
-  debug?: DebugLogger;
-  callId?: string;
-  /**
-   * Telemetry adapter for tracing validation and retries
-   */
-  telemetry?: TelemetryAdapter;
-  /**
-   * Parent span for creating hierarchical traces
-   */
-  parentSpan?: { id: string; traceId: string; name: string; kind: string; startTime: number; parentId?: string };
-};
-
-export const runWithRetries = async <T>(options: RetryOptions<T>) => {
-  const { telemetry, parentSpan } = options;
-  
-  // Start validation/retry span if telemetry is enabled
-  const retrySpan = telemetry?.startSpan({
-    name: "struktur.validation_retry",
-    kind: "CHAIN",
-    parentSpan,
-    attributes: {
-      "retry.max_attempts": options.maxAttempts ?? 3,
-      "retry.schema_name": options.schemaName ?? "extract",
-    },
-  });
-
-  const ajv = createAjv();
-  const maxAttempts = options.maxAttempts ?? 3;
-  const messages: ModelMessage[] = [{ role: "user", content: options.user }];
-  const debug = options.debug;
-  const callId =
-    options.callId ??
-    `call_${Date.now()}_${Math.random().toString(36).slice(2, 11)}`;
-
-  let usage: Usage = { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
-  let lastError: Error | undefined;
-
-  // Log LLM call start
-  const systemLength = options.system.length;
-  const userLength =
-    typeof options.user === "string"
-      ? options.user.length
-      : JSON.stringify(options.user).length;
-
-  debug?.llmCallStart({
-    callId,
-    model: JSON.stringify(options.model),
-    schemaName: options.schemaName,
-    systemLength,
-    userLength,
-    artifactCount: Array.isArray(options.user) ? options.user.length : 0,
-  });
-
-  debug?.promptSystem({ callId, system: options.system });
-  debug?.promptUser({ callId, user: options.user });
-
-  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
-    const executor = options.execute ?? generateStructured;
-    const isFinalAttempt = attempt === maxAttempts;
-    const useStrictValidation = options.strict === true || isFinalAttempt;
-
-    debug?.validationStart({
-      callId,
-      attempt,
-      maxAttempts,
-      strict: useStrictValidation,
-    });
-
-    const startTime = Date.now();
-    const result = await executor({
-      model: options.model,
-      schema: options.schema,
-      schemaName: options.schemaName,
-      system: options.system,
-      user: options.user,
-      messages,
-      strict: options.strict,
-      telemetry,
-      parentSpan: retrySpan,
-    });
-    const durationMs = Date.now() - startTime;
-
-    usage = {
-      inputTokens: usage.inputTokens + result.usage.inputTokens,
-      outputTokens: usage.outputTokens + result.usage.outputTokens,
-      totalTokens: usage.totalTokens + result.usage.totalTokens,
-    };
-
-    debug?.rawResponse({ callId, response: result.data });
-
-    try {
-      if (useStrictValidation) {
-        const validated = validateOrThrow<T>(
-          ajv,
-          options.schema as never,
-          result.data,
-        );
-
-        debug?.validationSuccess({ callId, attempt });
-        debug?.llmCallComplete({
-          callId,
-          success: true,
-          inputTokens: usage.inputTokens,
-          outputTokens: usage.outputTokens,
-          totalTokens: usage.totalTokens,
-          durationMs,
-        });
-
-        // Record successful validation
-        if (retrySpan && telemetry) {
-          telemetry.recordEvent(retrySpan, {
-            type: "validation",
-            attempt,
-            maxAttempts,
-            schema: options.schema,
-            input: result.data,
-            success: true,
-            latencyMs: durationMs,
-          });
-          telemetry.endSpan(retrySpan, {
-            status: "ok",
-            output: validated,
-            latencyMs: durationMs,
-          });
-        }
-
-        return { data: validated, usage };
-      } else {
-        const validationResult = validateAllowingMissingRequired<T>(
-          ajv,
-          options.schema as never,
-          result.data,
-          isFinalAttempt,
-        );
-
-        if (validationResult.valid) {
-          debug?.validationSuccess({ callId, attempt });
-          debug?.llmCallComplete({
-            callId,
-            success: true,
-            inputTokens: usage.inputTokens,
-            outputTokens: usage.outputTokens,
-            totalTokens: usage.totalTokens,
-            durationMs,
-          });
-
-          // Record successful validation
-          if (retrySpan && telemetry) {
-            telemetry.recordEvent(retrySpan, {
-              type: "validation",
-              attempt,
-              maxAttempts,
-              schema: options.schema,
-              input: result.data,
-              success: true,
-              latencyMs: durationMs,
-            });
-            telemetry.endSpan(retrySpan, {
-              status: "ok",
-              output: validationResult.data,
-              latencyMs: durationMs,
-            });
-          }
-
-          return { data: validationResult.data, usage };
-        }
-
-        throw new SchemaValidationError(
-          "Schema validation failed",
-          validationResult.errors,
-        );
-      }
-    } catch (error) {
-      lastError = error as Error;
-
-      if (error instanceof SchemaValidationError) {
-        debug?.validationFailed({
-          callId,
-          attempt,
-          errors: error.errors,
-        });
-
-        // Record failed validation
-        if (retrySpan && telemetry) {
-          telemetry.recordEvent(retrySpan, {
-            type: "validation",
-            attempt,
-            maxAttempts,
-            schema: options.schema,
-            input: result.data,
-            success: false,
-            errors: error.errors,
-            latencyMs: durationMs,
-          });
-        }
-
-        // Emit retry event before attempting retry
-        const nextAttempt = attempt + 1;
-        if (nextAttempt <= maxAttempts) {
-          await options.events?.onRetry?.({
-            attempt: nextAttempt,
-            maxAttempts,
-            reason: "schema_validation_failed",
-          });
-
-          debug?.retry({
-            callId,
-            attempt: nextAttempt,
-            maxAttempts,
-            reason: "schema_validation_failed",
-          });
-        }
-
-        const errorPayload = JSON.stringify(error.errors, null, 2);
-        const errorMessage = `<validation-errors>\n${errorPayload}\n</validation-errors>`;
-        messages.push({ role: "user", content: errorMessage });
-        await options.events?.onMessage?.({
-          role: "user",
-          content: errorMessage,
-        });
-        continue;
-      }
-
-      debug?.llmCallComplete({
-        callId,
-        success: false,
-        inputTokens: usage.inputTokens,
-        outputTokens: usage.outputTokens,
-        totalTokens: usage.totalTokens,
-        durationMs,
-        error: (error as Error).message,
-      });
-
-      // Record error in telemetry
-      if (retrySpan && telemetry) {
-        telemetry.endSpan(retrySpan, {
-          status: "error",
-          error: error as Error,
-          latencyMs: durationMs,
-        });
-      }
-
-      break;
-    }
-  }
-
-  throw lastError ?? new Error("Unknown extraction error");
-};

package/src/llm/message.test.ts

@@ -1,42 +0,0 @@
-import { test, expect } from "bun:test";
-import type { Artifact } from "../types";
-import { buildUserContent } from "./message";
-
-const makeArtifact = (contents: Artifact["contents"]): Artifact => ({
-  id: "a1",
-  type: "text",
-  raw: async () => Buffer.from(""),
-  contents,
-});
-
-test("buildUserContent returns text when no images", () => {
-  const artifacts = [makeArtifact([{ text: "hello" }])];
-  const content = buildUserContent("prompt", artifacts);
-
-  expect(content).toBe("prompt");
-});
-
-test("buildUserContent appends images in order", () => {
-  const artifacts: Artifact[] = [
-    makeArtifact([
-      { media: [{ type: "image", base64: "base" }] },
-      { media: [{ type: "image", url: "https://example.com/img.png" }] },
-    ]),
-    {
-      id: "a2",
-      type: "image",
-      raw: async () => Buffer.from(""),
-      contents: [{ media: [{ type: "image", contents: Buffer.from([1]) }] }],
-    },
-  ];
-
-  const content = buildUserContent("prompt", artifacts);
-  expect(Array.isArray(content)).toBe(true);
-
-  if (Array.isArray(content)) {
-    expect(content[0]).toEqual({ type: "text", text: "prompt" });
-    expect(content[1]).toEqual({ type: "image", image: "base" });
-    expect(content[2]).toEqual({ type: "image", image: "https://example.com/img.png" });
-    expect(content[3]).toEqual({ type: "image", image: Buffer.from([1]) });
-  }
-});

package/src/llm/message.ts

@@ -1,47 +0,0 @@
-import type { Artifact } from "../types";
-
-export type ImagePart = {
-  type: "image";
-  image: string | Buffer;
-};
-
-export type TextPart = {
-  type: "text";
-  text: string;
-};
-
-export type UserContent = string | Array<TextPart | ImagePart>;
-
-const collectImages = (artifacts: Artifact[]): ImagePart[] => {
-  const parts: ImagePart[] = [];
-
-  for (const artifact of artifacts) {
-    for (const content of artifact.contents) {
-      if (!content.media?.length) {
-        continue;
-      }
-
-      for (const media of content.media) {
-        if (media.contents) {
-          parts.push({ type: "image", image: media.contents });
-        } else if (media.base64) {
-          parts.push({ type: "image", image: media.base64 });
-        } else if (media.url) {
-          parts.push({ type: "image", image: media.url });
-        }
-      }
-    }
-  }
-
-  return parts;
-};
-
-export const buildUserContent = (text: string, artifacts: Artifact[]): UserContent => {
-  const images = collectImages(artifacts);
-
-  if (images.length === 0) {
-    return text;
-  }
-
-  return [{ type: "text", text }, ...images];
-};

package/src/llm/models.test.ts

@@ -1,82 +0,0 @@
-import { test, expect } from "bun:test";
-import { __testing__ } from "./models";
-
-test("parseOpenAiModels returns model ids", () => {
-  const models = __testing__.parseOpenAiModels({
-    object: "list",
-    data: [{ id: "gpt-4o-mini" }, { id: "gpt-4o" }],
-  });
-
-  expect(models).toEqual(["gpt-4o-mini", "gpt-4o"]);
-});
-
-test("parseAnthropicModels returns model ids", () => {
-  const models = __testing__.parseAnthropicModels({
-    data: [{ id: "claude-3-5-sonnet-20241022" }],
-  });
-
-  expect(models).toEqual(["claude-3-5-sonnet-20241022"]);
-});
-
-test("parseGoogleModels strips models prefix", () => {
-  const models = __testing__.parseGoogleModels({
-    models: [{ name: "models/gemini-1.5-flash" }],
-  });
-
-  expect(models).toEqual(["gemini-1.5-flash"]);
-});
-
-test("parseOpenRouterModels returns model ids", () => {
-  const models = __testing__.parseOpenRouterModels({
-    data: [{ id: "openai/gpt-4o" }, { id: "anthropic/claude-3.5-sonnet" }],
-  });
-
-  expect(models).toEqual(["openai/gpt-4o", "anthropic/claude-3.5-sonnet"]);
-});
-
-test("parseOpenAiModels handles empty data", () => {
-  const models = __testing__.parseOpenAiModels({});
-  expect(models).toEqual([]);
-});
-
-test("parseOpenAiModels filters out undefined ids", () => {
-  const models = __testing__.parseOpenAiModels({
-    data: [{ id: "gpt-4" }, { notId: "bad" }],
-  });
-  expect(models).toEqual(["gpt-4"]);
-});
-
-test("parseGoogleModels handles empty models", () => {
-  const models = __testing__.parseGoogleModels({});
-  expect(models).toEqual([]);
-});
-
-test("pickCheapestModel prefers known cheap models", () => {
-  const models = ["gpt-4o", "gpt-4o-mini"];
-  expect(__testing__.pickCheapestModel("openai", models)).toBe("gpt-4o-mini");
-});
-
-test("pickCheapestModel returns first model if no preference matches", () => {
-  const models = ["unknown-model-1", "unknown-model-2"];
-  expect(__testing__.pickCheapestModel("openai", models)).toBe("unknown-model-1");
-});
-
-test("pickCheapestModel matches prefix for versioned models", () => {
-  const models = ["gpt-4o-mini-2024-07-18", "gpt-4o-2024-05-13"];
-  expect(__testing__.pickCheapestModel("openai", models)).toBe("gpt-4o-mini-2024-07-18");
-});
-
-test("pickCheapestModel handles anthropic preferences", () => {
-  const models = ["claude-3-opus", "claude-3-5-haiku-20241022"];
-  expect(__testing__.pickCheapestModel("anthropic", models)).toBe("claude-3-5-haiku-20241022");
-});
-
-test("pickCheapestModel handles google preferences", () => {
-  const models = ["gemini-1.5-pro", "gemini-2.0-flash"];
-  expect(__testing__.pickCheapestModel("google", models)).toBe("gemini-2.0-flash");
-});
-
-test("pickCheapestModel handles unknown provider", () => {
-  const models = ["model-a", "model-b"];
-  expect(__testing__.pickCheapestModel("unknown", models)).toBe("model-a");
-});