@struktur/sdk 0.1.0

package/src/strategies/utils.ts

@@ -0,0 +1,56 @@
+import type { Artifact, ExtractionEvents, Usage } from "../types";
+import type { DebugLogger } from "../debug/logger";
+import { batchArtifacts, type BatchOptions } from "../chunking/ArtifactBatcher";
+import { buildUserContent } from "../llm/message";
+import { runWithRetries } from "../llm/RetryingRunner";
+
+export const serializeSchema = (schema: unknown) => {
+  return JSON.stringify(schema);
+};
+
+export const mergeUsage = (usages: Usage[]) => {
+  return usages.reduce(
+    (acc, usage) => ({
+      inputTokens: acc.inputTokens + usage.inputTokens,
+      outputTokens: acc.outputTokens + usage.outputTokens,
+      totalTokens: acc.totalTokens + usage.totalTokens,
+    }),
+    { inputTokens: 0, outputTokens: 0, totalTokens: 0 }
+  );
+};
+
+export const getBatches = (
+  artifacts: Artifact[],
+  options: BatchOptions,
+  debug?: DebugLogger
+) => {
+  return batchArtifacts(artifacts, { ...options, debug });
+};
+
+export const extractWithPrompt = async <T>(options: {
+  model: unknown;
+  schema: unknown;
+  system: string;
+  user: string;
+  artifacts: Artifact[];
+  events?: ExtractionEvents;
+  execute?: typeof runWithRetries<T>;
+  strict?: boolean;
+  debug?: DebugLogger;
+  callId?: string;
+}) => {
+  const userContent = buildUserContent(options.user, options.artifacts);
+  const result = await runWithRetries<T>({
+    model: options.model,
+    schema: options.schema,
+    system: options.system,
+    user: userContent,
+    events: options.events,
+    execute: options.execute,
+    strict: options.strict,
+    debug: options.debug,
+    callId: options.callId,
+  });
+
+  return result;
+};

package/src/tokenization.test.ts

@@ -0,0 +1,119 @@
+import { test, expect } from "bun:test";
+import type { Artifact } from "./types";
+import {
+  countArtifactTokens,
+  estimateTextTokens,
+  estimateImageTokens,
+  countContentTokens,
+} from "./tokenization";
+
+test("estimateTextTokens uses default ratio", () => {
+  expect(estimateTextTokens("abcd")).toBe(1);
+  expect(estimateTextTokens("abcdefgh")).toBe(2);
+});
+
+test("estimateTextTokens uses custom ratio", () => {
+  expect(estimateTextTokens("abcd", { textTokenRatio: 2 })).toBe(2);
+  expect(estimateTextTokens("abcdefgh", { textTokenRatio: 2 })).toBe(4);
+});
+
+test("estimateImageTokens uses default image tokens", () => {
+  expect(estimateImageTokens({ type: "image" })).toBe(1000);
+});
+
+test("estimateImageTokens uses custom image tokens", () => {
+  expect(estimateImageTokens({ type: "image" }, { defaultImageTokens: 500 })).toBe(500);
+});
+
+test("countArtifactTokens honors artifact.tokens override", () => {
+  const artifact: Artifact = {
+    id: "a1",
+    type: "text",
+    raw: async () => Buffer.from(""),
+    contents: [{ text: "hello" }],
+    tokens: 42,
+  };
+
+  expect(countArtifactTokens(artifact)).toBe(42);
+});
+
+test("countArtifactTokens sums text and media tokens", () => {
+  const artifact: Artifact = {
+    id: "a2",
+    type: "text",
+    raw: async () => Buffer.from(""),
+    contents: [
+      {
+        text: "abcdefgh",
+        media: [{ type: "image" }],
+      },
+    ],
+  };
+
+  const tokens = countArtifactTokens(artifact);
+  expect(tokens).toBe(1002);
+});
+
+test("countContentTokens counts text tokens", () => {
+  const tokens = countContentTokens({ text: "abcdefgh" });
+  expect(tokens).toBe(2);
+});
+
+test("countContentTokens counts image tokens", () => {
+  const tokens = countContentTokens({ media: [{ type: "image" }] });
+  expect(tokens).toBe(1000);
+});
+
+test("countContentTokens counts image text tokens", () => {
+  const tokens = countContentTokens({
+    media: [{ type: "image", text: "abcd" }],
+  });
+  expect(tokens).toBe(1001);
+});
+
+test("countContentTokens sums multiple images", () => {
+  const tokens = countContentTokens({
+    media: [{ type: "image" }, { type: "image" }],
+  });
+  expect(tokens).toBe(2000);
+});
+
+test("countContentTokens handles empty content", () => {
+  const tokens = countContentTokens({});
+  expect(tokens).toBe(0);
+});
+
+test("countArtifactTokens sums multiple contents", () => {
+  const artifact: Artifact = {
+    id: "a1",
+    type: "text",
+    raw: async () => Buffer.from(""),
+    contents: [
+      { text: "abcd" },
+      { text: "efgh" },
+      { media: [{ type: "image" }] },
+    ],
+  };
+
+  const tokens = countArtifactTokens(artifact);
+  expect(tokens).toBe(1002);
+});
+
+test("countArtifactTokens with custom options", () => {
+  const artifact: Artifact = {
+    id: "a1",
+    type: "text",
+    raw: async () => Buffer.from(""),
+    contents: [
+      { text: "abcdefgh" },
+      { media: [{ type: "image" }] },
+    ],
+  };
+
+  const tokens = countArtifactTokens(artifact, {
+    textTokenRatio: 2,
+    defaultImageTokens: 500,
+  });
+
+  expect(tokens).toBe(504);
+});

package/src/tokenization.ts

@@ -0,0 +1,71 @@
+import type { Artifact, ArtifactContent, ArtifactImage } from "./types";
+
+export type TokenCountOptions = {
+  textTokenRatio?: number;
+  defaultImageTokens?: number;
+};
+
+const defaultOptions: Required<TokenCountOptions> = {
+  textTokenRatio: 4,
+  defaultImageTokens: 1000,
+};
+
+const mergeOptions = (options?: TokenCountOptions) => ({
+  ...defaultOptions,
+  ...(options ?? {}),
+});
+
+export const estimateTextTokens = (text: string, options?: TokenCountOptions) => {
+  const { textTokenRatio } = mergeOptions(options);
+  return Math.ceil(text.length / textTokenRatio);
+};
+
+export const estimateImageTokens = (
+  _image: ArtifactImage,
+  options?: TokenCountOptions
+) => {
+  const { defaultImageTokens } = mergeOptions(options);
+  return defaultImageTokens;
+};
+
+export const countContentTokens = (
+  content: ArtifactContent,
+  options?: TokenCountOptions
+) => {
+  let tokens = 0;
+
+  if (content.text) {
+    tokens += estimateTextTokens(content.text, options);
+  }
+
+  if (content.media?.length) {
+    for (const media of content.media) {
+      tokens += estimateImageTokens(media, options);
+      if (media.text) {
+        tokens += estimateTextTokens(media.text, options);
+      }
+    }
+  }
+
+  return tokens;
+};
+
+export const countArtifactTokens = (
+  artifact: Artifact,
+  options?: TokenCountOptions
+) => {
+  if (typeof artifact.tokens === "number") {
+    return artifact.tokens;
+  }
+
+  return artifact.contents.reduce(
+    (total, content) => total + countContentTokens(content, options),
+    0
+  );
+};
+
+export const countArtifactImages = (artifact: Artifact) => {
+  return artifact.contents.reduce((count, content) => {
+    return count + (content.media?.length ?? 0);
+  }, 0);
+};

package/src/types.test.ts

@@ -0,0 +1,25 @@
+import { test, expect } from "bun:test";
+import type { Artifact, ExtractionResult, ExtractionStrategy, Usage } from "./types";
+
+test("types can be used to build core DTOs", async () => {
+  const usage: Usage = { inputTokens: 1, outputTokens: 2, totalTokens: 3 };
+  const artifact: Artifact = {
+    id: "a1",
+    type: "text",
+    raw: async () => Buffer.from(""),
+    contents: [{ text: "hello" }],
+  };
+
+  const strategy: ExtractionStrategy<{ title: string }> = {
+    name: "test",
+    run: async () => ({ data: { title: "ok" }, usage }),
+  };
+
+  const result: ExtractionResult<{ title: string }> = await strategy.run({
+    artifacts: [artifact],
+    schema: { type: "object" },
+    strategy,
+  });
+
+  expect(result.data.title).toBe("ok");
+});

package/src/types.ts

@@ -0,0 +1,116 @@
+import type { JSONSchemaType } from "ajv";
+import type { DebugLogger } from "./debug/logger";
+
+export type ArtifactType = "text" | "image" | "pdf" | "file";
+
+export type ImageType = "embedded" | "screenshot";
+
+export type ArtifactImage = {
+  type: "image";
+  url?: string;
+  base64?: string;
+  contents?: Buffer;
+  text?: string;
+  x?: number;
+  y?: number;
+  width?: number;
+  height?: number;
+  imageType?: ImageType;
+};
+
+export type ArtifactContent = {
+  page?: number;
+  text?: string;
+  media?: ArtifactImage[];
+};
+
+export interface Artifact {
+  id: string;
+  type: ArtifactType;
+  raw: () => Promise<Buffer>;
+  contents: ArtifactContent[];
+  metadata?: Record<string, unknown>;
+  tokens?: number;
+}
+
+export type Usage = {
+  inputTokens: number;
+  outputTokens: number;
+  totalTokens: number;
+};
+
+export type ExtractionResult<T> = {
+  data: T;
+  usage: Usage;
+  error?: Error;
+};
+
+export type StepInfo = {
+  step: number;
+  total?: number;
+  label?: string;
+};
+
+export type ProgressInfo = {
+  current: number;
+  total: number;
+  percent?: number;
+};
+
+export type MessageInfo = {
+  role: "system" | "user" | "assistant" | "tool";
+  content: unknown;
+};
+
+export type TokenUsageInfo = Usage & {
+  model?: string;
+};
+
+export type RetryInfo = {
+  attempt: number;
+  maxAttempts: number;
+  reason?: string;
+};
+
+export type ExtractionEvents = {
+  onStep?: (info: StepInfo) => void | Promise<void>;
+  onMessage?: (info: MessageInfo) => void | Promise<void>;
+  onProgress?: (info: ProgressInfo) => void | Promise<void>;
+  onTokenUsage?: (info: TokenUsageInfo) => void | Promise<void>;
+  onRetry?: (info: RetryInfo) => void | Promise<void>;
+};
+
+export type AnyJSONSchema = Record<string, unknown>;
+export type TypedJSONSchema<T> = JSONSchemaType<T>;
+
+export type ProviderModelsResult = {
+  provider: string;
+  ok: boolean;
+  models?: string[];
+  error?: string;
+};
+
+export type ExtractionOptions<T> = {
+  artifacts: Artifact[];
+  /**
+   * JSON Schema for the extracted output.
+   * Exactly one of `schema`, `fields`, or an inline schema via the CLI must be provided.
+   */
+  schema?: TypedJSONSchema<T> | AnyJSONSchema;
+  /**
+   * Shorthand schema definition as a comma-separated string of `name` or `name:type` tokens.
+   * E.g. `"title, price:number"`. Defaults to `string` when no type is specified.
+   * Mutually exclusive with `schema`.
+   */
+  fields?: string;
+  strategy: ExtractionStrategy<T>;
+  events?: ExtractionEvents;
+  debug?: DebugLogger;
+  strict?: boolean;
+}
+
+export interface ExtractionStrategy<T> {
+  name: string;
+  run(options: ExtractionOptions<T>): Promise<ExtractionResult<T>>;
+  getEstimatedSteps?: (artifacts: Artifact[]) => number;
+}

package/src/validation/AGENTS.md

@@ -0,0 +1,6 @@
+Validation module
+
+- Purpose: Ajv schema validation and error shaping.
+- Key files: `validator.ts`.
+- Design: `validateOrThrow` compiles schemas and throws `SchemaValidationError` on failure; `createAjv` registers `ajv-formats` for common schema formats.
+- Tests: `validator.test.ts`.

package/src/validation/validator.test.ts

@@ -0,0 +1,172 @@
+import { test, expect } from "bun:test";
+import type { JSONSchemaType, ErrorObject } from "ajv";
+import { createAjv, validateOrThrow, SchemaValidationError, isRequiredError, validateAllowingMissingRequired } from "./validator";
+
+type Person = {
+  name: string;
+  age: number;
+};
+
+const personSchema: JSONSchemaType<Person> = {
+  type: "object",
+  properties: {
+    name: { type: "string" },
+    age: { type: "number" },
+  },
+  required: ["name", "age"],
+  additionalProperties: false,
+};
+
+type NestedSchema = {
+  user: {
+    name: string;
+    email: string;
+  };
+  items: string[];
+};
+
+const nestedSchema: JSONSchemaType<NestedSchema> = {
+  type: "object",
+  properties: {
+    user: {
+      type: "object",
+      properties: {
+        name: { type: "string" },
+        email: { type: "string", format: "email" },
+      },
+      required: ["name", "email"],
+    },
+    items: {
+      type: "array",
+      items: { type: "string" },
+    },
+  },
+  required: ["user", "items"],
+};
+
+test("validateOrThrow returns typed data for valid input", () => {
+  const ajv = createAjv();
+  const data = validateOrThrow<Person>(ajv, personSchema, {
+    name: "Ada",
+    age: 33,
+  });
+
+  expect(data.name).toBe("Ada");
+  expect(data.age).toBe(33);
+});
+
+test("validateOrThrow throws SchemaValidationError for invalid input", () => {
+  const ajv = createAjv();
+  try {
+    validateOrThrow<Person>(ajv, personSchema, { name: "Ada" });
+    throw new Error("Expected validation error");
+  } catch (error) {
+    expect(error).toBeInstanceOf(SchemaValidationError);
+    const validationError = error as SchemaValidationError;
+    expect(validationError.errors.length).toBeGreaterThan(0);
+  }
+});
+
+test("createAjv supports common formats", () => {
+  const ajv = createAjv();
+  const schema: JSONSchemaType<string> = { type: "string", format: "email" };
+
+  const data = validateOrThrow<string>(ajv, schema, "test@example.com");
+  expect(data).toBe("test@example.com");
+
+  try {
+    validateOrThrow<string>(ajv, schema, "not-an-email");
+    throw new Error("Expected validation error");
+  } catch (error) {
+    expect(error).toBeInstanceOf(SchemaValidationError);
+  }
+});
+
+test("isRequiredError identifies required constraint violations", () => {
+  const requiredError = { 
+    keyword: "required", 
+    params: { missingProperty: "name" },
+    instancePath: "",
+    schemaPath: "#/required"
+  } as unknown as ErrorObject;
+  const typeError = { 
+    keyword: "type", 
+    params: { type: "string" },
+    instancePath: "/age",
+    schemaPath: "#/properties/age/type"
+  } as unknown as ErrorObject;
+  
+  expect(isRequiredError(requiredError)).toBe(true);
+  expect(isRequiredError(typeError)).toBe(false);
+});
+
+test("validateAllowingMissingRequired ignores required errors but catches type errors", () => {
+  const ajv = createAjv();
+  
+  const resultMissingRequired = validateAllowingMissingRequired<Person>(
+    ajv,
+    personSchema,
+    { name: "Ada" }
+  );
+  expect(resultMissingRequired.valid).toBe(true);
+  if (resultMissingRequired.valid) {
+    expect(resultMissingRequired.data.name).toBe("Ada");
+  }
+  
+  const resultTypeError = validateAllowingMissingRequired<Person>(
+    ajv,
+    personSchema,
+    { name: "Ada", age: "thirty-three" }
+  );
+  expect(resultTypeError.valid).toBe(false);
+  if (!resultTypeError.valid) {
+    expect(resultTypeError.errors.some(e => e.keyword === "type")).toBe(true);
+  }
+  
+  const resultValid = validateAllowingMissingRequired<Person>(
+    ajv,
+    personSchema,
+    { name: "Ada", age: 33 }
+  );
+  expect(resultValid.valid).toBe(true);
+  if (resultValid.valid) {
+    expect(resultValid.data.name).toBe("Ada");
+    expect(resultValid.data.age).toBe(33);
+  }
+});
+
+test("validateAllowingMissingRequired handles format errors", () => {
+  const ajv = createAjv();
+  
+  const result = validateAllowingMissingRequired<NestedSchema>(
+    ajv,
+    nestedSchema,
+    {
+      user: { name: "Ada", email: "not-an-email" },
+      items: ["a", "b"]
+    }
+  );
+  
+  expect(result.valid).toBe(false);
+  if (!result.valid) {
+    expect(result.errors.some(e => e.keyword === "format")).toBe(true);
+  }
+});
+
+test("validateAllowingMissingRequired handles nested required field errors", () => {
+  const ajv = createAjv();
+  
+  const result = validateAllowingMissingRequired<NestedSchema>(
+    ajv,
+    nestedSchema,
+    {
+      user: { name: "Ada" },
+      items: ["a"]
+    }
+  );
+  
+  expect(result.valid).toBe(true);
+  if (result.valid) {
+    expect(result.data.user.name).toBe("Ada");
+  }
+});

package/src/validation/validator.ts

@@ -0,0 +1,82 @@
+import Ajv, { type AnySchema, type ErrorObject, type JSONSchemaType } from "ajv";
+import addFormats from "ajv-formats";
+
+export type ValidationErrors = ErrorObject[];
+
+export type ValidationMode = 'strict' | 'lenient';
+
+export type ValidationResult<T> = 
+  | { valid: true; data: T }
+  | { valid: false; errors: ErrorObject[] };
+
+export class SchemaValidationError extends Error {
+  public readonly errors: ValidationErrors;
+
+  constructor(message: string, errors: ValidationErrors) {
+    super(message);
+    this.name = "SchemaValidationError";
+    this.errors = errors;
+  }
+}
+
+export const createAjv = () => {
+  const ajv = new Ajv({
+    allErrors: true,
+    strict: false,
+    allowUnionTypes: true,
+  });
+  addFormats(ajv);
+  return ajv;
+};
+
+export type SchemaInput<T> = JSONSchemaType<T> | AnySchema;
+
+export const validateOrThrow = <T>(
+  ajv: Ajv,
+  schema: SchemaInput<T>,
+  data: unknown
+): T => {
+  const validate = ajv.compile<T>(schema);
+  const valid = validate(data);
+
+  if (!valid) {
+    const errors = validate.errors ?? [];
+    const message = "Schema validation failed";
+    throw new SchemaValidationError(message, errors);
+  }
+
+  return data as T;
+};
+
+export const isRequiredError = (error: ErrorObject): boolean => {
+  return error.keyword === "required";
+};
+
+export const validateAllowingMissingRequired = <T>(
+  ajv: Ajv,
+  schema: SchemaInput<T>,
+  data: unknown,
+  isFinalAttempt: boolean = true
+): ValidationResult<T> => {
+  const validate = ajv.compile<T>(schema);
+  const valid = validate(data);
+
+  if (valid) {
+    return { valid: true, data: data as T };
+  }
+
+  const errors = validate.errors ?? [];
+  const nonRequiredErrors = errors.filter((error) => !isRequiredError(error));
+
+  if (nonRequiredErrors.length === 0) {
+    // Only required field errors
+    // On final attempt, accept partial data
+    // On non-final attempts, return invalid to trigger retry
+    if (isFinalAttempt) {
+      return { valid: true, data: data as T };
+    }
+    return { valid: false, errors };
+  }
+
+  return { valid: false, errors: nonRequiredErrors };
+};

package/tsconfig.json

@@ -0,0 +1,22 @@
+{
+  "compilerOptions": {
+    "lib": ["ESNext"],
+    "target": "ESNext",
+    "module": "Preserve",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+    "noUncheckedIndexedAccess": true,
+    "noImplicitOverride": true,
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}