@struktur/sdk 0.1.0

package/src/strategies/SequentialAutoMergeStrategy.ts

@@ -0,0 +1,222 @@
+import type { ExtractionResult, ExtractionStrategy } from "../types";
+import type { ExtractionOptions } from "../types";
+import { buildExtractorPrompt } from "../prompts/ExtractorPrompt";
+import { buildDeduplicationPrompt } from "../prompts/DeduplicationPrompt";
+import {
+  extractWithPrompt,
+  getBatches,
+  mergeUsage,
+  serializeSchema,
+} from "./utils";
+import { SmartDataMerger } from "../merge/SmartDataMerger";
+import {
+  findExactDuplicatesWithHashing,
+  deduplicateByIndices,
+} from "../merge/Deduplicator";
+import { runWithRetries } from "../llm/RetryingRunner";
+
+export type SequentialAutoMergeStrategyConfig = {
+  model: unknown;
+  chunkSize: number;
+  maxImages?: number;
+  outputInstructions?: string;
+  dedupeModel?: unknown;
+  execute?: typeof runWithRetries;
+  dedupeExecute?: typeof runWithRetries;
+  strict?: boolean;
+};
+
+const dedupeSchema = {
+  type: "object",
+  properties: {
+    keys: { type: "array", items: { type: "string" } },
+  },
+  required: ["keys"],
+  additionalProperties: false,
+} as const;
+
+const dedupeArrays = (data: Record<string, unknown>) => {
+  const result: Record<string, unknown> = { ...data };
+  for (const [key, value] of Object.entries(result)) {
+    if (Array.isArray(value)) {
+      const duplicates = findExactDuplicatesWithHashing(value);
+      result[key] = deduplicateByIndices(value, duplicates);
+    }
+  }
+  return result;
+};
+
+const removeByPath = (data: Record<string, unknown>, path: string) => {
+  const [root, indexStr] = path.split(".");
+  const index = Number(indexStr);
+  if (!root || Number.isNaN(index)) {
+    return data;
+  }
+
+  const value = data[root];
+  if (!Array.isArray(value)) {
+    return data;
+  }
+
+  const next = [...value];
+  next.splice(index, 1);
+  return { ...data, [root]: next };
+};
+
+export class SequentialAutoMergeStrategy<T> implements ExtractionStrategy<T> {
+  public name = "sequential-auto-merge";
+  private config: SequentialAutoMergeStrategyConfig;
+
+  constructor(config: SequentialAutoMergeStrategyConfig) {
+    this.config = config;
+  }
+
+  getEstimatedSteps(artifacts: ExtractionOptions<T>["artifacts"]): number {
+    const batches = getBatches(artifacts, {
+      maxTokens: this.config.chunkSize,
+      maxImages: this.config.maxImages,
+    });
+    return batches.length + 3;
+  }
+
+  async run(options: ExtractionOptions<T>): Promise<ExtractionResult<T>> {
+    const debug = options.debug;
+    const batches = getBatches(
+      options.artifacts,
+      {
+        maxTokens: this.config.chunkSize,
+        maxImages: this.config.maxImages,
+      },
+      debug,
+    );
+
+    const schema = serializeSchema(options.schema);
+    const merger = new SmartDataMerger(
+      options.schema as Record<string, unknown>,
+    );
+    let merged = {} as Record<string, unknown>;
+    const usages = [];
+    const totalSteps = this.getEstimatedSteps(options.artifacts);
+    let step = 1;
+
+    debug?.mergeStart({
+      mergeId: "sequential_auto_merge",
+      inputCount: batches.length,
+      strategy: this.name,
+    });
+
+    for (const [index, batch] of batches.entries()) {
+      const prompt = buildExtractorPrompt(
+        batch,
+        schema,
+        this.config.outputInstructions,
+      );
+      const result = await extractWithPrompt<T>({
+        model: this.config.model,
+        schema: options.schema,
+        system: prompt.system,
+        user: prompt.user,
+        artifacts: batch,
+        events: options.events,
+        execute: this.config.execute as never,
+        strict: options.strict ?? this.config.strict,
+        debug,
+        callId: `sequential_auto_batch_${index + 1}`,
+      });
+
+      merged = merger.merge(merged, result.data as Record<string, unknown>);
+      usages.push(result.usage);
+
+      // Log merge operation per field
+      for (const key of Object.keys(result.data as Record<string, unknown>)) {
+        const leftArray = Array.isArray(merged[key])
+          ? (merged[key] as unknown[]).length
+          : undefined;
+        const rightArray = Array.isArray(
+          (result.data as Record<string, unknown>)[key],
+        )
+          ? ((result.data as Record<string, unknown>)[key] as unknown[]).length
+          : undefined;
+
+        debug?.smartMergeField({
+          mergeId: "sequential_auto_merge",
+          field: key,
+          operation: "merge_arrays",
+          leftCount: leftArray,
+          rightCount: rightArray,
+        });
+      }
+
+      step += 1;
+      await options.events?.onStep?.({
+        step,
+        total: totalSteps,
+        label: `batch ${index + 1}/${batches.length}`,
+      });
+      debug?.step({
+        step,
+        total: totalSteps,
+        label: `batch ${index + 1}/${batches.length}`,
+        strategy: this.name,
+      });
+    }
+
+    debug?.mergeComplete({ mergeId: "sequential_auto_merge", success: true });
+
+    merged = dedupeArrays(merged);
+
+    const dedupePrompt = buildDeduplicationPrompt(schema, merged);
+
+    debug?.dedupeStart({
+      dedupeId: "sequential_auto_dedupe",
+      itemCount: Object.keys(merged).length,
+    });
+
+    const dedupeResponse = await runWithRetries<{ keys: string[] }>({
+      model: this.config.dedupeModel ?? this.config.model,
+      schema: dedupeSchema,
+      system: dedupePrompt.system,
+      user: dedupePrompt.user,
+      events: options.events,
+      execute: this.config.dedupeExecute,
+      strict: this.config.strict,
+      debug,
+      callId: "sequential_auto_dedupe",
+    });
+
+    step += 1;
+    await options.events?.onStep?.({
+      step,
+      total: totalSteps,
+      label: "dedupe",
+    });
+    debug?.step({
+      step,
+      total: totalSteps,
+      label: "dedupe",
+      strategy: this.name,
+    });
+
+    let deduped = merged;
+    for (const key of dedupeResponse.data.keys) {
+      deduped = removeByPath(deduped, key);
+    }
+
+    debug?.dedupeComplete({
+      dedupeId: "sequential_auto_dedupe",
+      duplicatesFound: dedupeResponse.data.keys.length,
+      itemsRemoved: dedupeResponse.data.keys.length,
+    });
+
+    return {
+      data: deduped as T,
+      usage: mergeUsage([...usages, dedupeResponse.usage]),
+    };
+  }
+}
+
+export const sequentialAutoMerge = <T>(
+  config: SequentialAutoMergeStrategyConfig,
+) => {
+  return new SequentialAutoMergeStrategy<T>(config);
+};

package/src/strategies/SequentialStrategy.test.ts

@@ -0,0 +1,53 @@
+import { test, expect } from "bun:test";
+import type { JSONSchemaType } from "ajv";
+import { SequentialStrategy } from "./SequentialStrategy";
+import type { Artifact, ExtractionOptions } from "../types";
+
+type Output = { title: string };
+
+const schema: JSONSchemaType<Output> = {
+  type: "object",
+  properties: { title: { type: "string" } },
+  required: ["title"],
+  additionalProperties: false,
+};
+
+const artifacts: Artifact[] = [
+  {
+    id: "a1",
+    type: "text",
+    raw: async () => Buffer.from(""),
+    contents: [{ text: "abcdefgh" }],
+  },
+  {
+    id: "a2",
+    type: "text",
+    raw: async () => Buffer.from(""),
+    contents: [{ text: "abcdefgh" }],
+  },
+];
+
+test("SequentialStrategy processes batches in order", async () => {
+  let calls = 0;
+  const strategy = new SequentialStrategy<Output>({
+    model: {},
+    chunkSize: 2,
+    execute: (async () => {
+      calls += 1;
+      return {
+        data: { title: `step-${calls}` },
+        usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+      };
+    }) as any,
+  });
+
+  const options: ExtractionOptions<Output> = {
+    artifacts,
+    schema,
+    strategy,
+  };
+
+  const result = await strategy.run(options);
+  expect(result.data.title).toBe("step-2");
+  expect(calls).toBe(2);
+});

package/src/strategies/SequentialStrategy.ts

@@ -0,0 +1,119 @@
+import type { ExtractionResult, ExtractionStrategy } from "../types";
+import type { ExtractionOptions } from "../types";
+import { buildSequentialPrompt } from "../prompts/SequentialExtractorPrompt";
+import {
+  extractWithPrompt,
+  getBatches,
+  mergeUsage,
+  serializeSchema,
+} from "./utils";
+import { runWithRetries } from "../llm/RetryingRunner";
+
+export type SequentialStrategyConfig = {
+  model: unknown;
+  chunkSize: number;
+  maxImages?: number;
+  outputInstructions?: string;
+  execute?: typeof runWithRetries;
+  strict?: boolean;
+};
+
+export class SequentialStrategy<T> implements ExtractionStrategy<T> {
+  public name = "sequential";
+  private config: SequentialStrategyConfig;
+
+  constructor(config: SequentialStrategyConfig) {
+    this.config = config;
+  }
+
+  getEstimatedSteps(artifacts: ExtractionOptions<T>["artifacts"]): number {
+    const batches = getBatches(artifacts, {
+      maxTokens: this.config.chunkSize,
+      maxImages: this.config.maxImages,
+    });
+    return batches.length + 2;
+  }
+
+  async run(options: ExtractionOptions<T>): Promise<ExtractionResult<T>> {
+    const debug = options.debug;
+    const batches = getBatches(
+      options.artifacts,
+      {
+        maxTokens: this.config.chunkSize,
+        maxImages: this.config.maxImages,
+      },
+      debug,
+    );
+
+    const schema = serializeSchema(options.schema);
+    let currentData: T | undefined;
+    const usages = [];
+    const totalSteps = this.getEstimatedSteps(options.artifacts);
+    let step = 1;
+
+    // Emit start event
+    await options.events?.onStep?.({
+      step,
+      total: totalSteps,
+      label: batches.length > 1 ? `batch 1/${batches.length}` : "extract",
+    });
+    debug?.step({
+      step,
+      total: totalSteps,
+      label: batches.length > 1 ? `batch 1/${batches.length}` : "extract",
+      strategy: this.name,
+    });
+
+    for (const [index, batch] of batches.entries()) {
+      const previousData = currentData ? JSON.stringify(currentData) : "{}";
+      const prompt = buildSequentialPrompt(
+        batch,
+        schema,
+        previousData,
+        this.config.outputInstructions,
+      );
+
+      const result = await extractWithPrompt<T>({
+        model: this.config.model,
+        schema: options.schema,
+        system: prompt.system,
+        user: prompt.user,
+        artifacts: batch,
+        events: options.events,
+        execute: this.config.execute as never,
+        strict: options.strict ?? this.config.strict,
+        debug,
+        callId: `sequential_batch_${index + 1}`,
+      });
+
+      currentData = result.data;
+      usages.push(result.usage);
+
+      step += 1;
+      // Only emit progress if there are more batches
+      if (index < batches.length - 1) {
+        await options.events?.onStep?.({
+          step,
+          total: totalSteps,
+          label: `batch ${index + 2}/${batches.length}`,
+        });
+        debug?.step({
+          step,
+          total: totalSteps,
+          label: `batch ${index + 2}/${batches.length}`,
+          strategy: this.name,
+        });
+      }
+    }
+
+    if (!currentData) {
+      throw new Error("No data extracted from sequential strategy");
+    }
+
+    return { data: currentData, usage: mergeUsage(usages) };
+  }
+}
+
+export const sequential = <T>(config: SequentialStrategyConfig) => {
+  return new SequentialStrategy<T>(config);
+};

package/src/strategies/SimpleStrategy.test.ts

@@ -0,0 +1,46 @@
+import { test, expect } from "bun:test";
+import type { JSONSchemaType } from "ajv";
+import { SimpleStrategy } from "./SimpleStrategy";
+import type { Artifact, ExtractionOptions } from "../types";
+
+type Output = { title: string };
+
+const schema: JSONSchemaType<Output> = {
+  type: "object",
+  properties: { title: { type: "string" } },
+  required: ["title"],
+  additionalProperties: false,
+};
+
+const artifacts: Artifact[] = [
+  {
+    id: "a1",
+    type: "text",
+    raw: async () => Buffer.from(""),
+    contents: [{ text: "hello" }],
+  },
+];
+
+test("SimpleStrategy runs once", async () => {
+  let calls = 0;
+  const strategy = new SimpleStrategy<Output>({
+    model: {},
+    execute: (async () => {
+      calls += 1;
+      return {
+        data: { title: "ok" },
+        usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+      };
+    }) as any,
+  });
+
+  const options: ExtractionOptions<Output> = {
+    artifacts,
+    schema,
+    strategy,
+  };
+
+  const result = await strategy.run(options);
+  expect(result.data.title).toBe("ok");
+  expect(calls).toBe(1);
+});

package/src/strategies/SimpleStrategy.ts

@@ -0,0 +1,74 @@
+import type { ExtractionResult, ExtractionStrategy } from "../types";
+import type { ExtractionOptions } from "../types";
+import { buildExtractorPrompt } from "../prompts/ExtractorPrompt";
+import { extractWithPrompt, serializeSchema } from "./utils";
+import { runWithRetries } from "../llm/RetryingRunner";
+
+export type SimpleStrategyConfig = {
+  model: unknown;
+  outputInstructions?: string;
+  execute?: typeof runWithRetries;
+  strict?: boolean;
+};
+
+export class SimpleStrategy<T> implements ExtractionStrategy<T> {
+  public name = "simple";
+  private config: SimpleStrategyConfig;
+
+  constructor(config: SimpleStrategyConfig) {
+    this.config = config;
+  }
+
+  getEstimatedSteps(): number {
+    return 3;
+  }
+
+  async run(options: ExtractionOptions<T>): Promise<ExtractionResult<T>> {
+    const debug = options.debug;
+    const schema = serializeSchema(options.schema);
+    const { system, user } = buildExtractorPrompt(
+      options.artifacts,
+      schema,
+      this.config.outputInstructions,
+    );
+
+    // Emit start event before extraction begins
+    await options.events?.onStep?.({
+      step: 1,
+      total: this.getEstimatedSteps(),
+      label: "extract",
+    });
+    debug?.step({
+      step: 1,
+      total: this.getEstimatedSteps(),
+      label: "extract",
+      strategy: this.name,
+    });
+
+    const result = await extractWithPrompt<T>({
+      model: this.config.model,
+      schema: options.schema,
+      system,
+      user,
+      artifacts: options.artifacts,
+      events: options.events,
+      execute: this.config.execute as never,
+      strict: options.strict ?? this.config.strict,
+      debug,
+      callId: "simple_extract",
+    });
+
+    debug?.step({
+      step: 2,
+      total: this.getEstimatedSteps(),
+      label: "complete",
+      strategy: this.name,
+    });
+
+    return { data: result.data, usage: result.usage };
+  }
+}
+
+export const simple = <T>(config: SimpleStrategyConfig) => {
+  return new SimpleStrategy<T>(config);
+};

package/src/strategies/concurrency.test.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "bun:test";
+import { runConcurrently } from "./concurrency";
+
+test("runConcurrently runs tasks in batches", async () => {
+  const started: number[] = [];
+  const tasks = [1, 2, 3, 4, 5].map((value) => async () => {
+    started.push(value);
+    await new Promise((resolve) => setTimeout(resolve, 5));
+    return value;
+  });
+
+  const results = await runConcurrently(tasks, 2);
+
+  expect(results).toEqual([1, 2, 3, 4, 5]);
+  expect(started).toEqual([1, 2, 3, 4, 5]);
+});

package/src/strategies/concurrency.ts

@@ -0,0 +1,14 @@
+export const runConcurrently = async <T>(
+  tasks: Array<() => Promise<T>>,
+  concurrency: number
+): Promise<T[]> => {
+  const results: T[] = [];
+
+  for (let i = 0; i < tasks.length; i += concurrency) {
+    const chunk = tasks.slice(i, i + concurrency).map((task) => task());
+    const chunkResults = await Promise.all(chunk);
+    results.push(...chunkResults);
+  }
+
+  return results;
+};

package/src/strategies/index.test.ts

@@ -0,0 +1,20 @@
+import { test, expect } from "bun:test";
+import * as strategies from "./index";
+
+test("strategies index re-exports constructors and helpers", () => {
+  expect(typeof strategies.SimpleStrategy).toBe("function");
+  expect(typeof strategies.ParallelStrategy).toBe("function");
+  expect(typeof strategies.SequentialStrategy).toBe("function");
+  expect(typeof strategies.ParallelAutoMergeStrategy).toBe("function");
+  expect(typeof strategies.SequentialAutoMergeStrategy).toBe("function");
+  expect(typeof strategies.DoublePassStrategy).toBe("function");
+  expect(typeof strategies.DoublePassAutoMergeStrategy).toBe("function");
+
+  expect(typeof strategies.simple).toBe("function");
+  expect(typeof strategies.parallel).toBe("function");
+  expect(typeof strategies.sequential).toBe("function");
+  expect(typeof strategies.parallelAutoMerge).toBe("function");
+  expect(typeof strategies.sequentialAutoMerge).toBe("function");
+  expect(typeof strategies.doublePass).toBe("function");
+  expect(typeof strategies.doublePassAutoMerge).toBe("function");
+});

package/src/strategies/index.ts

@@ -0,0 +1,7 @@
+export { SimpleStrategy, simple } from "./SimpleStrategy";
+export { ParallelStrategy, parallel } from "./ParallelStrategy";
+export { SequentialStrategy, sequential } from "./SequentialStrategy";
+export { ParallelAutoMergeStrategy, parallelAutoMerge } from "./ParallelAutoMergeStrategy";
+export { SequentialAutoMergeStrategy, sequentialAutoMerge } from "./SequentialAutoMergeStrategy";
+export { DoublePassStrategy, doublePass } from "./DoublePassStrategy";
+export { DoublePassAutoMergeStrategy, doublePassAutoMerge } from "./DoublePassAutoMergeStrategy";

package/src/strategies/utils.test.ts

@@ -0,0 +1,76 @@
+import { test, expect } from "bun:test";
+import type { Artifact } from "../types";
+import type { JSONSchemaType } from "ajv";
+import { batchArtifacts } from "../chunking/ArtifactBatcher";
+import { serializeSchema, mergeUsage, getBatches, extractWithPrompt } from "./utils";
+
+type Output = { title: string };
+
+const schema: JSONSchemaType<Output> = {
+  type: "object",
+  properties: { title: { type: "string" } },
+  required: ["title"],
+  additionalProperties: false,
+};
+
+const makeArtifact = (id: string, text: string): Artifact => ({
+  id,
+  type: "text",
+  raw: async () => Buffer.from(text),
+  contents: [{ text }],
+});
+
+test("serializeSchema returns JSON", () => {
+  expect(serializeSchema({ ok: true })).toBe("{\"ok\":true}");
+});
+
+test("mergeUsage sums token usage", () => {
+  const usage = mergeUsage([
+    { inputTokens: 1, outputTokens: 2, totalTokens: 3 },
+    { inputTokens: 4, outputTokens: 5, totalTokens: 9 },
+  ]);
+
+  expect(usage).toEqual({ inputTokens: 5, outputTokens: 7, totalTokens: 12 });
+});
+
+test("getBatches delegates to batchArtifacts", () => {
+  const artifacts = [makeArtifact("a1", "hello"), makeArtifact("a2", "world")];
+  const options = { maxTokens: 1 };
+
+  expect(getBatches(artifacts, options)).toEqual(batchArtifacts(artifacts, options));
+});
+
+test("extractWithPrompt builds user content and returns result", async () => {
+  const artifacts: Artifact[] = [
+    {
+      id: "a1",
+      type: "image",
+      raw: async () => Buffer.from(""),
+      contents: [
+        {
+          text: "hello",
+          media: [{ type: "image", base64: "abc" }],
+        },
+      ],
+    },
+  ];
+
+  let receivedUser: unknown;
+  const result = await extractWithPrompt<Output>({
+    model: {},
+    schema,
+    system: "sys",
+    user: "prompt",
+    artifacts,
+    execute: async (request) => {
+      receivedUser = request.user;
+      return {
+        data: { title: "ok" },
+        usage: { inputTokens: 1, outputTokens: 2, totalTokens: 3 },
+      };
+    },
+  });
+
+  expect(Array.isArray(receivedUser)).toBe(true);
+  expect(result.data.title).toBe("ok");
+});