@struktur/sdk 0.1.0

package/src/strategies/DoublePassAutoMergeStrategy.test.ts

@@ -0,0 +1,53 @@
+import { test, expect } from "bun:test";
+import type { JSONSchemaType } from "ajv";
+import { DoublePassAutoMergeStrategy } from "./DoublePassAutoMergeStrategy";
+import type { Artifact, ExtractionOptions } from "../types";
+
+type Output = { title: string };
+
+const schema: JSONSchemaType<Output> = {
+  type: "object",
+  properties: { title: { type: "string" } },
+  required: ["title"],
+  additionalProperties: false,
+};
+
+const artifacts: Artifact[] = [
+  {
+    id: "a1",
+    type: "text",
+    raw: async () => Buffer.from(""),
+    contents: [{ text: "abcdefgh" }],
+  },
+];
+
+test("DoublePassAutoMergeStrategy runs both passes", async () => {
+  let calls = 0;
+  const strategy = new DoublePassAutoMergeStrategy<Output>({
+    model: {},
+    chunkSize: 10,
+    execute: (async () => {
+      calls += 1;
+      return {
+        data: { title: `pass-${calls}` },
+        usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+      };
+    }) as any,
+    dedupeExecute: (async () => {
+      return {
+        data: { keys: [] },
+        usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+      };
+    }) as any,
+  });
+
+  const options: ExtractionOptions<Output> = {
+    artifacts,
+    schema,
+    strategy,
+  };
+
+  const result = await strategy.run(options);
+  expect(result.data.title).toBe("pass-2");
+  expect(calls).toBe(2);
+});

package/src/strategies/DoublePassAutoMergeStrategy.ts

@@ -0,0 +1,270 @@
+import type { ExtractionResult, ExtractionStrategy } from "../types";
+import type { ExtractionOptions } from "../types";
+import { buildExtractorPrompt } from "../prompts/ExtractorPrompt";
+import { buildDeduplicationPrompt } from "../prompts/DeduplicationPrompt";
+import { buildSequentialPrompt } from "../prompts/SequentialExtractorPrompt";
+import {
+  extractWithPrompt,
+  getBatches,
+  mergeUsage,
+  serializeSchema,
+} from "./utils";
+import { SmartDataMerger } from "../merge/SmartDataMerger";
+import {
+  findExactDuplicatesWithHashing,
+  deduplicateByIndices,
+} from "../merge/Deduplicator";
+import { runConcurrently } from "./concurrency";
+import { runWithRetries } from "../llm/RetryingRunner";
+
+export type DoublePassAutoMergeStrategyConfig = {
+  model: unknown;
+  chunkSize: number;
+  concurrency?: number;
+  maxImages?: number;
+  outputInstructions?: string;
+  dedupeModel?: unknown;
+  execute?: typeof runWithRetries;
+  dedupeExecute?: typeof runWithRetries;
+  strict?: boolean;
+};
+
+const dedupeSchema = {
+  type: "object",
+  properties: {
+    keys: { type: "array", items: { type: "string" } },
+  },
+  required: ["keys"],
+  additionalProperties: false,
+} as const;
+
+const dedupeArrays = (data: Record<string, unknown>) => {
+  const result: Record<string, unknown> = { ...data };
+  for (const [key, value] of Object.entries(result)) {
+    if (Array.isArray(value)) {
+      const duplicates = findExactDuplicatesWithHashing(value);
+      result[key] = deduplicateByIndices(value, duplicates);
+    }
+  }
+  return result;
+};
+
+const removeByPath = (data: Record<string, unknown>, path: string) => {
+  const [root, indexStr] = path.split(".");
+  const index = Number(indexStr);
+  if (!root || Number.isNaN(index)) {
+    return data;
+  }
+
+  const value = data[root];
+  if (!Array.isArray(value)) {
+    return data;
+  }
+
+  const next = [...value];
+  next.splice(index, 1);
+  return { ...data, [root]: next };
+};
+
+export class DoublePassAutoMergeStrategy<T> implements ExtractionStrategy<T> {
+  public name = "double-pass-auto-merge";
+  private config: DoublePassAutoMergeStrategyConfig;
+
+  constructor(config: DoublePassAutoMergeStrategyConfig) {
+    this.config = config;
+  }
+
+  getEstimatedSteps(artifacts: ExtractionOptions<T>["artifacts"]): number {
+    const batches = getBatches(artifacts, {
+      maxTokens: this.config.chunkSize,
+      maxImages: this.config.maxImages,
+    });
+    return batches.length * 2 + 3;
+  }
+
+  async run(options: ExtractionOptions<T>): Promise<ExtractionResult<T>> {
+    const debug = options.debug;
+    const batches = getBatches(
+      options.artifacts,
+      {
+        maxTokens: this.config.chunkSize,
+        maxImages: this.config.maxImages,
+      },
+      debug,
+    );
+
+    const schema = serializeSchema(options.schema);
+    const totalSteps = this.getEstimatedSteps(options.artifacts);
+    let step = 1;
+
+    const tasks = batches.map((batch, index) => async () => {
+      const prompt = buildExtractorPrompt(
+        batch,
+        schema,
+        this.config.outputInstructions,
+      );
+      const result = await extractWithPrompt<T>({
+        model: this.config.model,
+        schema: options.schema,
+        system: prompt.system,
+        user: prompt.user,
+        artifacts: batch,
+        events: options.events,
+        execute: this.config.execute as never,
+        strict: options.strict ?? this.config.strict,
+        debug,
+        callId: `double_pass_auto_1_batch_${index + 1}`,
+      });
+      step += 1;
+      await options.events?.onStep?.({
+        step,
+        total: totalSteps,
+        label: `pass 1 batch ${index + 1}/${batches.length}`,
+      });
+      debug?.step({
+        step,
+        total: totalSteps,
+        label: `pass 1 batch ${index + 1}/${batches.length}`,
+        strategy: this.name,
+      });
+      return result;
+    });
+
+    const results = await runConcurrently(
+      tasks,
+      this.config.concurrency ?? batches.length,
+    );
+
+    const merger = new SmartDataMerger(
+      options.schema as Record<string, unknown>,
+    );
+    let merged = {} as Record<string, unknown>;
+
+    debug?.mergeStart({
+      mergeId: "double_pass_auto_merge",
+      inputCount: results.length,
+      strategy: this.name,
+    });
+
+    for (let i = 0; i < results.length; i++) {
+      const result = results[i]!;
+      merged = merger.merge(merged, result.data as Record<string, unknown>);
+
+      // Log merge operation per field
+      for (const key of Object.keys(result.data as Record<string, unknown>)) {
+        const leftArray = Array.isArray(merged[key])
+          ? (merged[key] as unknown[]).length
+          : undefined;
+        const rightArray = Array.isArray(
+          (result.data as Record<string, unknown>)[key],
+        )
+          ? ((result.data as Record<string, unknown>)[key] as unknown[]).length
+          : undefined;
+
+        debug?.smartMergeField({
+          mergeId: "double_pass_auto_merge",
+          field: key,
+          operation: "merge_arrays",
+          leftCount: leftArray,
+          rightCount: rightArray,
+        });
+      }
+    }
+
+    debug?.mergeComplete({ mergeId: "double_pass_auto_merge", success: true });
+
+    merged = dedupeArrays(merged);
+
+    const dedupePrompt = buildDeduplicationPrompt(schema, merged);
+
+    debug?.dedupeStart({
+      dedupeId: "double_pass_auto_dedupe",
+      itemCount: Object.keys(merged).length,
+    });
+
+    const dedupeResponse = await runWithRetries<{ keys: string[] }>({
+      model: this.config.dedupeModel ?? this.config.model,
+      schema: dedupeSchema,
+      system: dedupePrompt.system,
+      user: dedupePrompt.user,
+      events: options.events,
+      execute: this.config.dedupeExecute,
+      strict: this.config.strict,
+      debug,
+      callId: "double_pass_auto_dedupe",
+    });
+
+    step += 1;
+    await options.events?.onStep?.({
+      step,
+      total: totalSteps,
+      label: "pass 1 dedupe",
+    });
+    debug?.step({
+      step,
+      total: totalSteps,
+      label: "pass 1 dedupe",
+      strategy: this.name,
+    });
+
+    let deduped = merged;
+    for (const key of dedupeResponse.data.keys) {
+      deduped = removeByPath(deduped, key);
+    }
+
+    debug?.dedupeComplete({
+      dedupeId: "double_pass_auto_dedupe",
+      duplicatesFound: dedupeResponse.data.keys.length,
+      itemsRemoved: dedupeResponse.data.keys.length,
+    });
+
+    let currentData = deduped as T;
+    const usages = [...results.map((r) => r.usage), dedupeResponse.usage];
+
+    for (const [index, batch] of batches.entries()) {
+      const prompt = buildSequentialPrompt(
+        batch,
+        schema,
+        JSON.stringify(currentData),
+        this.config.outputInstructions,
+      );
+
+      const result = await extractWithPrompt<T>({
+        model: this.config.model,
+        schema: options.schema,
+        system: prompt.system,
+        user: prompt.user,
+        artifacts: batch,
+        events: options.events,
+        execute: this.config.execute as never,
+        strict: this.config.strict,
+        debug,
+        callId: `double_pass_auto_2_batch_${index + 1}`,
+      });
+
+      currentData = result.data;
+      usages.push(result.usage);
+
+      step += 1;
+      await options.events?.onStep?.({
+        step,
+        total: totalSteps,
+        label: `pass 2 batch ${index + 1}/${batches.length}`,
+      });
+      debug?.step({
+        step,
+        total: totalSteps,
+        label: `pass 2 batch ${index + 1}/${batches.length}`,
+        strategy: this.name,
+      });
+    }
+
+    return { data: currentData, usage: mergeUsage(usages) };
+  }
+}
+
+export const doublePassAutoMerge = <T>(
+  config: DoublePassAutoMergeStrategyConfig,
+) => {
+  return new DoublePassAutoMergeStrategy<T>(config);
+};

package/src/strategies/DoublePassStrategy.test.ts

@@ -0,0 +1,48 @@
+import { test, expect } from "bun:test";
+import type { JSONSchemaType } from "ajv";
+import { DoublePassStrategy } from "./DoublePassStrategy";
+import type { Artifact, ExtractionOptions } from "../types";
+
+type Output = { title: string };
+
+const schema: JSONSchemaType<Output> = {
+  type: "object",
+  properties: { title: { type: "string" } },
+  required: ["title"],
+  additionalProperties: false,
+};
+
+const artifacts: Artifact[] = [
+  {
+    id: "a1",
+    type: "text",
+    raw: async () => Buffer.from(""),
+    contents: [{ text: "abcdefgh" }],
+  },
+];
+
+test("DoublePassStrategy runs second pass", async () => {
+  let calls = 0;
+  const strategy = new DoublePassStrategy<Output>({
+    model: {},
+    mergeModel: {},
+    chunkSize: 10,
+    execute: (async () => {
+      calls += 1;
+      return {
+        data: { title: `pass-${calls}` },
+        usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+      };
+    }) as any,
+  });
+
+  const options: ExtractionOptions<Output> = {
+    artifacts,
+    schema,
+    strategy,
+  };
+
+  const result = await strategy.run(options);
+  expect(result.data.title).toBe("pass-3");
+  expect(calls).toBe(3);
+});

package/src/strategies/DoublePassStrategy.ts

@@ -0,0 +1,179 @@
+import type { ExtractionResult, ExtractionStrategy } from "../types";
+import type { ExtractionOptions } from "../types";
+import { buildExtractorPrompt } from "../prompts/ExtractorPrompt";
+import { buildParallelMergerPrompt } from "../prompts/ParallelMergerPrompt";
+import { buildSequentialPrompt } from "../prompts/SequentialExtractorPrompt";
+import {
+  extractWithPrompt,
+  getBatches,
+  mergeUsage,
+  serializeSchema,
+} from "./utils";
+import { runConcurrently } from "./concurrency";
+import { runWithRetries } from "../llm/RetryingRunner";
+
+export type DoublePassStrategyConfig = {
+  model: unknown;
+  mergeModel: unknown;
+  chunkSize: number;
+  concurrency?: number;
+  maxImages?: number;
+  outputInstructions?: string;
+  execute?: typeof runWithRetries;
+  strict?: boolean;
+};
+
+export class DoublePassStrategy<T> implements ExtractionStrategy<T> {
+  public name = "double-pass";
+  private config: DoublePassStrategyConfig;
+
+  constructor(config: DoublePassStrategyConfig) {
+    this.config = config;
+  }
+
+  getEstimatedSteps(artifacts: ExtractionOptions<T>["artifacts"]): number {
+    const batches = getBatches(artifacts, {
+      maxTokens: this.config.chunkSize,
+      maxImages: this.config.maxImages,
+    });
+    return batches.length * 2 + 3;
+  }
+
+  async run(options: ExtractionOptions<T>): Promise<ExtractionResult<T>> {
+    const debug = options.debug;
+    const batches = getBatches(
+      options.artifacts,
+      {
+        maxTokens: this.config.chunkSize,
+        maxImages: this.config.maxImages,
+      },
+      debug,
+    );
+
+    const schema = serializeSchema(options.schema);
+    const totalSteps = this.getEstimatedSteps(options.artifacts);
+    let step = 1;
+
+    const tasks = batches.map((batch, index) => async () => {
+      const prompt = buildExtractorPrompt(
+        batch,
+        schema,
+        this.config.outputInstructions,
+      );
+      const result = await extractWithPrompt<T>({
+        model: this.config.model,
+        schema: options.schema,
+        system: prompt.system,
+        user: prompt.user,
+        artifacts: batch,
+        events: options.events,
+        execute: this.config.execute as never,
+        strict: options.strict ?? this.config.strict,
+        debug,
+        callId: `double_pass_1_batch_${index + 1}`,
+      });
+      step += 1;
+      await options.events?.onStep?.({
+        step,
+        total: totalSteps,
+        label: `pass 1 batch ${index + 1}/${batches.length}`,
+      });
+      debug?.step({
+        step,
+        total: totalSteps,
+        label: `pass 1 batch ${index + 1}/${batches.length}`,
+        strategy: this.name,
+      });
+      return result;
+    });
+
+    const results = await runConcurrently(
+      tasks,
+      this.config.concurrency ?? batches.length,
+    );
+
+    debug?.mergeStart({
+      mergeId: "double_pass_1_merge",
+      inputCount: results.length,
+      strategy: this.name,
+    });
+
+    const mergePrompt = buildParallelMergerPrompt(
+      schema,
+      results.map((r) => r.data),
+    );
+    const merged = await extractWithPrompt<T>({
+      model: this.config.mergeModel,
+      schema: options.schema,
+      system: mergePrompt.system,
+      user: mergePrompt.user,
+      artifacts: [],
+      events: options.events,
+      execute: this.config.execute as never,
+      strict: this.config.strict,
+      debug,
+      callId: "double_pass_1_merge",
+    });
+
+    step += 1;
+    await options.events?.onStep?.({
+      step,
+      total: totalSteps,
+      label: "pass 1 merge",
+    });
+    debug?.step({
+      step,
+      total: totalSteps,
+      label: "pass 1 merge",
+      strategy: this.name,
+    });
+    debug?.mergeComplete({ mergeId: "double_pass_1_merge", success: true });
+
+    let currentData = merged.data;
+    const usages = [...results.map((r) => r.usage), merged.usage];
+
+    for (const [index, batch] of batches.entries()) {
+      const prompt = buildSequentialPrompt(
+        batch,
+        schema,
+        JSON.stringify(currentData),
+        this.config.outputInstructions,
+      );
+
+      const result = await extractWithPrompt<T>({
+        model: this.config.model,
+        schema: options.schema,
+        system: prompt.system,
+        user: prompt.user,
+        artifacts: batch,
+        events: options.events,
+        execute: this.config.execute as never,
+        strict: this.config.strict,
+        debug,
+        callId: `double_pass_2_batch_${index + 1}`,
+      });
+
+      currentData = result.data;
+      usages.push(result.usage);
+
+      step += 1;
+      await options.events?.onStep?.({
+        step,
+        total: totalSteps,
+        label: `pass 2 batch ${index + 1}/${batches.length}`,
+      });
+      debug?.step({
+        step,
+        total: totalSteps,
+        label: `pass 2 batch ${index + 1}/${batches.length}`,
+        strategy: this.name,
+      });
+    }
+
+    return { data: currentData, usage: mergeUsage(usages) };
+  }
+}
+
+export const doublePass = <T>(config: DoublePassStrategyConfig) => {
+  return new DoublePassStrategy<T>(config);
+};

package/src/strategies/ParallelAutoMergeStrategy.test.ts

@@ -0,0 +1,152 @@
+import { test, expect } from "bun:test";
+import type { JSONSchemaType } from "ajv";
+import { ParallelAutoMergeStrategy, __testing__ } from "./ParallelAutoMergeStrategy";
+import type { Artifact, ExtractionOptions } from "../types";
+
+type Output = { items: Array<{ id: number }> };
+
+const schema: JSONSchemaType<Output> = {
+  type: "object",
+  properties: {
+    items: {
+      type: "array",
+      items: {
+        type: "object",
+        properties: { id: { type: "number" } },
+        required: ["id"],
+        additionalProperties: false,
+      },
+    },
+  },
+  required: ["items"],
+  additionalProperties: false,
+};
+
+const artifacts: Artifact[] = [
+  {
+    id: "a1",
+    type: "text",
+    raw: async () => Buffer.from(""),
+    contents: [{ text: "abcdefgh" }],
+  },
+  {
+    id: "a2",
+    type: "text",
+    raw: async () => Buffer.from(""),
+    contents: [{ text: "abcdefgh" }],
+  },
+];
+
+test("ParallelAutoMergeStrategy deduplicates arrays", async () => {
+  const strategy = new ParallelAutoMergeStrategy<Output>({
+    model: {},
+    chunkSize: 2,
+    execute: (async () => {
+      return {
+        data: { items: [{ id: 1 }, { id: 1 }] },
+        usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+      };
+    }) as any,
+    dedupeExecute: (async () => {
+      return {
+        data: { keys: [] },
+        usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+      };
+    }) as any,
+  });
+
+  const options: ExtractionOptions<Output> = {
+    artifacts,
+    schema,
+    strategy,
+  };
+
+  const result = await strategy.run(options);
+  expect(result.data.items.length).toBe(1);
+});
+
+test("dedupeArrays removes duplicates from all array fields", () => {
+  const data = {
+    items: [{ id: 1 }, { id: 1 }, { id: 2 }],
+    names: ["a", "a", "b"],
+    count: 5,
+  };
+
+  const result = __testing__.dedupeArrays(data);
+
+  expect(result.items).toEqual([{ id: 1 }, { id: 2 }]);
+  expect(result.names).toEqual(["a", "b"]);
+  expect(result.count).toBe(5);
+});
+
+test("dedupeArrays handles non-array fields", () => {
+  const data = {
+    title: "test",
+    count: 42,
+  };
+
+  const result = __testing__.dedupeArrays(data);
+
+  expect(result).toEqual({ title: "test", count: 42 });
+});
+
+test("removeByPath removes item at path", () => {
+  const data = {
+    items: [{ id: 1 }, { id: 2 }, { id: 3 }],
+  };
+
+  const result = __testing__.removeByPath(data, "items.1");
+
+  expect(result.items).toEqual([{ id: 1 }, { id: 3 }]);
+});
+
+test("removeByPath handles first item", () => {
+  const data = {
+    items: [{ id: 1 }, { id: 2 }],
+  };
+
+  const result = __testing__.removeByPath(data, "items.0");
+
+  expect(result.items).toEqual([{ id: 2 }]);
+});
+
+test("removeByPath handles last item", () => {
+  const data = {
+    items: [{ id: 1 }, { id: 2 }],
+  };
+
+  const result = __testing__.removeByPath(data, "items.1");
+
+  expect(result.items).toEqual([{ id: 1 }]);
+});
+
+test("removeByPath returns unchanged data for invalid path", () => {
+  const data = {
+    items: [{ id: 1 }],
+  };
+
+  expect(__testing__.removeByPath(data, "")).toEqual(data);
+  expect(__testing__.removeByPath(data, "items")).toEqual(data);
+  expect(__testing__.removeByPath(data, "items.abc")).toEqual(data);
+  expect(__testing__.removeByPath(data, "missing.0")).toEqual(data);
+});
+
+test("removeByPath returns unchanged data for non-array field", () => {
+  const data = {
+    title: "test",
+  };
+
+  const result = __testing__.removeByPath(data, "title.0");
+
+  expect(result).toEqual(data);
+});
+
+test("removeByPath does not mutate original data", () => {
+  const data = {
+    items: [{ id: 1 }, { id: 2 }],
+  };
+
+  __testing__.removeByPath(data, "items.0");
+
+  expect(data.items).toEqual([{ id: 1 }, { id: 2 }]);
+});