@struktur/sdk 2.1.2 → 2.3.0

package/src/strategies/ParallelAutoMergeStrategy.ts

@@ -1,345 +0,0 @@
-import type { ExtractionResult, ExtractionStrategy } from "../types";
-import type { ExtractionOptions } from "../types";
-import { buildExtractorPrompt } from "../prompts/ExtractorPrompt";
-import { buildDeduplicationPrompt } from "../prompts/DeduplicationPrompt";
-import {
-  extractWithPrompt,
-  getBatches,
-  mergeUsage,
-  serializeSchema,
-} from "./utils";
-import { runConcurrently } from "./concurrency";
-import { SmartDataMerger } from "../merge/SmartDataMerger";
-import {
-  findExactDuplicatesWithHashing,
-  deduplicateByIndices,
-} from "../merge/Deduplicator";
-import { runWithRetries } from "../llm/RetryingRunner";
-
-export type ParallelAutoMergeStrategyConfig = {
-  model: unknown;
-  chunkSize: number;
-  concurrency?: number;
-  maxImages?: number;
-  outputInstructions?: string;
-  dedupeModel?: unknown;
-  execute?: typeof runWithRetries;
-  dedupeExecute?: typeof runWithRetries;
-  strict?: boolean;
-};
-
-const dedupeSchema = {
-  type: "object",
-  properties: {
-    keys: { type: "array", items: { type: "string" } },
-  },
-  required: ["keys"],
-  additionalProperties: false,
-} as const;
-
-const dedupeArrays = (data: Record<string, unknown>) => {
-  const result: Record<string, unknown> = { ...data };
-  for (const [key, value] of Object.entries(result)) {
-    if (Array.isArray(value)) {
-      const duplicates = findExactDuplicatesWithHashing(value);
-      result[key] = deduplicateByIndices(value, duplicates);
-    }
-  }
-  return result;
-};
-
-const removeByPath = (data: Record<string, unknown>, path: string) => {
-  const [root, indexStr] = path.split(".");
-  const index = Number(indexStr);
-  if (!root || Number.isNaN(index)) {
-    return data;
-  }
-
-  const value = data[root];
-  if (!Array.isArray(value)) {
-    return data;
-  }
-
-  const next = [...value];
-  next.splice(index, 1);
-  return { ...data, [root]: next };
-};
-
-export class ParallelAutoMergeStrategy<T> implements ExtractionStrategy<T> {
-  public name = "parallel-auto-merge";
-  private config: ParallelAutoMergeStrategyConfig;
-
-  constructor(config: ParallelAutoMergeStrategyConfig) {
-    this.config = config;
-  }
-
-  getEstimatedSteps(artifacts: ExtractionOptions<T>["artifacts"]): number {
-    const batches = getBatches(artifacts, {
-      maxTokens: this.config.chunkSize,
-      maxImages: this.config.maxImages,
-    });
-    return batches.length + 3;
-  }
-
-  async run(options: ExtractionOptions<T>): Promise<ExtractionResult<T>> {
-    const debug = options.debug;
-    const { telemetry } = options;
-    
-    // Create strategy-level span
-    const strategySpan = telemetry?.startSpan({
-      name: "strategy.parallel-auto-merge",
-      kind: "CHAIN",
-      attributes: {
-        "strategy.name": this.name,
-        "strategy.artifacts.count": options.artifacts.length,
-        "strategy.chunk_size": this.config.chunkSize,
-        "strategy.concurrency": this.config.concurrency,
-      },
-    });
-    
-    const batches = getBatches(
-      options.artifacts,
-      {
-        maxTokens: this.config.chunkSize,
-        maxImages: this.config.maxImages,
-      },
-      debug,
-      telemetry ?? undefined,
-      strategySpan,
-    );
-
-    const schema = serializeSchema(options.schema);
-    const totalSteps = this.getEstimatedSteps(options.artifacts);
-    let step = 1;
-
-    const tasks = batches.map((batch, index) => async () => {
-      const prompt = buildExtractorPrompt(
-        batch,
-        schema,
-        this.config.outputInstructions,
-      );
-      const result = await extractWithPrompt<T>({
-        model: this.config.model,
-        schema: options.schema,
-        system: prompt.system,
-        user: prompt.user,
-        artifacts: batch,
-        events: options.events,
-        execute: this.config.execute as never,
-        strict: options.strict ?? this.config.strict,
-        debug,
-        callId: `parallel_auto_batch_${index + 1}`,
-        telemetry: telemetry ?? undefined,
-        parentSpan: strategySpan,
-      });
-      step += 1;
-      await options.events?.onStep?.({
-        step,
-        total: totalSteps,
-        label: `batch ${index + 1}/${batches.length}`,
-      });
-      debug?.step({
-        step,
-        total: totalSteps,
-        label: `batch ${index + 1}/${batches.length}`,
-        strategy: this.name,
-      });
-      return result;
-    });
-
-    const results = await runConcurrently(
-      tasks,
-      this.config.concurrency ?? batches.length,
-    );
-
-    const merger = new SmartDataMerger(
-      options.schema as Record<string, unknown>,
-    );
-    let merged = {} as Record<string, unknown>;
-
-    debug?.mergeStart({
-      mergeId: "parallel_auto_smart_merge",
-      inputCount: results.length,
-      strategy: this.name,
-    });
-    
-    // Create smart merge span
-    const mergeSpan = telemetry?.startSpan({
-      name: "struktur.smart_merge",
-      kind: "CHAIN",
-      parentSpan: strategySpan,
-      attributes: {
-        "merge.strategy": "smart",
-        "merge.input_count": results.length,
-      },
-    });
-
-    for (let i = 0; i < results.length; i++) {
-      const result = results[i]!;
-      const prevSize = Object.keys(merged).length;
-      merged = merger.merge(merged, result.data as Record<string, unknown>);
-      const newSize = Object.keys(merged).length;
-
-      // Log merge operation per field
-      for (const key of Object.keys(result.data as Record<string, unknown>)) {
-        const leftArray = Array.isArray(merged[key])
-          ? (merged[key] as unknown[]).length
-          : undefined;
-        const rightArray = Array.isArray(
-          (result.data as Record<string, unknown>)[key],
-        )
-          ? ((result.data as Record<string, unknown>)[key] as unknown[]).length
-          : undefined;
-
-        debug?.smartMergeField({
-          mergeId: "parallel_auto_smart_merge",
-          field: key,
-          operation: "merge_arrays",
-          leftCount: leftArray,
-          rightCount: rightArray,
-        });
-        
-        // Record merge event in telemetry
-        if (mergeSpan && telemetry) {
-          telemetry.recordEvent(mergeSpan, {
-            type: "merge",
-            strategy: "smart",
-            inputCount: rightArray ?? 1,
-            outputCount: leftArray ?? 1,
-          });
-        }
-      }
-    }
-
-    debug?.mergeComplete({
-      mergeId: "parallel_auto_smart_merge",
-      success: true,
-    });
-    
-    // End merge span
-    if (mergeSpan && telemetry) {
-      telemetry.endSpan(mergeSpan, {
-        status: "ok",
-        output: merged,
-      });
-    }
-
-    merged = dedupeArrays(merged);
-    
-    // Create exact dedupe span
-    const exactDedupeSpan = telemetry?.startSpan({
-      name: "struktur.exact_dedupe",
-      kind: "CHAIN",
-      parentSpan: strategySpan,
-      attributes: {
-        "dedupe.method": "exact_hashing",
-      },
-    });
-    
-    // End exact dedupe span
-    if (exactDedupeSpan && telemetry) {
-      telemetry.recordEvent(exactDedupeSpan, {
-        type: "merge",
-        strategy: "exact_hash_dedupe",
-        inputCount: Object.keys(merged).length,
-        outputCount: Object.keys(merged).length,
-      });
-      telemetry.endSpan(exactDedupeSpan, {
-        status: "ok",
-        output: merged,
-      });
-    }
-
-    const dedupePrompt = buildDeduplicationPrompt(schema, merged);
-
-    debug?.dedupeStart({
-      dedupeId: "parallel_auto_dedupe",
-      itemCount: Object.keys(merged).length,
-    });
-    
-    // Create LLM dedupe span
-    const llmDedupeSpan = telemetry?.startSpan({
-      name: "struktur.llm_dedupe",
-      kind: "CHAIN",
-      parentSpan: strategySpan,
-      attributes: {
-        "dedupe.method": "llm",
-      },
-    });
-
-    const dedupeResponse = await runWithRetries<{ keys: string[] }>({
-      model: this.config.dedupeModel ?? this.config.model,
-      schema: dedupeSchema,
-      system: dedupePrompt.system,
-      user: dedupePrompt.user,
-      events: options.events,
-      execute: this.config.dedupeExecute,
-      strict: this.config.strict,
-      debug,
-      callId: "parallel_auto_dedupe",
-      telemetry: telemetry ?? undefined,
-      parentSpan: llmDedupeSpan,
-    });
-
-    step += 1;
-    await options.events?.onStep?.({
-      step,
-      total: totalSteps,
-      label: "dedupe",
-    });
-    debug?.step({
-      step,
-      total: totalSteps,
-      label: "dedupe",
-      strategy: this.name,
-    });
-
-    let deduped = merged;
-    for (const key of dedupeResponse.data.keys) {
-      deduped = removeByPath(deduped, key);
-    }
-
-    debug?.dedupeComplete({
-      dedupeId: "parallel_auto_dedupe",
-      duplicatesFound: dedupeResponse.data.keys.length,
-      itemsRemoved: dedupeResponse.data.keys.length,
-    });
-    
-    // End LLM dedupe span
-    if (llmDedupeSpan && telemetry) {
-      telemetry.recordEvent(llmDedupeSpan, {
-        type: "merge",
-        strategy: "llm_dedupe",
-        inputCount: Object.keys(merged).length,
-        outputCount: Object.keys(deduped).length,
-        deduped: dedupeResponse.data.keys.length,
-      });
-      telemetry.endSpan(llmDedupeSpan, {
-        status: "ok",
-        output: deduped,
-      });
-    }
-
-    // End strategy span
-    telemetry?.endSpan(strategySpan!, {
-      status: "ok",
-      output: deduped,
-    });
-
-    return {
-      data: deduped as T,
-      usage: mergeUsage([...results.map((r) => r.usage), dedupeResponse.usage]),
-    };
-  }
-}
-
-export const parallelAutoMerge = <T>(
-  config: ParallelAutoMergeStrategyConfig,
-) => {
-  return new ParallelAutoMergeStrategy<T>(config);
-};
-
-export const __testing__ = {
-  dedupeArrays,
-  removeByPath,
-};

package/src/strategies/ParallelStrategy.test.ts

@@ -1,61 +0,0 @@
-import { test, expect } from "bun:test";
-import type { JSONSchemaType } from "ajv";
-import { ParallelStrategy } from "./ParallelStrategy";
-import type { Artifact, ExtractionOptions } from "../types";
-
-type Output = { title: string };
-
-const schema: JSONSchemaType<Output> = {
-  type: "object",
-  properties: { title: { type: "string" } },
-  required: ["title"],
-  additionalProperties: false,
-};
-
-const artifacts: Artifact[] = [
-  {
-    id: "a1",
-    type: "text",
-    raw: async () => Buffer.from(""),
-    contents: [{ text: "abcdefgh" }],
-  },
-  {
-    id: "a2",
-    type: "text",
-    raw: async () => Buffer.from(""),
-    contents: [{ text: "abcdefgh" }],
-  },
-];
-
-test("ParallelStrategy merges batch results", async () => {
-  let calls = 0;
-  const strategy = new ParallelStrategy<Output>({
-    model: {},
-    mergeModel: {},
-    chunkSize: 2,
-    execute: (async (request: any) => {
-      calls += 1;
-      const userText = typeof request.user === "string" ? request.user : "";
-      if (userText.includes("<json-objects>")) {
-        return {
-          data: { title: "merged" },
-          usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
-        };
-      }
-      return {
-        data: { title: "chunk" },
-        usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
-      };
-    }) as any,
-  });
-
-  const options: ExtractionOptions<Output> = {
-    artifacts,
-    schema,
-    strategy,
-  };
-
-  const result = await strategy.run(options);
-  expect(result.data.title).toBe("merged");
-  expect(calls).toBe(3);
-});

package/src/strategies/ParallelStrategy.ts

@@ -1,208 +0,0 @@
-import type { ExtractionResult, ExtractionStrategy } from "../types";
-import type { ExtractionOptions } from "../types";
-import { buildExtractorPrompt } from "../prompts/ExtractorPrompt";
-import { buildParallelMergerPrompt } from "../prompts/ParallelMergerPrompt";
-import {
-  extractWithPrompt,
-  getBatches,
-  mergeUsage,
-  serializeSchema,
-} from "./utils";
-import { runConcurrently } from "./concurrency";
-import { runWithRetries } from "../llm/RetryingRunner";
-
-export type ParallelStrategyConfig = {
-  model: unknown;
-  mergeModel: unknown;
-  chunkSize: number;
-  concurrency?: number;
-  maxImages?: number;
-  outputInstructions?: string;
-  execute?: typeof runWithRetries;
-  strict?: boolean;
-};
-
-export class ParallelStrategy<T> implements ExtractionStrategy<T> {
-  public name = "parallel";
-  private config: ParallelStrategyConfig;
-
-  constructor(config: ParallelStrategyConfig) {
-    this.config = config;
-  }
-
-  getEstimatedSteps(artifacts: ExtractionOptions<T>["artifacts"]): number {
-    const batches = getBatches(artifacts, {
-      maxTokens: this.config.chunkSize,
-      maxImages: this.config.maxImages,
-    });
-    return batches.length + 3;
-  }
-
-  async run(options: ExtractionOptions<T>): Promise<ExtractionResult<T>> {
-    const debug = options.debug;
-    const { telemetry } = options;
-    
-    // Create strategy-level span
-    const strategySpan = telemetry?.startSpan({
-      name: "strategy.parallel",
-      kind: "CHAIN",
-      attributes: {
-        "strategy.name": this.name,
-        "strategy.artifacts.count": options.artifacts.length,
-        "strategy.chunk_size": this.config.chunkSize,
-        "strategy.concurrency": this.config.concurrency,
-      },
-    });
-    
-    const batches = getBatches(
-      options.artifacts,
-      {
-        maxTokens: this.config.chunkSize,
-        maxImages: this.config.maxImages,
-      },
-      debug,
-      telemetry ?? undefined,
-      strategySpan,
-    );
-
-    const schema = serializeSchema(options.schema);
-    const totalSteps = this.getEstimatedSteps(options.artifacts);
-    let step = 1;
-
-    // Emit start event
-    await options.events?.onStep?.({
-      step,
-      total: totalSteps,
-      label: batches.length > 1 ? `batch 1/${batches.length}` : "extract",
-    });
-    debug?.step({
-      step,
-      total: totalSteps,
-      label: batches.length > 1 ? `batch 1/${batches.length}` : "extract",
-      strategy: this.name,
-    });
-
-    const tasks = batches.map((batch, index) => async () => {
-      const prompt = buildExtractorPrompt(
-        batch,
-        schema,
-        this.config.outputInstructions,
-      );
-      const result = await extractWithPrompt<T>({
-        model: this.config.model,
-        schema: options.schema,
-        system: prompt.system,
-        user: prompt.user,
-        artifacts: batch,
-        events: options.events,
-        execute: this.config.execute as never,
-        strict: options.strict ?? this.config.strict,
-        debug,
-        callId: `parallel_batch_${index + 1}`,
-        telemetry: telemetry ?? undefined,
-        parentSpan: strategySpan,
-      });
-      // Emit progress after batch completes (if there are more batches)
-      const completedIndex = index + 1;
-      if (completedIndex < batches.length) {
-        step += 1;
-        await options.events?.onStep?.({
-          step,
-          total: totalSteps,
-          label: `batch ${completedIndex + 1}/${batches.length}`,
-        });
-        debug?.step({
-          step,
-          total: totalSteps,
-          label: `batch ${completedIndex + 1}/${batches.length}`,
-          strategy: this.name,
-        });
-      }
-      return result;
-    });
-
-    const results = await runConcurrently(
-      tasks,
-      this.config.concurrency ?? batches.length,
-    );
-
-    debug?.mergeStart({
-      mergeId: "parallel_merge",
-      inputCount: results.length,
-      strategy: this.name,
-    });
-    
-    // Create merge span
-    const mergeSpan = telemetry?.startSpan({
-      name: "struktur.merge",
-      kind: "CHAIN",
-      parentSpan: strategySpan,
-      attributes: {
-        "merge.strategy": "parallel",
-        "merge.input_count": results.length,
-      },
-    });
-
-    const mergePrompt = buildParallelMergerPrompt(
-      schema,
-      results.map((r) => r.data),
-    );
-    const merged = await extractWithPrompt<T>({
-      model: this.config.mergeModel,
-      schema: options.schema,
-      system: mergePrompt.system,
-      user: mergePrompt.user,
-      artifacts: [],
-      events: options.events,
-      execute: this.config.execute as never,
-      strict: this.config.strict,
-      debug,
-      callId: "parallel_merge",
-      telemetry: telemetry ?? undefined,
-      parentSpan: mergeSpan,
-    });
-
-    step += 1;
-    await options.events?.onStep?.({
-      step,
-      total: totalSteps,
-      label: "merge",
-    });
-    debug?.step({
-      step,
-      total: totalSteps,
-      label: "merge",
-      strategy: this.name,
-    });
-    debug?.mergeComplete({ mergeId: "parallel_merge", success: true });
-    
-    // End merge span
-    if (mergeSpan && telemetry) {
-      telemetry.recordEvent(mergeSpan, {
-        type: "merge",
-        strategy: "parallel",
-        inputCount: results.length,
-        outputCount: 1,
-      });
-      telemetry.endSpan(mergeSpan, {
-        status: "ok",
-        output: merged.data,
-      });
-    }
-    
-    // End strategy span
-    telemetry?.endSpan(strategySpan!, {
-      status: "ok",
-      output: merged.data,
-    });
-
-    return {
-      data: merged.data,
-      usage: mergeUsage([...results.map((r) => r.usage), merged.usage]),
-    };
-  }
-}
-
-export const parallel = <T>(config: ParallelStrategyConfig) => {
-  return new ParallelStrategy<T>(config);
-};

package/src/strategies/SequentialAutoMergeStrategy.test.ts

@@ -1,66 +0,0 @@
-import { test, expect } from "bun:test";
-import type { JSONSchemaType } from "ajv";
-import { SequentialAutoMergeStrategy } from "./SequentialAutoMergeStrategy";
-import type { Artifact, ExtractionOptions } from "../types";
-
-type Output = { items: Array<{ id: number }> };
-
-const schema: JSONSchemaType<Output> = {
-  type: "object",
-  properties: {
-    items: {
-      type: "array",
-      items: {
-        type: "object",
-        properties: { id: { type: "number" } },
-        required: ["id"],
-        additionalProperties: false,
-      },
-    },
-  },
-  required: ["items"],
-  additionalProperties: false,
-};
-
-const artifacts: Artifact[] = [
-  {
-    id: "a1",
-    type: "text",
-    raw: async () => Buffer.from(""),
-    contents: [{ text: "abcdefgh" }],
-  },
-  {
-    id: "a2",
-    type: "text",
-    raw: async () => Buffer.from(""),
-    contents: [{ text: "abcdefgh" }],
-  },
-];
-
-test("SequentialAutoMergeStrategy merges and dedupes", async () => {
-  const strategy = new SequentialAutoMergeStrategy<Output>({
-    model: {},
-    chunkSize: 2,
-    execute: (async () => {
-      return {
-        data: { items: [{ id: 1 }, { id: 1 }] },
-        usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
-      };
-    }) as any,
-    dedupeExecute: (async () => {
-      return {
-        data: { keys: [] },
-        usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
-      };
-    }) as any,
-  });
-
-  const options: ExtractionOptions<Output> = {
-    artifacts,
-    schema,
-    strategy,
-  };
-
-  const result = await strategy.run(options);
-  expect(result.data.items.length).toBe(1);
-});