@struktur/sdk 2.1.2 → 2.3.0

package/src/parsers/runner.ts

@@ -1,177 +0,0 @@
-import os from "node:os";
-import path from "node:path";
-import { rm, writeFile } from "node:fs/promises";
-import type { Artifact } from "../types";
-import type { ParserDef, ParserInput } from "./types";
-import type { NpmParserModule } from "./npm";
-import {
-  hydrateSerializedArtifacts,
-  validateSerializedArtifacts,
-} from "../artifacts/input";
-
-const parseCommandOutput = (stdout: string): Artifact[] => {
-  let parsed: unknown;
-  try {
-    parsed = JSON.parse(stdout);
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    throw new Error(`Parser command produced invalid JSON: ${message}\nOutput: ${stdout.slice(0, 200)}`);
-  }
-  const serialized = validateSerializedArtifacts(parsed);
-  return hydrateSerializedArtifacts(serialized);
-};
-
-const spawnAndCapture = async (command: string, stdinBuffer?: Buffer): Promise<string> => {
-  if (!command.trim()) {
-    throw new Error(`Empty command: ${command}`);
-  }
-
-  const proc = Bun.spawn(["sh", "-c", command], {
-    stdout: "pipe",
-    stderr: "pipe",
-    stdin: stdinBuffer ? "pipe" : "ignore",
-  });
-
-  if (stdinBuffer && proc.stdin) {
-    // Bun's FileSink uses write/end, not the WritableStream API
-    const sink = proc.stdin as { write: (data: Uint8Array) => void; end: () => void };
-    sink.write(stdinBuffer);
-    sink.end();
-  }
-
-  const [stdout, stderr, exitCode] = await Promise.all([
-    new Response(proc.stdout).text(),
-    new Response(proc.stderr).text(),
-    proc.exited,
-  ]);
-
-  if (exitCode !== 0) {
-    throw new Error(
-      `Parser command exited with code ${exitCode}: ${command}\nStderr: ${stderr.slice(0, 500)}`
-    );
-  }
-
-  return stdout;
-};
-
-const runNpmParser = async (
-  pkg: string,
-  input: ParserInput,
-  mimeType: string,
-): Promise<Artifact[]> => {
-  const mod = (await import(pkg)) as NpmParserModule;
-
-  const hasParseFile = typeof mod.parseFile === "function";
-  const hasParseStream = typeof mod.parseStream === "function";
-
-  if (!hasParseFile && !hasParseStream) {
-    throw new Error(
-      `npm parser package "${pkg}" exports neither parseFile nor parseStream`
-    );
-  }
-
-  if (input.kind === "file") {
-    // Prefer parseFile for zero-copy
-    if (hasParseFile) {
-      return mod.parseFile!(input.path, mimeType);
-    }
-    // Fallback: open file as stream
-    const file = Bun.file(input.path);
-    const stream = file.stream() as ReadableStream<Uint8Array>;
-    return mod.parseStream!(stream, mimeType);
-  }
-
-  // input.kind === "buffer"
-  if (hasParseStream) {
-    // Prefer parseStream for buffers
-    const stream = new ReadableStream<Uint8Array>({
-      start(controller) {
-        controller.enqueue(input.buffer);
-        controller.close();
-      },
-    });
-    return mod.parseStream!(stream, mimeType);
-  }
-
-  // Fallback: write buffer to temp file, call parseFile, clean up
-  const tmpFile = path.join(os.tmpdir(), `struktur-parse-${crypto.randomUUID()}`);
-  try {
-    await writeFile(tmpFile, input.buffer);
-    return await mod.parseFile!(tmpFile, mimeType);
-  } finally {
-    await rm(tmpFile, { force: true });
-  }
-};
-
-const runCommandFileParser = async (
-  command: string,
-  input: ParserInput,
-): Promise<Artifact[]> => {
-  let filePath: string;
-  let tempFile: string | null = null;
-
-  if (input.kind === "file") {
-    filePath = input.path;
-  } else {
-    // Write buffer to temp file
-    tempFile = path.join(os.tmpdir(), `struktur-parse-${crypto.randomUUID()}`);
-    await writeFile(tempFile, input.buffer);
-    filePath = tempFile;
-  }
-
-  try {
-    const interpolated = command.replace(/FILE_PATH/g, filePath);
-    const stdout = await spawnAndCapture(interpolated);
-    return parseCommandOutput(stdout);
-  } finally {
-    if (tempFile) {
-      await rm(tempFile, { force: true });
-    }
-  }
-};
-
-const runCommandStdinParser = async (
-  command: string,
-  input: ParserInput,
-): Promise<Artifact[]> => {
-  let buffer: Buffer;
-
-  if (input.kind === "file") {
-    const file = Bun.file(input.path);
-    buffer = Buffer.from(await file.arrayBuffer());
-  } else {
-    buffer = input.buffer;
-  }
-
-  const stdout = await spawnAndCapture(command, buffer);
-  return parseCommandOutput(stdout);
-};
-
-export const runParser = async (
-  def: ParserDef,
-  input: ParserInput,
-  mimeType: string,
-): Promise<Artifact[]> => {
-  switch (def.type) {
-    case "npm":
-      return runNpmParser(def.package, input, mimeType);
-    case "command-file":
-      return runCommandFileParser(def.command, input);
-    case "command-stdin":
-      return runCommandStdinParser(def.command, input);
-    case "inline": {
-      let buffer: Buffer;
-      if (input.kind === "file") {
-        const file = Bun.file(input.path);
-        buffer = Buffer.from(await file.arrayBuffer());
-      } else {
-        buffer = input.buffer;
-      }
-      return [await def.handler(buffer)];
-    }
-    default: {
-      const _exhaustive: never = def;
-      throw new Error(`Unknown parser type: ${(_exhaustive as { type: string }).type}`);
-    }
-  }
-};

package/src/parsers/types.ts

@@ -1,29 +0,0 @@
-import type { Artifact } from "../types";
-
-export type NpmParserDef = {
-  type: "npm";
-  package: string; // e.g. "@myorg/pdf-parser"
-};
-
-export type CommandFileDef = {
-  type: "command-file";
-  command: string; // must contain FILE_PATH placeholder
-};
-
-export type CommandStdinDef = {
-  type: "command-stdin";
-  command: string;
-};
-
-export type InlineParserDef = {
-  type: "inline";
-  handler: (buffer: Buffer) => Promise<Artifact>;
-};
-
-export type ParserDef = NpmParserDef | CommandFileDef | CommandStdinDef | InlineParserDef;
-
-export type ParsersConfig = Record<string, ParserDef>; // keyed by MIME type
-
-export type ParserInput =
-  | { kind: "file"; path: string }
-  | { kind: "buffer"; buffer: Buffer };

package/src/prompts/AGENTS.md

@@ -1,8 +0,0 @@
-Prompts module
-
-- Purpose: generate optimized extraction/merge/dedupe prompt texts with thinking guidance and format artifacts as XML.
-- Design: prompt builders return `{ system, user }` strings with concise instructions and `<thinking>` sections; artifacts render into XML blocks with image refs.
-- Changes: System prompts now include structured `<thinking>` guidance, `<rules>` sections, and moved `outputInstructions` above schema for better context flow.
-- Key files: `ExtractorPrompt.ts`, `SequentialExtractorPrompt.ts`, `ParallelMergerPrompt.ts`, `DeduplicationPrompt.ts`, `formatArtifacts.ts`.
-- Design: prompt builders return `{ system, user }` strings; artifacts render into XML blocks with image refs.
-- Tests: `ExtractorPrompt.test.ts`, `SequentialExtractorPrompt.test.ts`, `ParallelMergerPrompt.test.ts`, `DeduplicationPrompt.test.ts`.

package/src/prompts/DeduplicationPrompt.test.ts

@@ -1,41 +0,0 @@
-import { test, expect } from "bun:test";
-import { buildDeduplicationPrompt } from "./DeduplicationPrompt";
-
-test("buildDeduplicationPrompt includes keys format instructions", () => {
-  const { system, user } = buildDeduplicationPrompt("{}", { items: [] });
-  expect(system).toContain("keys");
-  expect(user).toContain("duplicate");
-});
-
-test("buildDeduplicationPrompt embeds schema", () => {
-  const schema = '{"type":"object","properties":{"items":{"type":"array"}}}';
-  const { user } = buildDeduplicationPrompt(schema, { items: [] });
-  expect(user).toContain(schema);
-});
-
-test("buildDeduplicationPrompt embeds data", () => {
-  const data = { items: [{ id: 1 }, { id: 1 }] };
-  const { user } = buildDeduplicationPrompt("{}", data);
-  expect(user).toContain(JSON.stringify(data));
-});
-
-test("buildDeduplicationPrompt uses default example keys in example", () => {
-  const { user } = buildDeduplicationPrompt("{}", { items: [] });
-  expect(user).toContain("items.3");
-  expect(user).toContain("items.5");
-});
-
-test("buildDeduplicationPrompt includes thinking section", () => {
-  const { system } = buildDeduplicationPrompt("{}", { items: [] });
-  expect(system).toContain("<thinking>");
-});
-
-test("buildDeduplicationPrompt includes rules section", () => {
-  const { system } = buildDeduplicationPrompt("{}", { items: [] });
-  expect(system).toContain("<rules>");
-});
-
-test("buildDeduplicationPrompt includes task", () => {
-  const { user } = buildDeduplicationPrompt("{}", { items: [] });
-  expect(user).toContain("<task>");
-});

package/src/prompts/DeduplicationPrompt.ts

@@ -1,37 +0,0 @@
-export const buildDeduplicationPrompt = (
-  schema: string,
-  data: unknown,
-  exampleKeys: string[] = ["items.3", "items.5"]
-) => {
-  const system = `You are a deduplication engine. Identify duplicate entries in structured data.
-
-<thinking>
-Before deduplicating, consider:
-1. Which fields indicate uniqueness for each entity type?
-2. Are entries duplicates if they share key fields but differ in minor details?
-3. Which entry should be kept (prefer more complete data)?
-</thinking>
-
-<rules>
-- Identify entries that represent the same entity
-- Return paths to duplicates using dot notation (e.g., "items.3", "items.5")
-- Output ONLY JSON in format: { "keys": ["path1", "path2"] }
-- No markdown, no explanations
-</rules>`;
-
-  const user = `<json-schema>
-${schema}
-</json-schema>
-
-<json-data>
-${JSON.stringify(data)}
-</json-data>
-
-<task>Identify duplicate entries in the data and return their paths in the format: { "keys": ["path1", "path2"] }</task>
-
-<example>
-If items at indices 3 and 5 are duplicates, return: { "keys": ["items.3", "items.5"] }
-</example>`;
-
-  return { system, user };
-};

package/src/prompts/ExtractorPrompt.test.ts

@@ -1,21 +0,0 @@
-import { test, expect } from "bun:test";
-import { buildExtractorPrompt } from "./ExtractorPrompt";
-import type { Artifact } from "../types";
-
-const artifacts: Artifact[] = [
-  {
-    id: "a1",
-    type: "pdf",
-    raw: async () => Buffer.from(""),
-    contents: [
-      { page: 1, text: "Hello", media: [{ type: "image", url: "https://x" }] },
-    ],
-  },
-];
-
-test("buildExtractorPrompt includes schema and artifacts", () => {
-  const { system, user } = buildExtractorPrompt(artifacts, "{\"type\":\"object\"}");
-  expect(system).toContain("<json-schema>");
-  expect(user).toContain("<artifacts>");
-  expect(user).toContain("<image");
-});

package/src/prompts/ExtractorPrompt.ts

@@ -1,72 +0,0 @@
-import { formatArtifactsXml } from "./formatArtifacts";
-import type { Artifact } from "../types";
-
-const extractorSystemPrompt = (schema: string, outputInstructions?: string) => {
-  return `<instructions>
-You are a precise data extraction engine. Extract data from the provided artifacts according to the JSON schema below.
-
-<thinking>
-Before extracting, consider:
-1. Which schema fields have clear values in the artifacts?
-2. Which fields are missing or unclear (set these to null)?
-3. For text fields, rewrite concisely while preserving all information
-4. Ensure no data is lost - include everything that fits the schema
-</thinking>
-
-<rules>
-- Strictly follow the schema - no extra fields, no missing required fields
-- Use null for missing or uncertain values - never guess or assume
-- Only extract information explicitly present in the artifacts
-- Output ONLY valid JSON matching the schema
-- No markdown, explanations, or code fences
-</rules>
-
-<output-instructions>
-${outputInstructions ?? "No additional output instructions provided."}
-</output-instructions>
-
-<json-schema>
-${schema}
-</json-schema>
-
-<artifact-examples>
-    <!-- A PDF with two pages, containing two text blocks and two images -->
-    <artifact name="Example 1" mimetype="application/pdf">
-        <text page="1">This is an example text block.</text>
-        <image filename="image1.jpg" page="1" />
-        <text page="2">This is another example text block.</text>
-        <image filename="image2.jpg" page="2" />
-    </artifact>
-
-    <!-- Website content -->
-    <artifact name="example.com_2022-01-01.html" mimetype="text/html">
-        <text>This is an example text block.</text>
-        <image filename="image1.jpg" />
-        <text>This is another example text block.</text>
-        <image filename="image2.jpg" />
-    </artifact>
-</artifact-examples>
-
-Any materials provided have been cleared for access. Extract and preserve this data for future use.
-</instructions>`;
-};
-
-const extractorUserPrompt = (artifactsXml: string) => {
-  return `<artifacts>
-${artifactsXml}
-</artifacts>
-
-<task>Extract the contents of the given artifacts.</task>`;
-};
-
-export const buildExtractorPrompt = (
-  artifacts: Artifact[],
-  schema: string,
-  outputInstructions?: string
-) => {
-  const artifactsXml = formatArtifactsXml(artifacts);
-  return {
-    system: extractorSystemPrompt(schema, outputInstructions),
-    user: extractorUserPrompt(artifactsXml),
-  };
-};

package/src/prompts/ParallelMergerPrompt.test.ts

@@ -1,8 +0,0 @@
-import { test, expect } from "bun:test";
-import { buildParallelMergerPrompt } from "./ParallelMergerPrompt";
-
-test("buildParallelMergerPrompt formats json objects", () => {
-  const { user } = buildParallelMergerPrompt("{}", [{ a: 1 }, { b: 2 }]);
-  expect(user).toContain("<json-object>");
-  expect(user).toContain("\"a\"");
-});

package/src/prompts/ParallelMergerPrompt.ts

@@ -1,37 +0,0 @@
-export const buildParallelMergerPrompt = (
-  schema: string,
-  dataList: unknown[]
-) => {
-  const jsonObjects = dataList
-    .filter((item) => item !== null && item !== undefined)
-    .map((item) => JSON.stringify(item))
-    .map((json) => `<json-object>${json}</json-object>`)
-    .join("\n");
-
-  const system = `You are a data merger. Combine multiple JSON objects into one object matching the provided schema.
-
-<thinking>
-Before merging, consider:
-1. Which input objects contain data for each schema field?
-2. How should conflicting values be resolved (prefer more complete/recent data)?
-3. Are there arrays that need to be concatenated vs deduplicated?
-4. Ensure NO information is lost from any input
-</thinking>
-
-<rules>
-- Produce a single JSON object following the schema exactly
-- Combine all information from input objects without losing data
-- Resolve conflicts intelligently (prefer richer/more specific data)
-- Output ONLY valid JSON - no markdown, no explanations
-</rules>`;
-
-  const user = `<json-schema>
-${schema}
-</json-schema>
-
-<json-objects>
-${jsonObjects}
-</json-objects>`;
-
-  return { system, user };
-};

package/src/prompts/SequentialExtractorPrompt.test.ts

@@ -1,24 +0,0 @@
-import { test, expect } from "bun:test";
-import { buildSequentialPrompt } from "./SequentialExtractorPrompt";
-import type { Artifact } from "../types";
-
-const artifacts: Artifact[] = [
-  {
-    id: "a1",
-    type: "pdf",
-    raw: async () => Buffer.from(""),
-    contents: [{ page: 1, text: "Hello" }],
-  },
-];
-
-test("buildSequentialPrompt embeds previous data", () => {
-  const { system, user } = buildSequentialPrompt(
-    artifacts,
-    "{\"type\":\"object\"}",
-    "{\"existing\":true}"
-  );
-
-  expect(system).toContain("JSON schema");
-  expect(user).toContain("<previous-data>");
-  expect(user).toContain("existing");
-});

package/src/prompts/SequentialExtractorPrompt.ts

@@ -1,82 +0,0 @@
-import { formatArtifactsXml } from "./formatArtifacts";
-import type { Artifact } from "../types";
-
-const sequentialSystemPrompt = (schema: string, outputInstructions?: string) => {
-  return `<instructions>
-You are a precise data extraction engine. Extract data from provided artifacts according to the JSON schema, enriching any previous data you receive.
-
-<thinking>
-Before extracting, consider:
-1. Review previous data - what needs to be preserved vs enriched?
-2. Which new fields have clear values in the artifacts?
-3. Which fields remain missing or unclear (keep null from previous or set to null)?
-4. Can new information improve the structure of existing data?
-5. Ensure NO information is lost from previous data
-</thinking>
-
-<rules>
-- Merge new artifacts into existing data - do not create fresh objects
-- Preserve ALL previous data - losing information breaks the processing chain
-- Use null for missing/uncertain values in new fields
-- Only extract information explicitly present in the artifacts
-- Output ONLY valid JSON matching the schema
-- No markdown, explanations, or code fences
-</rules>
-
-<image-handling>
-Some schema properties may reference artifact IDs (e.g., 'xxx_artifact_id' fields).
-When assigning images to properties:
-- Use format: artifact:ID/images/imageNUM.EXT (e.g., 'artifact:123456/images/image1.jpg')
-- Only reference images you can actually see in the provided documents/images
-- Image references are visible in artifact XML or written on images
-- NEVER make up artifact IDs or use normal URLs
-</image-handling>
-
-<output-instructions>
-${outputInstructions ?? "No additional output instructions provided."}
-</output-instructions>
-
-<json-schema>
-${schema}
-</json-schema>
-
-<how-to-output>
-Return the complete extracted data as valid JSON matching the schema.
-Include all information from previous data, enriched with the new artifacts.
-</how-to-output>
-</instructions>`;
-};
-
-const sequentialUserPrompt = (
-  artifactsXml: string,
-  previousData: string,
-  outputInstructions?: string
-) => {
-  return `${artifactsXml}
-
-<previous-data>
-${previousData}
-</previous-data>
-
-<task>
-    Extract the contents of the given artifacts and ADD/MERGE them into the previous data contained in the <previous-data> tag.
-    You MUST NOT lose any information from the previous data. All previous data must be included in your response.
-</task>
-
-<output-instructions>
-${outputInstructions ?? ""}
-</output-instructions>`;
-};
-
-export const buildSequentialPrompt = (
-  artifacts: Artifact[],
-  schema: string,
-  previousData: string,
-  outputInstructions?: string
-) => {
-  const artifactsXml = formatArtifactsXml(artifacts);
-  return {
-    system: sequentialSystemPrompt(schema, outputInstructions),
-    user: sequentialUserPrompt(artifactsXml, previousData, outputInstructions),
-  };
-};

package/src/prompts/formatArtifacts.test.ts

@@ -1,39 +0,0 @@
-import { test, expect } from "bun:test";
-import type { Artifact } from "../types";
-import { formatArtifactsXml } from "./formatArtifacts";
-
-test("formatArtifactsXml escapes text and builds image refs", () => {
-  const artifacts: Artifact[] = [
-    {
-      id: "a&<>\"'",
-      type: "text",
-      raw: async () => Buffer.from(""),
-      contents: [
-        {
-          page: 2,
-          text: "Hello & <world> \"quote\" 'apostrophe'",
-        },
-        {
-          page: 1,
-          media: [
-            { type: "image", url: "https://example.com/image.png" },
-            { type: "image", base64: "abc" },
-            { type: "image", contents: Buffer.from([1, 2, 3]) },
-          ],
-        },
-      ],
-    },
-  ];
-
-  const result = formatArtifactsXml(artifacts);
-  const expected = [
-    "<artifact id=\"a&amp;&lt;&gt;&quot;&apos;\" type=\"text\">",
-    "  <text page=\"2\">Hello &amp; &lt;world&gt; &quot;quote&quot; &apos;apostrophe&apos;</text>",
-    "  <image ref=\"https://example.com/image.png\" page=\"1\" />",
-    "  <image ref=\"artifact:a&amp;&lt;&gt;&quot;&apos;/images/image2.png\" page=\"1\" />",
-    "  <image ref=\"artifact:a&amp;&lt;&gt;&quot;&apos;/images/image3.bin\" page=\"1\" />",
-    "</artifact>",
-  ].join("\n");
-
-  expect(result).toBe(expected);
-});

package/src/prompts/formatArtifacts.ts

@@ -1,46 +0,0 @@
-import type { Artifact, ArtifactImage } from "../types";
-
-const imageRefFor = (artifactId: string, index: number, image: ArtifactImage) => {
-  if (image.url) {
-    return image.url;
-  }
-
-  const extension = image.base64 ? "png" : "bin";
-  return `artifact:${artifactId}/images/image${index + 1}.${extension}`;
-};
-
-const escapeXml = (value: string) => {
-  return value
-    .replace(/&/g, "&")
-    .replace(/</g, "&lt;")
-    .replace(/>/g, "&gt;")
-    .replace(/\"/g, "&quot;")
-    .replace(/'/g, "&apos;");
-};
-
-export const formatArtifactsXml = (artifacts: Artifact[]) => {
-  const parts: string[] = [];
-
-  for (const artifact of artifacts) {
-    parts.push(`<artifact id="${escapeXml(artifact.id)}" type="${artifact.type}">`);
-
-    for (const content of artifact.contents) {
-      if (content.text) {
-        const pageAttr = content.page !== undefined ? ` page="${content.page}"` : "";
-        parts.push(`  <text${pageAttr}>${escapeXml(content.text)}</text>`);
-      }
-
-      if (content.media?.length) {
-        content.media.forEach((media, index) => {
-          const ref = imageRefFor(artifact.id, index, media);
-          const pageAttr = content.page !== undefined ? ` page="${content.page}"` : "";
-          parts.push(`  <image ref="${escapeXml(ref)}"${pageAttr} />`);
-        });
-      }
-    }
-
-    parts.push("</artifact>");
-  }
-
-  return parts.join("\n");
-};

package/src/strategies/AGENTS.md

@@ -1,6 +0,0 @@
-Strategies module
-
-- Purpose: orchestrate extraction flows (simple, parallel, sequential, auto-merge, double-pass).
-- Key files: `SimpleStrategy.ts`, `ParallelStrategy.ts`, `SequentialStrategy.ts`, `ParallelAutoMergeStrategy.ts`, `SequentialAutoMergeStrategy.ts`, `DoublePassStrategy.ts`, `DoublePassAutoMergeStrategy.ts`, `utils.ts`, `concurrency.ts`.
-- Design: strategies own config (chunk size, concurrency, models) and call prompt + retry helpers. Strategies emit `events.onStep` updates and implement `getEstimatedSteps` for progress tracking.
-- Tests: strategy-specific `*.test.ts` files.