@struktur/sdk 0.1.0

package/src/debug/logger.test.ts

@@ -0,0 +1,244 @@
+import { test, expect, beforeEach, afterEach } from "bun:test";
+import { createDebugLogger } from "./logger";
+
+let stderrOutput: string[];
+const originalStderrWrite = process.stderr.write;
+
+beforeEach(() => {
+  stderrOutput = [];
+  process.stderr.write = (chunk: unknown) => {
+    if (typeof chunk === "string") {
+      stderrOutput.push(chunk);
+    }
+    return true;
+  };
+});
+
+afterEach(() => {
+  process.stderr.write = originalStderrWrite;
+});
+
+test("createDebugLogger with enabled=false is a no-op", () => {
+  const logger = createDebugLogger(false);
+  logger.cliInit({ args: { test: true } });
+  expect(stderrOutput.length).toBe(0);
+});
+
+test("createDebugLogger with enabled=true logs to stderr", () => {
+  const logger = createDebugLogger(true);
+  logger.cliInit({ args: { test: true } });
+  expect(stderrOutput.length).toBe(1);
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.type).toBe("cli_init");
+  expect(parsed.args).toEqual({ test: true });
+  expect(parsed.timestamp).toBeDefined();
+});
+
+test("cliInit logs correct type", () => {
+  const logger = createDebugLogger(true);
+  logger.cliInit({ args: { strategy: "simple" } });
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.type).toBe("cli_init");
+});
+
+test("schemaLoaded logs source and size", () => {
+  const logger = createDebugLogger(true);
+  logger.schemaLoaded({ source: "file.json", schemaSize: 100 });
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.type).toBe("schema_loaded");
+  expect(parsed.source).toBe("file.json");
+  expect(parsed.schemaSize).toBe(100);
+});
+
+test("artifactsLoaded logs artifact details", () => {
+  const logger = createDebugLogger(true);
+  logger.artifactsLoaded({
+    count: 2,
+    artifacts: [
+      { id: "a1", type: "text", contentCount: 1, tokens: 10 },
+      { id: "a2", type: "pdf", contentCount: 3 },
+    ],
+    totalTokens: 1010,
+    totalImages: 2,
+  });
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.type).toBe("artifacts_loaded");
+  expect(parsed.count).toBe(2);
+  expect(parsed.totalTokens).toBe(1010);
+  expect(parsed.totalImages).toBe(2);
+});
+
+test("chunkingStart logs chunking parameters", () => {
+  const logger = createDebugLogger(true);
+  logger.chunkingStart({
+    artifactId: "a1",
+    totalTokens: 100,
+    maxTokens: 50,
+    maxImages: 5,
+  });
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.type).toBe("chunking_start");
+  expect(parsed.artifactId).toBe("a1");
+  expect(parsed.maxTokens).toBe(50);
+});
+
+test("llmCallStart logs call details", () => {
+  const logger = createDebugLogger(true);
+  logger.llmCallStart({
+    callId: "call-1",
+    model: "gpt-4",
+    schemaName: "extract",
+    systemLength: 100,
+    userLength: 200,
+    artifactCount: 3,
+  });
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.type).toBe("llm_call_start");
+  expect(parsed.callId).toBe("call-1");
+  expect(parsed.artifactCount).toBe(3);
+});
+
+test("llmCallComplete logs success with duration", () => {
+  const logger = createDebugLogger(true);
+  logger.llmCallComplete({
+    callId: "call-1",
+    success: true,
+    inputTokens: 100,
+    outputTokens: 50,
+    totalTokens: 150,
+    durationMs: 1234,
+  });
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.type).toBe("llm_call_complete");
+  expect(parsed.success).toBe(true);
+  expect(parsed.durationMs).toBe(1234);
+});
+
+test("llmCallComplete logs failure with error", () => {
+  const logger = createDebugLogger(true);
+  logger.llmCallComplete({
+    callId: "call-1",
+    success: false,
+    inputTokens: 100,
+    outputTokens: 0,
+    totalTokens: 100,
+    error: "API error",
+  });
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.success).toBe(false);
+  expect(parsed.error).toBe("API error");
+});
+
+test("retry logs retry attempt", () => {
+  const logger = createDebugLogger(true);
+  logger.retry({
+    callId: "call-1",
+    attempt: 2,
+    maxAttempts: 3,
+    reason: "schema_validation_failed",
+  });
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.type).toBe("retry");
+  expect(parsed.attempt).toBe(2);
+  expect(parsed.reason).toBe("schema_validation_failed");
+});
+
+test("validationStart logs validation attempt", () => {
+  const logger = createDebugLogger(true);
+  logger.validationStart({
+    callId: "call-1",
+    attempt: 1,
+    maxAttempts: 3,
+    strict: false,
+  });
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.type).toBe("validation_start");
+  expect(parsed.strict).toBe(false);
+});
+
+test("validationSuccess logs successful validation", () => {
+  const logger = createDebugLogger(true);
+  logger.validationSuccess({ callId: "call-1", attempt: 1 });
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.type).toBe("validation_success");
+});
+
+test("validationFailed logs validation errors", () => {
+  const logger = createDebugLogger(true);
+  logger.validationFailed({
+    callId: "call-1",
+    attempt: 1,
+    errors: [{ keyword: "required", message: "missing field" }],
+  });
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.type).toBe("validation_failed");
+  expect(parsed.errors).toBeDefined();
+});
+
+test("mergeStart logs merge operation", () => {
+  const logger = createDebugLogger(true);
+  logger.mergeStart({
+    mergeId: "merge-1",
+    inputCount: 3,
+    strategy: "parallel",
+  });
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.type).toBe("merge_start");
+  expect(parsed.inputCount).toBe(3);
+});
+
+test("mergeComplete logs merge result", () => {
+  const logger = createDebugLogger(true);
+  logger.mergeComplete({ mergeId: "merge-1", success: true });
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.type).toBe("merge_complete");
+  expect(parsed.success).toBe(true);
+});
+
+test("dedupeStart logs deduplication start", () => {
+  const logger = createDebugLogger(true);
+  logger.dedupeStart({ dedupeId: "dedupe-1", itemCount: 10 });
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.type).toBe("dedupe_start");
+  expect(parsed.itemCount).toBe(10);
+});
+
+test("dedupeComplete logs deduplication result", () => {
+  const logger = createDebugLogger(true);
+  logger.dedupeComplete({
+    dedupeId: "dedupe-1",
+    duplicatesFound: 3,
+    itemsRemoved: 3,
+  });
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.type).toBe("dedupe_complete");
+  expect(parsed.duplicatesFound).toBe(3);
+});
+
+test("extractionComplete logs final result", () => {
+  const logger = createDebugLogger(true);
+  logger.extractionComplete({
+    success: true,
+    totalInputTokens: 100,
+    totalOutputTokens: 50,
+    totalTokens: 150,
+  });
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.type).toBe("extraction_complete");
+  expect(parsed.success).toBe(true);
+});
+
+test("smartMergeField logs field merge operation", () => {
+  const logger = createDebugLogger(true);
+  logger.smartMergeField({
+    mergeId: "merge-1",
+    field: "items",
+    operation: "merge_arrays",
+    leftCount: 5,
+    rightCount: 3,
+    resultCount: 8,
+  });
+  const parsed = JSON.parse(stderrOutput[0]!);
+  expect(parsed.type).toBe("smart_merge_field");
+  expect(parsed.operation).toBe("merge_arrays");
+});

package/src/debug/logger.ts

@@ -0,0 +1,211 @@
+import type { Artifact, ArtifactContent, ExtractionEvents, Usage, StepInfo, ProgressInfo, RetryInfo, TokenUsageInfo } from "../types";
+
+export type DebugLogger = ReturnType<typeof createDebugLogger>;
+
+export const createDebugLogger = (enabled: boolean) => {
+  const log = (entry: Record<string, unknown>) => {
+    if (!enabled) return;
+    const timestamp = new Date().toISOString();
+    const logEntry = { timestamp, ...entry };
+    process.stderr.write(JSON.stringify(logEntry) + "\n");
+  };
+
+  return {
+    // CLI initialization
+    cliInit: (data: { args: Record<string, unknown> }) => {
+      log({ type: "cli_init", ...data });
+    },
+
+    schemaLoaded: (data: { source: string; schemaSize: number }) => {
+      log({ type: "schema_loaded", ...data });
+    },
+
+    artifactsLoaded: (data: { 
+      count: number; 
+      artifacts: Array<{ id: string; type: string; contentCount: number; tokens?: number }>;
+      totalTokens: number;
+      totalImages: number;
+    }) => {
+      log({ type: "artifacts_loaded", ...data });
+    },
+
+    modelResolved: (data: { modelSpec: string; resolvedModel: string }) => {
+      log({ type: "model_resolved", ...data });
+    },
+
+    strategyCreated: (data: { strategy: string; config: Record<string, unknown> }) => {
+      log({ type: "strategy_created", ...data });
+    },
+
+    // Chunking
+    chunkingStart: (data: { 
+      artifactId: string; 
+      totalTokens: number; 
+      maxTokens: number;
+      maxImages?: number;
+    }) => {
+      log({ type: "chunking_start", ...data });
+    },
+
+    chunkingSplit: (data: {
+      artifactId: string;
+      originalContentCount: number;
+      splitContentCount: number;
+      splitReason: "text_too_long" | "content_limit";
+      originalTokens: number;
+      chunkSize: number;
+    }) => {
+      log({ type: "chunking_split", ...data });
+    },
+
+    chunkingResult: (data: {
+      artifactId: string;
+      chunksCreated: number;
+      chunkSizes: number[];
+    }) => {
+      log({ type: "chunking_result", ...data });
+    },
+
+    batchingStart: (data: {
+      totalArtifacts: number;
+      maxTokens: number;
+      maxImages?: number;
+      modelMaxTokens?: number;
+      effectiveMaxTokens: number;
+    }) => {
+      log({ type: "batching_start", ...data });
+    },
+
+    batchCreated: (data: {
+      batchIndex: number;
+      artifactCount: number;
+      totalTokens: number;
+      totalImages: number;
+      artifactIds: string[];
+    }) => {
+      log({ type: "batch_created", ...data });
+    },
+
+    batchingComplete: (data: {
+      totalBatches: number;
+      batches: Array<{ index: number; artifactCount: number; tokens: number; images: number }>;
+    }) => {
+      log({ type: "batching_complete", ...data });
+    },
+
+    // Strategy execution
+    strategyRunStart: (data: { strategy: string; estimatedSteps: number; artifactCount: number }) => {
+      log({ type: "strategy_run_start", ...data });
+    },
+
+    step: (data: StepInfo & { strategy: string }) => {
+      log({ type: "step", ...data });
+    },
+
+    progress: (data: ProgressInfo & { strategy: string; context?: string }) => {
+      log({ type: "progress", ...data });
+    },
+
+    // LLM calls
+    llmCallStart: (data: {
+      callId: string;
+      model: string;
+      schemaName?: string;
+      systemLength: number;
+      userLength: number;
+      artifactCount: number;
+    }) => {
+      log({ type: "llm_call_start", ...data });
+    },
+
+    llmCallComplete: (data: {
+      callId: string;
+      success: boolean;
+      inputTokens: number;
+      outputTokens: number;
+      totalTokens: number;
+      durationMs?: number;
+      error?: string;
+    }) => {
+      log({ type: "llm_call_complete", ...data });
+    },
+
+    // Retry events
+    retry: (data: RetryInfo & { callId: string }) => {
+      log({ type: "retry", ...data });
+    },
+
+    // Validation
+    validationStart: (data: { callId: string; attempt: number; maxAttempts: number; strict: boolean }) => {
+      log({ type: "validation_start", ...data });
+    },
+
+    validationSuccess: (data: { callId: string; attempt: number }) => {
+      log({ type: "validation_success", ...data });
+    },
+
+    validationFailed: (data: { callId: string; attempt: number; errors: unknown[] }) => {
+      log({ type: "validation_failed", ...data });
+    },
+
+    // Merging
+    mergeStart: (data: { mergeId: string; inputCount: number; strategy: string }) => {
+      log({ type: "merge_start", ...data });
+    },
+
+    mergeComplete: (data: { mergeId: string; success: boolean; error?: string }) => {
+      log({ type: "merge_complete", ...data });
+    },
+
+    // Deduplication
+    dedupeStart: (data: { dedupeId: string; itemCount: number }) => {
+      log({ type: "dedupe_start", ...data });
+    },
+
+    dedupeComplete: (data: { dedupeId: string; duplicatesFound: number; itemsRemoved: number }) => {
+      log({ type: "dedupe_complete", ...data });
+    },
+
+    // Token usage tracking
+    tokenUsage: (data: TokenUsageInfo & { context: string }) => {
+      log({ type: "token_usage", ...data });
+    },
+
+    // Results
+    extractionComplete: (data: {
+      success: boolean;
+      totalInputTokens: number;
+      totalOutputTokens: number;
+      totalTokens: number;
+      error?: string;
+    }) => {
+      log({ type: "extraction_complete", ...data });
+    },
+
+    // Prompt details (verbose)
+    promptSystem: (data: { callId: string; system: string }) => {
+      log({ type: "prompt_system", ...data });
+    },
+
+    promptUser: (data: { callId: string; user: unknown }) => {
+      log({ type: "prompt_user", ...data });
+    },
+
+    // Raw response
+    rawResponse: (data: { callId: string; response: unknown }) => {
+      log({ type: "raw_response", ...data });
+    },
+
+    // Smart merge details
+    smartMergeField: (data: {
+      mergeId: string;
+      field: string;
+      operation: "merge_arrays" | "merge_objects" | "replace" | "concat";
+      leftCount?: number;
+      rightCount?: number;
+      resultCount?: number;
+    }) => {
+      log({ type: "smart_merge_field", ...data });
+    },
+  };
+};

package/src/extract.test.ts

@@ -0,0 +1,22 @@
+import { test, expect } from "bun:test";
+import { extract } from "./extract";
+import type { ExtractionStrategy, ExtractionOptions } from "./types";
+
+test("extract delegates to strategy", async () => {
+  const strategy: ExtractionStrategy<{ ok: boolean }> = {
+    name: "mock",
+    run: async () => ({
+      data: { ok: true },
+      usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+    }),
+  };
+
+  const options: ExtractionOptions<{ ok: boolean }> = {
+    artifacts: [],
+    schema: {},
+    strategy,
+  };
+
+  const result = await extract(options);
+  expect(result.data.ok).toBe(true);
+});

package/src/extract.ts

@@ -0,0 +1,114 @@
+import type { ExtractionOptions, ExtractionResult } from "./types";
+import { buildSchemaFromFields } from "./fields";
+
+const emptyUsage = { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
+
+/**
+ * Resolve and validate the schema from ExtractionOptions.
+ * Exactly one of `schema` or `fields` must be provided.
+ */
+const resolveSchema = <T>(options: ExtractionOptions<T>) => {
+  const hasSchema = options.schema !== undefined;
+  const hasFields = options.fields !== undefined;
+
+  if (hasSchema && hasFields) {
+    throw new Error(
+      "Provide either `schema` or `fields`, not both. They are mutually exclusive.",
+    );
+  }
+
+  if (!hasSchema && !hasFields) {
+    throw new Error(
+      "A schema definition is required. Provide `schema` (a JSON Schema object) or `fields` (a shorthand fields string).",
+    );
+  }
+
+  if (hasFields) {
+    return buildSchemaFromFields(options.fields as string);
+  }
+
+  return options.schema as NonNullable<typeof options.schema>;
+};
+
+export const extract = async <T>(
+  options: ExtractionOptions<T>,
+): Promise<ExtractionResult<T>> => {
+  const debug = options.debug;
+
+  // Validate mutual exclusion and resolve the concrete schema early so that
+  // every strategy receives a fully-populated options object.
+  let resolvedOptions: ExtractionOptions<T>;
+  try {
+    const schema = resolveSchema(options);
+    resolvedOptions = { ...options, schema };
+  } catch (error) {
+    debug?.extractionComplete({
+      success: false,
+      totalInputTokens: 0,
+      totalOutputTokens: 0,
+      totalTokens: 0,
+      error: (error as Error).message,
+    });
+    return {
+      data: null as unknown as T,
+      usage: emptyUsage,
+      error: error as Error,
+    };
+  }
+
+  try {
+    const total = resolvedOptions.strategy.getEstimatedSteps?.(resolvedOptions.artifacts);
+
+    debug?.strategyRunStart({
+      strategy: resolvedOptions.strategy.name,
+      estimatedSteps: total ?? 1,
+      artifactCount: resolvedOptions.artifacts.length,
+    });
+
+    await resolvedOptions.events?.onStep?.({ step: 1, total, label: "start" });
+    debug?.step({
+      step: 1,
+      total,
+      label: "start",
+      strategy: resolvedOptions.strategy.name,
+    });
+
+    const result = await resolvedOptions.strategy.run(resolvedOptions);
+
+    await resolvedOptions.events?.onStep?.({
+      step: total ?? 1,
+      total,
+      label: "complete",
+    });
+    debug?.step({
+      step: total ?? 1,
+      total,
+      label: "complete",
+      strategy: resolvedOptions.strategy.name,
+    });
+
+    debug?.extractionComplete({
+      success: !result.error,
+      totalInputTokens: result.usage.inputTokens,
+      totalOutputTokens: result.usage.outputTokens,
+      totalTokens: result.usage.totalTokens,
+      error: result.error?.message,
+    });
+
+    return result;
+  } catch (error) {
+    debug?.extractionComplete({
+      success: false,
+      totalInputTokens: 0,
+      totalOutputTokens: 0,
+      totalTokens: 0,
+      error: (error as Error).message,
+    });
+
+    return {
+      data: null as unknown as T,
+      usage: emptyUsage,
+      error: error as Error,
+    };
+  }
+};