@struktur/sdk 0.1.0

package/src/fields.ts

@@ -0,0 +1,239 @@
+/**
+ * fields.ts — Shorthand schema builder.
+ *
+ * Parses a comma-separated fields string into a minimal JSON Schema object.
+ * Supported type expressions:
+ *
+ *   string (default)   title
+ *   number / float     price:number  or  price:float
+ *   boolean / bool     active:boolean  or  active:bool
+ *   integer            count:integer
+ *   int                count:int  (integer + multipleOf:1 to disallow fractions)
+ *   enum               status:enum{draft|published|archived}
+ *   array of scalar    tags:array{string}
+ *
+ * Aliases:
+ *   bool  → boolean
+ *   float → number
+ *   int   → integer (with multipleOf: 1)
+ *
+ * Examples:
+ *   parseFieldsString("title, description")
+ *   parseFieldsString("title, price:number")
+ *   parseFieldsString("title , price: number , active:boolean")
+ *   parseFieldsString("name, status:enum{draft|published}")
+ *   parseFieldsString("name, tags:array{string}")
+ *   parseFieldsString("count:int, ratio:float, enabled:bool")
+ */
+
+import type { AnyJSONSchema } from "./types";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type ScalarFieldType = "string" | "number" | "boolean" | "integer" | "int";
+
+export type ParsedField =
+  | { name: string; kind: "scalar"; type: ScalarFieldType }
+  | { name: string; kind: "enum"; values: string[] }
+  | { name: string; kind: "array"; items: ScalarFieldType };
+
+/** Legacy alias kept for backwards compatibility */
+export type FieldType = ScalarFieldType;
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const SCALAR_TYPES: ReadonlySet<string> = new Set([
+  "string",
+  "number",
+  "boolean",
+  "integer",
+  "int",
+]);
+
+/** Maps alias → canonical type accepted by this parser. */
+const SCALAR_ALIASES: Readonly<Record<string, ScalarFieldType>> = {
+  bool:  "boolean",
+  float: "number",
+  // Note: "int" stays as "int" (not aliased to "integer") so the schema
+  // builder can emit the extra multipleOf:1 constraint.
+};
+
+// ---------------------------------------------------------------------------
+// Internal parser helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract the content inside `prefix{...}` from a raw type string.
+ * Returns `null` if the pattern doesn't match.
+ */
+const extractBraces = (
+  rawType: string,
+  prefix: string,
+): string | null => {
+  if (!rawType.startsWith(prefix + "{") || !rawType.endsWith("}")) {
+    return null;
+  }
+  return rawType.slice(prefix.length + 1, -1);
+};
+
+const parseScalarType = (raw: string, fieldName: string): ScalarFieldType => {
+  // Resolve aliases first.
+  const resolved: string = SCALAR_ALIASES[raw] ?? raw;
+  if (!SCALAR_TYPES.has(resolved)) {
+    const allNames = [...Object.keys(SCALAR_ALIASES), ...SCALAR_TYPES].sort();
+    throw new Error(
+      `Unknown type "${raw}" for field "${fieldName}". ` +
+        `Scalar types: ${allNames.join(", ")}. ` +
+        `Complex types: enum{a|b|c}, array{string}.`,
+    );
+  }
+  return resolved as ScalarFieldType;
+};
+
+/**
+ * Parse a single `name` or `name:type` token into a ParsedField.
+ * Trims whitespace from name and type expression.
+ */
+const parseField = (token: string): ParsedField => {
+  const colonIndex = token.indexOf(":");
+
+  if (colonIndex === -1) {
+    const name = token.trim();
+    if (!name) throw new Error("Empty field name in fields string.");
+    return { name, kind: "scalar", type: "string" };
+  }
+
+  const name = token.slice(0, colonIndex).trim();
+  const rawType = token.slice(colonIndex + 1).trim();
+
+  if (!name) {
+    throw new Error(`Empty field name before colon in token: "${token}".`);
+  }
+  if (!rawType) {
+    throw new Error(
+      `Empty type after colon for field "${name}". ` +
+        `Omit the colon or specify a type.`,
+    );
+  }
+
+  // enum{a|b|c}
+  const enumContent = extractBraces(rawType, "enum");
+  if (enumContent !== null) {
+    const values = enumContent.split("|").map((v) => v.trim()).filter(Boolean);
+    if (values.length < 2) {
+      throw new Error(
+        `enum for field "${name}" must have at least two values separated by "|", got: "${enumContent}".`,
+      );
+    }
+    return { name, kind: "enum", values };
+  }
+
+  // array{itemType}
+  const arrayContent = extractBraces(rawType, "array");
+  if (arrayContent !== null) {
+    const itemType = arrayContent.trim();
+    if (!itemType) {
+      throw new Error(
+        `array for field "${name}" requires an item type, e.g. array{string}.`,
+      );
+    }
+    return { name, kind: "array", items: parseScalarType(itemType, name) };
+  }
+
+  // plain scalar
+  return { name, kind: "scalar", type: parseScalarType(rawType, name) };
+};
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse a comma-separated fields string into an array of ParsedField entries.
+ *
+ * @example
+ * parseFieldsString("title, price:number")
+ * // => [{ name: "title", kind: "scalar", type: "string" }, { name: "price", kind: "scalar", type: "number" }]
+ *
+ * parseFieldsString("status:enum{draft|published}")
+ * // => [{ name: "status", kind: "enum", values: ["draft", "published"] }]
+ *
+ * parseFieldsString("tags:array{string}")
+ * // => [{ name: "tags", kind: "array", items: "string" }]
+ */
+export const parseFieldsString = (fields: string): ParsedField[] => {
+  if (!fields.trim()) {
+    throw new Error("Fields string must not be empty.");
+  }
+
+  // Split on commas that are NOT inside braces so enum{a|b,c} would still
+  // work if someone added commas — but per spec values use |, so a simple
+  // brace-depth split is sufficient and keeps things robust.
+  const tokens: string[] = [];
+  let depth = 0;
+  let current = "";
+  for (const ch of fields) {
+    if (ch === "{") { depth++; current += ch; }
+    else if (ch === "}") { depth--; current += ch; }
+    else if (ch === "," && depth === 0) { tokens.push(current); current = ""; }
+    else { current += ch; }
+  }
+  if (current) tokens.push(current);
+
+  if (depth !== 0) {
+    throw new Error("Unmatched braces in fields string.");
+  }
+
+  return tokens.map((token) => parseField(token));
+};
+
+/**
+ * Build a minimal JSON Schema `object` from a parsed fields array.
+ * All fields are required; additionalProperties is false.
+ */
+export const buildSchemaFromParsedFields = (
+  fields: ParsedField[],
+): AnyJSONSchema => {
+  if (fields.length === 0) {
+    throw new Error("Cannot build a schema from an empty fields list.");
+  }
+
+  const properties: Record<string, unknown> = {};
+  const required: string[] = [];
+
+  for (const field of fields) {
+    if (field.kind === "scalar") {
+      properties[field.name] = field.type === "int"
+        ? { type: "integer", multipleOf: 1 }
+        : { type: field.type };
+    } else if (field.kind === "enum") {
+      properties[field.name] = { type: "string", enum: field.values };
+    } else {
+      // array
+      properties[field.name] = { type: "array", items: field.items === "int" ? { type: "integer", multipleOf: 1 } : { type: field.items } };
+    }
+    required.push(field.name);
+  }
+
+  return {
+    type: "object",
+    properties,
+    required,
+    additionalProperties: false,
+  };
+};
+
+/**
+ * Convenience: parse a fields string and immediately build a JSON Schema.
+ *
+ * @example
+ * buildSchemaFromFields("title, price:number")
+ * buildSchemaFromFields("status:enum{draft|published|archived}")
+ * buildSchemaFromFields("tags:array{string}")
+ */
+export const buildSchemaFromFields = (fields: string): AnyJSONSchema =>
+  buildSchemaFromParsedFields(parseFieldsString(fields));

package/src/index.test.ts

@@ -0,0 +1,20 @@
+import { test, expect } from "bun:test";
+import * as api from "./index";
+
+test("index re-exports main API", () => {
+  expect(typeof api.extract).toBe("function");
+  expect(typeof api.urlToArtifact).toBe("function");
+  expect(typeof api.fileToArtifact).toBe("function");
+  expect(typeof api.defaultArtifactProviders).toBe("object");
+  expect(typeof api.registerArtifactInputParser).toBe("function");
+  expect(typeof api.clearArtifactInputParsers).toBe("function");
+  expect(typeof api.validateSerializedArtifacts).toBe("function");
+  expect(typeof api.parseSerializedArtifacts).toBe("function");
+  expect(typeof api.hydrateSerializedArtifacts).toBe("function");
+  expect(typeof api.parse).toBe("function");
+  expect(typeof api.splitTextIntoContents).toBe("function");
+
+  expect(typeof api.simple).toBe("function");
+  expect(typeof api.parallel).toBe("function");
+  expect(typeof api.sequential).toBe("function");
+});

package/src/index.ts

@@ -0,0 +1,93 @@
+export type {
+  Artifact,
+  ArtifactContent,
+  ArtifactImage,
+  ArtifactType,
+  ExtractionEvents,
+  ExtractionOptions,
+  ExtractionResult,
+  ExtractionStrategy,
+  Usage,
+  AnyJSONSchema,
+  TypedJSONSchema,
+} from "./types";
+
+export { extract } from "./extract";
+export {
+  parseFieldsString,
+  buildSchemaFromParsedFields,
+  buildSchemaFromFields,
+} from "./fields";
+export type { ParsedField, FieldType } from "./fields";
+
+export type {
+  ArtifactInput,
+  ArtifactInputParser,
+  SerializedArtifact,
+  SerializedArtifactContent,
+  SerializedArtifactImage,
+  SerializedArtifacts,
+} from "./artifacts/input";
+
+export { urlToArtifact } from "./artifacts/urlToArtifact";
+export { fileToArtifact } from "./artifacts/fileToArtifact";
+export type { ArtifactProvider, ArtifactProviders } from "./artifacts/providers";
+export { defaultArtifactProviders } from "./artifacts/providers";
+export {
+  registerArtifactInputParser,
+  clearArtifactInputParsers,
+  validateSerializedArtifacts,
+  parseSerializedArtifacts,
+  hydrateSerializedArtifacts,
+  parse,
+  splitTextIntoContents,
+} from "./artifacts/input";
+
+export * from "./strategies";
+
+// Parsers public API
+export { collectStream } from "./parsers/collect";
+export type { ParserDef, ParsersConfig, InlineParserDef, NpmParserDef } from "./parsers/types";
+export { detectMimeType } from "./parsers/mime";
+export type { NpmParserEntry } from "./parsers/mime";
+export { runParser } from "./parsers/runner";
+export { parsePdf } from "./parsers/pdf";
+
+// Debug
+export { createDebugLogger } from "./debug/logger";
+
+// LLM models
+export {
+  listAllProviderModels,
+  listProviderModels,
+  resolveCheapestModel,
+} from "./llm/models";
+
+// Validation
+export { SchemaValidationError } from "./validation/validator";
+
+// Auth (for CLI and SDK users who want to manage tokens)
+export {
+  getDefaultModel,
+  setDefaultModel,
+  listAliases,
+  getAlias,
+  setAlias,
+  deleteAlias,
+  resolveAlias,
+  listParsers,
+  getParser,
+  setParser,
+  deleteParser,
+} from "./auth/config";
+export {
+  listStoredProviders,
+  setProviderToken,
+  deleteProviderToken,
+  resolveProviderToken,
+  getProviderTokenOrThrow,
+  resolveProviderEnvVar,
+  maskToken,
+  type TokenStorageType,
+  type TokenEntry,
+} from "./auth/tokens";

package/src/llm/AGENTS.md

@@ -0,0 +1,9 @@
+# LLM module
+
+- Purpose: wrap Vercel AI SDK calls, build multimodal messages, run validation retries, and query provider model lists/defaults.
+- Key files: `LLMClient.ts`, `RetryingRunner.ts`, `message.ts`, `models.ts`.
+- Design: `generateStructured` centralizes AI SDK usage; retry loop feeds validation errors back to the model.
+- Retry events: `runWithRetries` emits `onRetry` events with `{ attempt, maxAttempts, reason }` so the CLI can show retry progress (e.g. "Extracting data (retry 2/3)...").
+- Supported providers: openai, anthropic, google, opencode (Zen), openrouter.
+- OpenRouter provider routing: When a model has an `__openrouter_provider` property attached (set via hashtag syntax in the model string), `generateStructured` passes it as `providerOptions.openrouter.provider.order` to route requests to the specified provider.
+- Tests: `RetryingRunner.test.ts`, `models.test.ts`.

package/src/llm/LLMClient.test.ts

@@ -0,0 +1,196 @@
+import { test, expect, mock } from "bun:test";
+import type { ModelMessage } from "ai";
+
+type GenerateTextParams = {
+  model: unknown;
+  output: unknown;
+  system: string;
+  messages: ModelMessage[];
+  providerOptions?: unknown;
+};
+
+let generateTextImpl: (params: GenerateTextParams) => Promise<{
+  output: unknown;
+  usage?: Record<string, unknown>;
+}>;
+
+const calls: GenerateTextParams[] = [];
+
+mock.module("ai", () => ({
+  generateText: (params: GenerateTextParams) => {
+    calls.push(params);
+    return generateTextImpl(params);
+  },
+  Output: {
+    object: (config: unknown) => config,
+  },
+  jsonSchema: (schema: unknown) => ({ wrapped: schema }),
+}));
+
+const { generateStructured } = await import("./LLMClient");
+
+test("generateStructured maps prompt/completion token usage", async () => {
+  calls.length = 0;
+  generateTextImpl = async () => ({
+    output: { title: "ok" },
+    usage: { promptTokens: 2, completionTokens: 3, totalTokens: 9 },
+  });
+
+  const result = await generateStructured({
+    model: {},
+    schema: { type: "object" },
+    system: "sys",
+    user: "prompt",
+  });
+
+  expect(result.usage).toEqual({ inputTokens: 2, outputTokens: 3, totalTokens: 9 });
+  expect(calls[0]?.output).toEqual({ schema: { wrapped: { type: "object" } }, name: "extract" });
+  expect(calls[0]?.messages[0]).toEqual({ role: "user", content: "prompt" });
+});
+
+test("generateStructured uses explicit messages and totals usage", async () => {
+  calls.length = 0;
+  const messages: ModelMessage[] = [{ role: "user", content: "custom" }];
+  generateTextImpl = async (params) => ({
+    output: { title: "ok" },
+    usage: { inputTokens: 4, outputTokens: 6 },
+  });
+
+  const result = await generateStructured({
+    model: {},
+    schema: { type: "object" },
+    system: "sys",
+    user: "fallback",
+    messages,
+  });
+
+  expect(calls[0]?.messages).toBe(messages);
+  expect(result.usage).toEqual({ inputTokens: 4, outputTokens: 6, totalTokens: 10 });
+});
+
+test("generateStructured passes OpenRouter provider preference", async () => {
+  calls.length = 0;
+  generateTextImpl = async () => ({
+    output: { title: "ok" },
+    usage: { inputTokens: 1, outputTokens: 1 },
+  });
+
+  const model = { __openrouter_provider: "cerebras" };
+  await generateStructured({
+    model,
+    schema: { type: "object" },
+    system: "sys",
+    user: "prompt",
+  });
+
+  expect(calls[0]?.providerOptions).toEqual({
+    openrouter: {
+      provider: {
+        order: ["cerebras"],
+      },
+    },
+  });
+});
+
+test("generateStructured does not add openrouter providerOptions without preference", async () => {
+  calls.length = 0;
+  generateTextImpl = async () => ({
+    output: { title: "ok" },
+    usage: { inputTokens: 1, outputTokens: 1 },
+  });
+
+  await generateStructured({
+    model: {},
+    schema: { type: "object" },
+    system: "sys",
+    user: "prompt",
+  });
+
+  expect(calls[0]?.providerOptions).not.toHaveProperty("openrouter");
+});
+
+test("generateStructured uses inputTokens/outputTokens when promptTokens missing", async () => {
+  calls.length = 0;
+  generateTextImpl = async () => ({
+    output: { title: "ok" },
+    usage: { inputTokens: 5, outputTokens: 7 },
+  });
+
+  const result = await generateStructured({
+    model: {},
+    schema: { type: "object" },
+    system: "sys",
+    user: "prompt",
+  });
+
+  expect(result.usage).toEqual({ inputTokens: 5, outputTokens: 7, totalTokens: 12 });
+});
+
+test("generateStructured uses totalTokens from response when present", async () => {
+  calls.length = 0;
+  generateTextImpl = async () => ({
+    output: { title: "ok" },
+    usage: { inputTokens: 3, outputTokens: 4, totalTokens: 100 },
+  });
+
+  const result = await generateStructured({
+    model: {},
+    schema: { type: "object" },
+    system: "sys",
+    user: "prompt",
+  });
+
+  expect(result.usage.totalTokens).toBe(100);
+});
+
+test("generateStructured handles missing usage", async () => {
+  calls.length = 0;
+  generateTextImpl = async () => ({
+    output: { title: "ok" },
+  });
+
+  const result = await generateStructured({
+    model: {},
+    schema: { type: "object" },
+    system: "sys",
+    user: "prompt",
+  });
+
+  expect(result.usage).toEqual({ inputTokens: 0, outputTokens: 0, totalTokens: 0 });
+});
+
+test("generateStructured uses custom schema name", async () => {
+  calls.length = 0;
+  generateTextImpl = async () => ({
+    output: { title: "ok" },
+    usage: {},
+  });
+
+  await generateStructured({
+    model: {},
+    schema: { type: "object" },
+    schemaName: "custom_schema",
+    system: "sys",
+    user: "prompt",
+  });
+
+  expect(calls[0]?.output).toHaveProperty("name", "custom_schema");
+});
+
+test("generateStructured uses custom schema description", async () => {
+  calls.length = 0;
+  generateTextImpl = async () => ({
+    output: { title: "ok" },
+    usage: {},
+  });
+
+  await generateStructured({
+    model: {},
+    schema: { type: "object" },
+    schemaDescription: "Extract data",
+    system: "sys",
+    user: "prompt",
+  });
+
+  expect(calls[0]?.output).toHaveProperty("description", "Extract data");
+});

package/src/llm/LLMClient.ts

@@ -0,0 +1,106 @@
+import { generateText, Output, jsonSchema, type ModelMessage } from "ai";
+import type { AnyJSONSchema, Usage } from "../types";
+import type { UserContent } from "./message";
+
+type GenerateTextParams = Parameters<typeof generateText>[0];
+type ModelType = GenerateTextParams extends { model: infer M } ? M : unknown;
+type MessageType = Array<ModelMessage>;
+
+export type StructuredRequest<T> = {
+  model: ModelType | unknown;
+  system: string;
+  user: UserContent;
+  messages?: MessageType;
+  schema: unknown;
+  schemaName?: string;
+  schemaDescription?: string;
+  strict?: boolean;
+};
+
+export type StructuredResponse<T> = {
+  data: T;
+  usage: Usage;
+};
+
+const isZodSchema = (
+  schema: unknown,
+): schema is { safeParse: (data: unknown) => unknown } => {
+  return (
+    typeof schema === "object" &&
+    schema !== null &&
+    "safeParse" in schema &&
+    typeof (schema as { safeParse?: unknown }).safeParse === "function"
+  );
+};
+
+export const generateStructured = async <T>(
+  request: StructuredRequest<T>,
+): Promise<StructuredResponse<T>> => {
+  const schema = isZodSchema(request.schema)
+    ? request.schema
+    : jsonSchema(request.schema as AnyJSONSchema);
+
+  // Check for OpenRouter provider preference attached to the model
+  const preferredProvider = (
+    request.model as { __openrouter_provider?: string }
+  )?.__openrouter_provider;
+
+  if (preferredProvider && process.env.DEBUG) {
+    console.error(
+      `[DEBUG] Routing to OpenRouter provider: ${preferredProvider}`,
+    );
+  }
+
+  const providerOptions = preferredProvider
+    ? {
+        openrouter: {
+          provider: {
+            order: [preferredProvider],
+          },
+        },
+      }
+    : undefined;
+
+  const result = await generateText({
+    model: request.model as ModelType,
+    output: Output.object({
+      schema: schema as GenerateTextParams extends { schema: infer S }
+        ? S
+        : never,
+      name: request.schemaName ?? "extract",
+      description: request.schemaDescription,
+    }),
+    providerOptions: {
+      openai: {
+        strictJsonSchema: request.strict ?? false,
+      },
+    },
+    system: request.system,
+    messages: (request.messages ?? [
+      { role: "user", content: request.user },
+    ]) as MessageType,
+    ...(providerOptions ? { providerOptions } : {}),
+  });
+
+  const usageRaw = result.usage ?? {};
+  const inputTokens =
+    "promptTokens" in usageRaw
+      ? (usageRaw.promptTokens as number)
+      : ((usageRaw as { inputTokens?: number }).inputTokens ?? 0);
+  const outputTokens =
+    "completionTokens" in usageRaw
+      ? (usageRaw.completionTokens as number)
+      : ((usageRaw as { outputTokens?: number }).outputTokens ?? 0);
+  const totalTokens =
+    "totalTokens" in usageRaw
+      ? (usageRaw.totalTokens as number)
+      : inputTokens + outputTokens;
+
+  const usage: Usage = {
+    inputTokens,
+    outputTokens,
+    totalTokens,
+  };
+
+  return { data: result.output as T, usage };
+};