@struktur/sdk 1.2.1 → 2.1.0

package/src/llm/LLMClient.test.ts

@@ -194,3 +194,201 @@ test("generateStructured uses custom schema description", async () => {
 
   expect(calls[0]?.output).toHaveProperty("description", "Extract data");
 });
+
+test("generateStructured shows friendly error when model doesn't support images", async () => {
+  calls.length = 0;
+  generateTextImpl = async () => {
+    throw {
+      responseBody: '{"error":{"message":"No endpoints found that support image input","code":404}}',
+      statusCode: 404,
+    };
+  };
+
+  expect(
+    async () =>
+      await generateStructured({
+        model: { modelId: "meta-llama/llama-3.1-8b-instruct" },
+        schema: { type: "object" },
+        system: "sys",
+        user: [{ type: "text", text: "prompt" }, { type: "image", image: "base64data" }],
+      }),
+  ).toThrow(
+    'Model "meta-llama/llama-3.1-8b-instruct" does not support image input. Please use a model that supports images (e.g., gpt-4o, claude-3-5-sonnet, gemini-1.5-pro) or remove the --images and --screenshots flags.',
+  );
+});
+
+test("generateStructured rethrows other API errors", async () => {
+  calls.length = 0;
+  const originalError = new Error("Some other error");
+  generateTextImpl = async () => {
+    throw originalError;
+  };
+
+  expect(
+    async () =>
+      await generateStructured({
+        model: {},
+        schema: { type: "object" },
+        system: "sys",
+        user: "prompt",
+      }),
+  ).toThrow("Some other error");
+});
+
+test("generateStructured shows friendly error for internal server error", async () => {
+  calls.length = 0;
+  generateTextImpl = async () => {
+    throw {
+      statusCode: 200,
+      responseBody: undefined,
+      data: {
+        code: 500,
+        message: "Internal Server Error",
+        type: null,
+        param: null,
+      },
+    };
+  };
+
+  expect(
+    async () =>
+      await generateStructured({
+        model: { modelId: "openai/gpt-5-mini" },
+        schema: { type: "object" },
+        system: "sys",
+        user: "prompt",
+      }),
+  ).toThrow(
+    'Provider error for model "openai/gpt-5-mini": Internal server error. The model or provider may be experiencing issues. Please try again or use a different model.',
+  );
+});
+
+test("generateStructured shows friendly error for authentication failure", async () => {
+  calls.length = 0;
+  generateTextImpl = async () => {
+    throw {
+      statusCode: 401,
+      responseBody: '{"error":{"message":"Invalid API key"}}',
+      data: {
+        code: 401,
+        message: "Invalid API key",
+      },
+    };
+  };
+
+  expect(
+    async () =>
+      await generateStructured({
+        model: { modelId: "gpt-4o" },
+        schema: { type: "object" },
+        system: "sys",
+        user: "prompt",
+      }),
+  ).toThrow(
+    'Authentication failed for model "gpt-4o". Please check your API key is valid and has the necessary permissions.',
+  );
+});
+
+test("generateStructured shows friendly error for rate limit", async () => {
+  calls.length = 0;
+  generateTextImpl = async () => {
+    throw {
+      statusCode: 429,
+      responseBody: '{"error":{"message":"Rate limit exceeded"}}',
+      data: {
+        code: 429,
+        message: "Rate limit exceeded",
+      },
+    };
+  };
+
+  expect(
+    async () =>
+      await generateStructured({
+        model: { modelId: "claude-3-5-sonnet" },
+        schema: { type: "object" },
+        system: "sys",
+        user: "prompt",
+      }),
+  ).toThrow(
+    'Rate limit exceeded for model "claude-3-5-sonnet". Please wait a moment and try again, or use a different model.',
+  );
+});
+
+test("generateStructured shows friendly error for model not found", async () => {
+  calls.length = 0;
+  generateTextImpl = async () => {
+    throw {
+      statusCode: 404,
+      responseBody: '{"error":{"message":"Model not found"}}',
+      data: {
+        code: 404,
+        message: "Model not found",
+      },
+    };
+  };
+
+  expect(
+    async () =>
+      await generateStructured({
+        model: { modelId: "nonexistent-model" },
+        schema: { type: "object" },
+        system: "sys",
+        user: "prompt",
+      }),
+  ).toThrow(
+    'Model "nonexistent-model" not found or unavailable. Model not found Please check the model name or try a different model.',
+  );
+});
+
+test("generateStructured shows friendly error for access denied", async () => {
+  calls.length = 0;
+  generateTextImpl = async () => {
+    throw {
+      statusCode: 403,
+      responseBody: '{"error":{"message":"Access denied"}}',
+      data: {
+        code: 403,
+        message: "Access denied",
+      },
+    };
+  };
+
+  expect(
+    async () =>
+      await generateStructured({
+        model: { modelId: "gpt-4-turbo" },
+        schema: { type: "object" },
+        system: "sys",
+        user: "prompt",
+      }),
+  ).toThrow(
+    'Access denied for model "gpt-4-turbo". Your API key may not have access to this model. Please check your subscription or try a different model.',
+  );
+});
+
+test("generateStructured shows generic provider error message", async () => {
+  calls.length = 0;
+  generateTextImpl = async () => {
+    throw {
+      statusCode: 400,
+      responseBody: '{"error":{"message":"Context length exceeded"}}',
+      data: {
+        code: 400,
+        message: "Context length exceeded",
+      },
+    };
+  };
+
+  expect(
+    async () =>
+      await generateStructured({
+        model: { modelId: "gpt-3.5-turbo" },
+        schema: { type: "object" },
+        system: "sys",
+        user: "prompt",
+      }),
+  ).toThrow(
+    'Provider error for model "gpt-3.5-turbo": Context length exceeded',
+  );
+});

package/src/llm/LLMClient.ts

@@ -1,5 +1,5 @@
 import { generateText, Output, jsonSchema, type ModelMessage } from "ai";
-import type { AnyJSONSchema, Usage } from "../types";
+import type { AnyJSONSchema, Usage, TelemetryAdapter } from "../types";
 import type { UserContent } from "./message";
 
 type GenerateTextParams = Parameters<typeof generateText>[0];
@@ -15,6 +15,14 @@ export type StructuredRequest<T> = {
   schemaName?: string;
   schemaDescription?: string;
   strict?: boolean;
+  /**
+   * Telemetry adapter for tracing LLM calls
+   */
+  telemetry?: TelemetryAdapter;
+  /**
+   * Parent span for creating hierarchical traces
+   */
+  parentSpan?: { id: string; traceId: string; name: string; kind: string; startTime: number; parentId?: string };
 };
 
 export type StructuredResponse<T> = {
@@ -36,6 +44,21 @@ const isZodSchema = (
 export const generateStructured = async <T>(
   request: StructuredRequest<T>,
 ): Promise<StructuredResponse<T>> => {
+  const { telemetry, parentSpan } = request;
+  
+  // Start LLM span if telemetry is enabled
+  const llmSpan = telemetry?.startSpan({
+    name: "llm.generateStructured",
+    kind: "LLM",
+    parentSpan,
+    attributes: {
+      "llm.schema_name": request.schemaName ?? "extract",
+      "llm.strict": request.strict ?? false,
+    },
+  });
+
+  const startTime = Date.now();
+
   const schema = isZodSchema(request.schema)
     ? request.schema
     : jsonSchema(request.schema as AnyJSONSchema);
@@ -61,26 +84,128 @@ export const generateStructured = async <T>(
       }
     : undefined;
 
-  const result = await generateText({
-    model: request.model as ModelType,
-    output: Output.object({
-      schema: schema as GenerateTextParams extends { schema: infer S }
-        ? S
-        : never,
-      name: request.schemaName ?? "extract",
-      description: request.schemaDescription,
-    }),
-    providerOptions: {
-      openai: {
-        strictJsonSchema: request.strict ?? false,
+  let result;
+  try {
+    result = await generateText({
+      model: request.model as ModelType,
+      output: Output.object({
+        schema: schema as GenerateTextParams extends { schema: infer S }
+          ? S
+          : never,
+        name: request.schemaName ?? "extract",
+        description: request.schemaDescription,
+      }),
+      providerOptions: {
+        openai: {
+          strictJsonSchema: request.strict ?? false,
+        },
       },
-    },
-    system: request.system,
-    messages: (request.messages ?? [
-      { role: "user", content: request.user },
-    ]) as MessageType,
-    ...(providerOptions ? { providerOptions } : {}),
-  });
+      system: request.system,
+      messages: (request.messages ?? [
+        { role: "user", content: request.user },
+      ]) as MessageType,
+      ...(providerOptions ? { providerOptions } : {}),
+    });
+  } catch (error) {
+    // Determine model ID for error messages
+    const modelId =
+      typeof request.model === "object" && request.model !== null
+        ? (request.model as { modelId?: string }).modelId ??
+          JSON.stringify(request.model)
+        : String(request.model);
+
+    if (
+      error &&
+      typeof error === "object" &&
+      "responseBody" in error &&
+      "statusCode" in error
+    ) {
+      const apiError = error as {
+        responseBody: unknown;
+        statusCode: number;
+        data?: {
+          code?: number;
+          message?: string;
+          type?: string | null;
+          param?: string | null;
+        };
+      };
+
+      const responseBody = apiError.responseBody;
+      const errorData = apiError.data;
+
+      if (
+        typeof responseBody === "string" &&
+        responseBody.includes("No endpoints found that support image input")
+      ) {
+        throw new Error(
+          `Model "${modelId}" does not support image input. Please use a model that supports images (e.g., gpt-4o, claude-3-5-sonnet, gemini-1.5-pro) or remove the --images and --screenshots flags.`,
+        );
+      }
+
+      if (errorData?.code === 500 || errorData?.message?.includes("Internal Server Error")) {
+        throw new Error(
+          `Provider error for model "${modelId}": Internal server error. The model or provider may be experiencing issues. Please try again or use a different model.`,
+        );
+      }
+
+      if (apiError.statusCode === 401 || errorData?.code === 401) {
+        throw new Error(
+          `Authentication failed for model "${modelId}". Please check your API key is valid and has the necessary permissions.`,
+        );
+      }
+
+      if (apiError.statusCode === 403 || errorData?.code === 403) {
+        throw new Error(
+          `Access denied for model "${modelId}". Your API key may not have access to this model. Please check your subscription or try a different model.`,
+        );
+      }
+
+      if (apiError.statusCode === 429 || errorData?.code === 429) {
+        throw new Error(
+          `Rate limit exceeded for model "${modelId}". Please wait a moment and try again, or use a different model.`,
+        );
+      }
+
+      if (apiError.statusCode === 404 || errorData?.code === 404) {
+        const errorMsg = errorData?.message || "Model not found";
+        throw new Error(
+          `Model "${modelId}" not found or unavailable. ${errorMsg} Please check the model name or try a different model.`,
+        );
+      }
+
+      if (errorData?.message) {
+        throw new Error(
+          `Provider error for model "${modelId}": ${errorData.message}`,
+        );
+      }
+    }
+    
+    // Record error in telemetry
+    if (llmSpan && telemetry) {
+      const latencyMs = Date.now() - startTime;
+      telemetry.recordEvent(llmSpan, {
+        type: "llm_call",
+        model: modelId,
+        provider: "unknown", // Will be determined by the model
+        input: {
+          messages: request.messages ?? [{ role: "user", content: typeof request.user === "string" ? request.user : "" }],
+          temperature: undefined,
+          maxTokens: undefined,
+          schema: request.schema,
+        },
+        error: error instanceof Error ? error : new Error(String(error)),
+        latencyMs,
+      });
+      telemetry.endSpan(llmSpan, {
+        status: "error",
+        error: error instanceof Error ? error : new Error(String(error)),
+        latencyMs,
+      });
+    }
+    
+    throw error;
+  }
 
   const usageRaw = result.usage ?? {};
   const inputTokens =
@@ -102,5 +227,38 @@ export const generateStructured = async <T>(
     totalTokens,
   };
 
+  // Record successful LLM call in telemetry
+  if (llmSpan && telemetry) {
+    const latencyMs = Date.now() - startTime;
+    telemetry.recordEvent(llmSpan, {
+      type: "llm_call",
+      model: typeof request.model === "object" && request.model !== null
+        ? (request.model as { modelId?: string }).modelId ?? "unknown"
+        : String(request.model),
+      provider: preferredProvider ?? "unknown",
+      input: {
+        messages: request.messages ?? [{ role: "user", content: typeof request.user === "string" ? request.user : "" }],
+        temperature: undefined,
+        maxTokens: undefined,
+        schema: request.schema,
+      },
+      output: {
+        content: JSON.stringify(result.output),
+        structured: true,
+        usage: {
+          input: inputTokens,
+          output: outputTokens,
+          total: totalTokens,
+        },
+      },
+      latencyMs,
+    });
+    telemetry.endSpan(llmSpan, {
+      status: "ok",
+      output: result.output,
+      latencyMs,
+    });
+  }
+
   return { data: result.output as T, usage };
 };

package/src/llm/RetryingRunner.ts

@@ -5,7 +5,7 @@ import {
   validateAllowingMissingRequired,
 } from "../validation/validator";
 import type { ModelMessage } from "ai";
-import type { ExtractionEvents, Usage } from "../types";
+import type { ExtractionEvents, Usage, TelemetryAdapter } from "../types";
 import type { DebugLogger } from "../debug/logger";
 import { generateStructured } from "./LLMClient";
 import type { UserContent } from "./message";
@@ -22,9 +22,30 @@ export type RetryOptions<T> = {
   strict?: boolean;
   debug?: DebugLogger;
   callId?: string;
+  /**
+   * Telemetry adapter for tracing validation and retries
+   */
+  telemetry?: TelemetryAdapter;
+  /**
+   * Parent span for creating hierarchical traces
+   */
+  parentSpan?: { id: string; traceId: string; name: string; kind: string; startTime: number; parentId?: string };
 };
 
 export const runWithRetries = async <T>(options: RetryOptions<T>) => {
+  const { telemetry, parentSpan } = options;
+  
+  // Start validation/retry span if telemetry is enabled
+  const retrySpan = telemetry?.startSpan({
+    name: "struktur.validation_retry",
+    kind: "CHAIN",
+    parentSpan,
+    attributes: {
+      "retry.max_attempts": options.maxAttempts ?? 3,
+      "retry.schema_name": options.schemaName ?? "extract",
+    },
+  });
+
   const ajv = createAjv();
   const maxAttempts = options.maxAttempts ?? 3;
   const messages: ModelMessage[] = [{ role: "user", content: options.user }];
@@ -76,6 +97,8 @@ export const runWithRetries = async <T>(options: RetryOptions<T>) => {
       user: options.user,
       messages,
       strict: options.strict,
+      telemetry,
+      parentSpan: retrySpan,
     });
     const durationMs = Date.now() - startTime;
 
@@ -105,6 +128,24 @@ export const runWithRetries = async <T>(options: RetryOptions<T>) => {
           durationMs,
         });
 
+        // Record successful validation
+        if (retrySpan && telemetry) {
+          telemetry.recordEvent(retrySpan, {
+            type: "validation",
+            attempt,
+            maxAttempts,
+            schema: options.schema,
+            input: result.data,
+            success: true,
+            latencyMs: durationMs,
+          });
+          telemetry.endSpan(retrySpan, {
+            status: "ok",
+            output: validated,
+            latencyMs: durationMs,
+          });
+        }
+
         return { data: validated, usage };
       } else {
         const validationResult = validateAllowingMissingRequired<T>(
@@ -125,6 +166,24 @@ export const runWithRetries = async <T>(options: RetryOptions<T>) => {
             durationMs,
           });
 
+          // Record successful validation
+          if (retrySpan && telemetry) {
+            telemetry.recordEvent(retrySpan, {
+              type: "validation",
+              attempt,
+              maxAttempts,
+              schema: options.schema,
+              input: result.data,
+              success: true,
+              latencyMs: durationMs,
+            });
+            telemetry.endSpan(retrySpan, {
+              status: "ok",
+              output: validationResult.data,
+              latencyMs: durationMs,
+            });
+          }
+
           return { data: validationResult.data, usage };
         }
 
@@ -143,6 +202,20 @@ export const runWithRetries = async <T>(options: RetryOptions<T>) => {
           errors: error.errors,
         });
 
+        // Record failed validation
+        if (retrySpan && telemetry) {
+          telemetry.recordEvent(retrySpan, {
+            type: "validation",
+            attempt,
+            maxAttempts,
+            schema: options.schema,
+            input: result.data,
+            success: false,
+            errors: error.errors,
+            latencyMs: durationMs,
+          });
+        }
+
         // Emit retry event before attempting retry
         const nextAttempt = attempt + 1;
         if (nextAttempt <= maxAttempts) {
@@ -180,6 +253,15 @@ export const runWithRetries = async <T>(options: RetryOptions<T>) => {
         error: (error as Error).message,
       });
 
+      // Record error in telemetry
+      if (retrySpan && telemetry) {
+        telemetry.endSpan(retrySpan, {
+          status: "error",
+          error: error as Error,
+          latencyMs: durationMs,
+        });
+      }
+
       break;
     }
   }

package/src/llm/resolveModel.ts

@@ -0,0 +1,86 @@
+import { resolveProviderEnvVar, resolveProviderToken } from "../auth/tokens";
+
+export const resolveModel = async (model: string) => {
+  (globalThis as { AI_SDK_LOG_WARNINGS?: boolean }).AI_SDK_LOG_WARNINGS ??= false;
+  process.env.AI_SDK_LOG_WARNINGS ??= "false";
+  const [provider, ...rest] = model.split("/");
+  const modelName = rest.join("/");
+
+  if (!provider || !modelName) {
+    throw new Error(`Invalid model format: ${model}. Expected format: provider/model (e.g., openai/gpt-4)`);
+  }
+
+  const envVar = resolveProviderEnvVar(provider);
+  if (envVar && !process.env[envVar]) {
+    const storedToken = await resolveProviderToken(provider);
+    if (storedToken) {
+      process.env[envVar] = storedToken;
+    }
+  }
+
+  switch (provider) {
+    case "openai": {
+      const { openai } = await import("@ai-sdk/openai");
+      return openai(modelName);
+    }
+    case "anthropic": {
+      const { anthropic } = await import("@ai-sdk/anthropic");
+      return anthropic(modelName);
+    }
+    case "google": {
+      const { google } = await import("@ai-sdk/google");
+      return google(modelName);
+    }
+    case "opencode": {
+      const envVar = resolveProviderEnvVar("opencode");
+      let apiKey = envVar ? process.env[envVar] : undefined;
+      if (!apiKey) {
+        apiKey = await resolveProviderToken("opencode");
+      }
+      if (!apiKey) {
+        throw new Error("OpenCode API key is required. Set OPENCODE_API_KEY environment variable or run 'struktur auth set --provider opencode --token <token>'");
+      }
+      
+      if (modelName.startsWith("claude-")) {
+        const { createAnthropic } = await import("@ai-sdk/anthropic");
+        return createAnthropic({
+          apiKey,
+          baseURL: "https://opencode.ai/zen/v1",
+        })(modelName);
+      } else if (modelName.startsWith("gemini-")) {
+        const { createGoogleGenerativeAI } = await import("@ai-sdk/google");
+        return createGoogleGenerativeAI({
+          apiKey,
+          baseURL: "https://opencode.ai/zen/v1",
+        })(modelName);
+      } else {
+        const { createOpenAI } = await import("@ai-sdk/openai");
+        return createOpenAI({
+          apiKey,
+          baseURL: "https://opencode.ai/zen/v1",
+        })(modelName);
+      }
+    }
+    case "openrouter": {
+      const { openrouter } = await import("@openrouter/ai-sdk-provider");
+      const hashIndex = modelName.indexOf("#");
+      const actualModelName = hashIndex >= 0 ? modelName.slice(0, hashIndex) : modelName;
+      const preferredProvider = hashIndex >= 0 ? modelName.slice(hashIndex + 1) : undefined;
+      
+      const modelInstance = openrouter(actualModelName);
+      
+      if (preferredProvider) {
+        Object.defineProperty(modelInstance, "__openrouter_provider", {
+          value: preferredProvider,
+          writable: false,
+          enumerable: false,
+          configurable: false,
+        });
+      }
+      
+      return modelInstance;
+    }
+    default:
+      throw new Error(`Unsupported model provider: ${provider}. Supported providers: openai, anthropic, google, opencode, openrouter`);
+  }
+};