@artemiskit/adapter-openai 0.1.6 → 0.1.9

package/CHANGELOG.md

@@ -1,5 +1,35 @@
 # @artemiskit/adapter-openai
 
+## 0.1.9
+
+### Patch Changes
+
+- Updated dependencies [d5ca7c6]
+  - @artemiskit/core@0.2.2
+
+## 0.1.8
+
+### Patch Changes
+
+- fix: improve LLM grader compatibility with reasoning models
+
+  - Remove temperature parameter from LLM grader (reasoning models like o1, o3, gpt-5-mini only support temperature=1)
+  - Increase maxTokens from 200 to 1000 to accommodate reasoning models that use tokens for internal thinking
+  - Improve grader prompt for stricter JSON-only output format
+  - Add fallback parsing for malformed JSON responses
+  - Add markdown code block stripping from grader responses
+  - Add `modelFamily` configuration option to Azure OpenAI provider for correct parameter detection when deployment names differ from model names
+
+- Updated dependencies
+  - @artemiskit/core@0.2.1
+
+## 0.1.7
+
+### Patch Changes
+
+- Updated dependencies [d2c3835]
+  - @artemiskit/core@0.2.0
+
 ## 0.1.6
 
 ### Patch Changes

package/dist/client.d.ts

@@ -5,15 +5,23 @@
 import type { AdapterConfig, GenerateOptions, GenerateResult, ModelCapabilities, ModelClient } from '@artemiskit/core';
 export declare class OpenAIAdapter implements ModelClient {
     private client;
+    private embeddingClient;
     private config;
     readonly provider: string;
     constructor(config: AdapterConfig);
     generate(options: GenerateOptions): Promise<GenerateResult>;
     stream(options: GenerateOptions, onChunk: (chunk: string) => void): AsyncIterable<string>;
-    embed(text: string): Promise<number[]>;
+    embed(text: string, model?: string): Promise<number[]>;
     capabilities(): Promise<ModelCapabilities>;
     close(): Promise<void>;
     private normalizePrompt;
     private mapFinishReason;
+    /**
+     * Determine if model requires max_completion_tokens instead of max_tokens
+     * Newer models (GPT-4o, GPT-5, o1, o3) require the new parameter
+     *
+     * @param model - The deployment name or model name used in the API call
+     */
+    private shouldUseMaxCompletionTokens;
 }
 //# sourceMappingURL=client.d.ts.map

package/dist/client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,WAAW,EACZ,MAAM,kBAAkB,CAAC;AAK1B,qBAAa,aAAc,YAAW,WAAW;IAC/C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,MAAM,CAAiD;IAC/D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAEd,MAAM,EAAE,aAAa;IA6B3B,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;IA4C1D,MAAM,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC;IAqB1F,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAWtC,YAAY,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAW1C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,eAAe;IAUvB,OAAO,CAAC,eAAe;CAcxB"}
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,WAAW,EACZ,MAAM,kBAAkB,CAAC;AAK1B,qBAAa,aAAc,YAAW,WAAW;IAC/C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,eAAe,CAAuB;IAC9C,OAAO,CAAC,MAAM,CAAiD;IAC/D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAEd,MAAM,EAAE,aAAa;IAyC3B,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;IAsD1D,MAAM,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC;IAqB1F,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAmBtD,YAAY,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAW1C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,eAAe;IAUvB,OAAO,CAAC,eAAe;IAevB;;;;;OAKG;IACH,OAAO,CAAC,4BAA4B;CAUrC"}

package/dist/index.js

@@ -5516,6 +5516,7 @@ var openai_default = OpenAI;
 // src/client.ts
 class OpenAIAdapter {
   client;
+  embeddingClient = null;
   config;
   provider;
   constructor(config) {
@@ -5531,6 +5532,16 @@ class OpenAIAdapter {
         timeout: azureConfig.timeout ?? 60000,
         maxRetries: azureConfig.maxRetries ?? 2
       });
+      if (azureConfig.embeddingDeploymentName) {
+        this.embeddingClient = new openai_default({
+          apiKey: azureConfig.apiKey,
+          baseURL: `https://${azureConfig.resourceName}.openai.azure.com/openai/deployments/${azureConfig.embeddingDeploymentName}`,
+          defaultQuery: { "api-version": azureConfig.apiVersion },
+          defaultHeaders: { "api-key": azureConfig.apiKey ?? "" },
+          timeout: azureConfig.timeout ?? 60000,
+          maxRetries: azureConfig.maxRetries ?? 2
+        });
+      }
     } else {
       const openaiConfig = config;
       this.provider = "openai";
@@ -5547,10 +5558,12 @@ class OpenAIAdapter {
     const startTime = Date.now();
     const model = options.model || this.config.defaultModel || "gpt-4";
     const messages = this.normalizePrompt(options.prompt);
+    const useNewTokenParam = this.shouldUseMaxCompletionTokens(model);
+    const tokenParams = options.maxTokens ? useNewTokenParam ? { max_completion_tokens: options.maxTokens } : { max_tokens: options.maxTokens } : {};
     const response = await this.client.chat.completions.create({
       model,
       messages,
-      max_tokens: options.maxTokens,
+      ...tokenParams,
       temperature: options.temperature,
       top_p: options.topP,
       seed: options.seed,
@@ -5600,10 +5613,11 @@ class OpenAIAdapter {
       }
     }
   }
-  async embed(text) {
-    const model = this.config.provider === "azure-openai" ? "text-embedding-ada-002" : "text-embedding-3-small";
-    const response = await this.client.embeddings.create({
-      model,
+  async embed(text, model) {
+    const embeddingModel = model || (this.config.provider === "azure-openai" ? "text-embedding-3-large" : "text-embedding-3-small");
+    const client = this.embeddingClient || this.client;
+    const response = await client.embeddings.create({
+      model: embeddingModel,
       input: text
     });
     return response.data[0].embedding;
@@ -5642,6 +5656,13 @@ class OpenAIAdapter {
         return "stop";
     }
   }
+  shouldUseMaxCompletionTokens(model) {
+    const azureConfig = this.config;
+    const modelToCheck = azureConfig.modelFamily || model;
+    const modelLower = modelToCheck.toLowerCase();
+    const newParamModels = ["gpt-4o", "gpt-4-turbo", "gpt-5", "o1", "o3", "chatgpt-4o"];
+    return newParamModels.some((m) => modelLower.includes(m));
+  }
 }
 export {
   OpenAIAdapter

package/dist/types.d.ts

@@ -20,5 +20,12 @@ export interface AzureOpenAIAdapterConfig {
     resourceName: string;
     deploymentName: string;
     apiVersion: string;
+    /** Optional separate deployment name for embedding models */
+    embeddingDeploymentName?: string;
+    /**
+     * Model family for parameter detection (e.g., 'gpt-5-mini' when deployment is '5-mini')
+     * Used to determine which API parameters to use (max_tokens vs max_completion_tokens)
+     */
+    modelFamily?: string;
 }
 //# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,EAAE,cAAc,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;CACpB"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,EAAE,cAAc,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@artemiskit/adapter-openai",
-  "version": "0.1.6",
+  "version": "0.1.9",
   "description": "OpenAI and Azure OpenAI adapter for ArtemisKit LLM evaluation toolkit",
   "type": "module",
   "license": "Apache-2.0",
@@ -30,7 +30,7 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@artemiskit/core": "0.1.6",
+    "@artemiskit/core": "workspace:*",
     "openai": "^4.28.0",
     "nanoid": "^5.0.0"
   },

package/src/client.ts

@@ -16,6 +16,7 @@ import type { AzureOpenAIAdapterConfig, OpenAIAdapterConfig } from './types';
 
 export class OpenAIAdapter implements ModelClient {
   private client: OpenAI;
+  private embeddingClient: OpenAI | null = null;
   private config: OpenAIAdapterConfig | AzureOpenAIAdapterConfig;
   readonly provider: string;
 
@@ -34,6 +35,18 @@ export class OpenAIAdapter implements ModelClient {
         timeout: azureConfig.timeout ?? 60000,
         maxRetries: azureConfig.maxRetries ?? 2,
       });
+
+      // Create separate client for embeddings if a different deployment is specified
+      if (azureConfig.embeddingDeploymentName) {
+        this.embeddingClient = new OpenAI({
+          apiKey: azureConfig.apiKey,
+          baseURL: `https://${azureConfig.resourceName}.openai.azure.com/openai/deployments/${azureConfig.embeddingDeploymentName}`,
+          defaultQuery: { 'api-version': azureConfig.apiVersion },
+          defaultHeaders: { 'api-key': azureConfig.apiKey ?? '' },
+          timeout: azureConfig.timeout ?? 60000,
+          maxRetries: azureConfig.maxRetries ?? 2,
+        });
+      }
     } else {
       const openaiConfig = config as OpenAIAdapterConfig;
       this.provider = 'openai';
@@ -54,10 +67,20 @@ export class OpenAIAdapter implements ModelClient {
 
     const messages = this.normalizePrompt(options.prompt);
 
+    // Determine which token limit parameter to use
+    // Newer models (GPT-4o, GPT-5, o1, o3) use max_completion_tokens
+    // Older models use max_tokens
+    const useNewTokenParam = this.shouldUseMaxCompletionTokens(model);
+    const tokenParams = options.maxTokens
+      ? useNewTokenParam
+        ? { max_completion_tokens: options.maxTokens }
+        : { max_tokens: options.maxTokens }
+      : {};
+
     const response = await this.client.chat.completions.create({
       model,
       messages,
-      max_tokens: options.maxTokens,
+      ...tokenParams,
       temperature: options.temperature,
       top_p: options.topP,
       seed: options.seed,
@@ -113,12 +136,20 @@ export class OpenAIAdapter implements ModelClient {
     }
   }
 
-  async embed(text: string): Promise<number[]> {
-    const model =
-      this.config.provider === 'azure-openai' ? 'text-embedding-ada-002' : 'text-embedding-3-small';
+  async embed(text: string, model?: string): Promise<number[]> {
+    // Use provided model, or fall back to provider defaults
+    const embeddingModel =
+      model ||
+      (this.config.provider === 'azure-openai'
+        ? 'text-embedding-3-large'
+        : 'text-embedding-3-small');
 
-    const response = await this.client.embeddings.create({
-      model,
+    // For Azure, use the dedicated embedding client if available
+    // Otherwise fall back to main client (which may fail if deployment doesn't support embeddings)
+    const client = this.embeddingClient || this.client;
+
+    const response = await client.embeddings.create({
+      model: embeddingModel,
       input: text,
     });
     return response.data[0].embedding;
@@ -163,4 +194,21 @@ export class OpenAIAdapter implements ModelClient {
         return 'stop';
     }
   }
+
+  /**
+   * Determine if model requires max_completion_tokens instead of max_tokens
+   * Newer models (GPT-4o, GPT-5, o1, o3) require the new parameter
+   *
+   * @param model - The deployment name or model name used in the API call
+   */
+  private shouldUseMaxCompletionTokens(model: string): boolean {
+    // First check modelFamily from config (most reliable for Azure deployments)
+    const azureConfig = this.config as AzureOpenAIAdapterConfig;
+    const modelToCheck = azureConfig.modelFamily || model;
+
+    const modelLower = modelToCheck.toLowerCase();
+    // Models that require max_completion_tokens
+    const newParamModels = ['gpt-4o', 'gpt-4-turbo', 'gpt-5', 'o1', 'o3', 'chatgpt-4o'];
+    return newParamModels.some((m) => modelLower.includes(m));
+  }
 }

package/src/types.ts

@@ -22,4 +22,11 @@ export interface AzureOpenAIAdapterConfig {
   resourceName: string;
   deploymentName: string;
   apiVersion: string;
+  /** Optional separate deployment name for embedding models */
+  embeddingDeploymentName?: string;
+  /**
+   * Model family for parameter detection (e.g., 'gpt-5-mini' when deployment is '5-mini')
+   * Used to determine which API parameters to use (max_tokens vs max_completion_tokens)
+   */
+  modelFamily?: string;
 }