@output.ai/llm 0.2.3 → 0.2.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@output.ai/llm",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "description": "Framework abstraction to interact with LLM models",
   "type": "module",
   "main": "src/index.js",

package/src/ai_sdk.js

@@ -61,6 +61,8 @@ const extraAiSdkOptionsFromPrompt = prompt => {
  * @param {object} args - Generation arguments
  * @param {string} args.prompt - Prompt file name
  * @param {Record<string, string | number>} [args.variables] - Variables to interpolate
+ * @throws {ValidationError} If the prompt config is invalid (e.g., snake_case fields)
+ * @throws {FatalError} If the prompt file is not found or template rendering fails
  * @returns {Promise<string>} Generated text
  */
 export async function generateText( { prompt, variables } ) {
@@ -83,6 +85,8 @@ export async function generateText( { prompt, variables } ) {
  * @param {z.ZodObject} args.schema - Output schema
  * @param {string} [args.schemaName] - Output schema name
  * @param {string} [args.schemaDescription] - Output schema description
+ * @throws {ValidationError} If the prompt config is invalid (e.g., snake_case fields)
+ * @throws {FatalError} If the prompt file is not found or template rendering fails
  * @returns {Promise<object>} Object matching the provided schema
  */
 export async function generateObject( args ) {
@@ -112,6 +116,8 @@ export async function generateObject( args ) {
  * @param {z.ZodType} args.schema - Output schema (array item)
  * @param {string} [args.schemaName] - Output schema name
  * @param {string} [args.schemaDescription] - Output schema description
+ * @throws {ValidationError} If the prompt config is invalid (e.g., snake_case fields)
+ * @throws {FatalError} If the prompt file is not found or template rendering fails
  * @returns {Promise<object>} Array where each element matches the schema
  */
 export async function generateArray( args ) {
@@ -139,6 +145,8 @@ export async function generateArray( args ) {
  * @param {string} args.prompt - Prompt file name
  * @param {Record<string, string | number>} [args.variables] - Variables to interpolate
  * @param {string[]} args.enum - Allowed values for the generation
+ * @throws {ValidationError} If the prompt config is invalid (e.g., snake_case fields)
+ * @throws {FatalError} If the prompt file is not found or template rendering fails
  * @returns {Promise<string>} One of the provided enum values
  */
 export async function generateEnum( args ) {

package/src/index.d.ts

@@ -47,9 +47,6 @@ export type Prompt = {
     /** Maximum tokens in the response */
     maxTokens?: number;
 
-    /** Additional provider-specific options */
-    options?: Record<string, Record<string, JSONValue>>;
-
     /** Provider-specific configurations */
     providerOptions?: Record<string, unknown>;
   };

package/src/parser.spec.js

@@ -56,4 +56,67 @@ This is just plain text without any message tags.`;
       parsePrompt( raw );
     } ).toThrow( /Expected format/ );
   } );
+
+  it( 'should use providerOptions with budgetTokens in camelCase', () => {
+    // Frontmatter uses canonical format: providerOptions.thinking.budgetTokens
+    const raw = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+temperature: 0.7
+maxTokens: 64000
+providerOptions:
+  thinking:
+    type: enabled
+    budgetTokens: 1500
+---
+
+<user>Test</user>`;
+
+    const result = parsePrompt( raw );
+
+    expect( result.config.provider ).toBe( 'anthropic' );
+    expect( result.config.model ).toBe( 'claude-sonnet-4-20250514' );
+    expect( result.config.temperature ).toBe( 0.7 );
+    expect( result.config.maxTokens ).toBe( 64000 );
+
+    // Now expects canonical schema format in front-matter
+    expect( result.config.providerOptions ).toBeDefined();
+    expect( result.config.providerOptions.thinking ).toBeDefined();
+    expect( result.config.providerOptions.thinking.type ).toBe( 'enabled' );
+    expect( result.config.providerOptions.thinking.budgetTokens ).toBe( 1500 );
+  } );
+
+  it( 'should parse snake_case fields as-is (validation catches them later)', () => {
+    // Parser extracts fields as-is from frontmatter; validation happens later
+    const raw = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+temperature: 0.7
+max_tokens: 64000
+providerOptions:
+  thinking:
+    type: enabled
+    budget_tokens: 1500
+---
+
+<user>Test</user>`;
+
+    const result = parsePrompt( raw );
+
+    // Parser extracts snake_case fields as-is
+    expect( result.config.provider ).toBe( 'anthropic' );
+    expect( result.config.model ).toBe( 'claude-sonnet-4-20250514' );
+    expect( result.config.temperature ).toBe( 0.7 );
+    // snake_case preserved here because we're only calling parsePrompt, not validatePrompt
+    expect( result.config.max_tokens ).toBe( 64000 );
+    // camelCase not set because we only call parsePrompt
+    expect( result.config.maxTokens ).toBeUndefined();
+
+    // Snake_case fields in nested objects also preserved
+    expect( result.config.providerOptions ).toBeDefined();
+    expect( result.config.providerOptions.thinking ).toBeDefined();
+    expect( result.config.providerOptions.thinking.type ).toBe( 'enabled' );
+    expect( result.config.providerOptions.thinking.budget_tokens ).toBe( 1500 ); // snake_case preserved
+    expect( result.config.providerOptions.thinking.budgetTokens ).toBeUndefined(); // camelCase not set
+  } );
 } );

package/src/prompt_loader.spec.js

@@ -129,4 +129,21 @@ temperature: 0.7
     expect( result.config.model ).toBe( 'claude-sonnet-4' );
   } );
 
+  it( 'should use camelCase config keys', () => {
+    const promptContent = `---
+provider: anthropic
+model: claude-3-5-sonnet-20241022
+maxTokens: 1024
+temperature: 0.7
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    const result = loadPrompt( 'test', {} );
+
+    expect( result.config.maxTokens ).toBe( 1024 );
+  } );
+
 } );

package/src/prompt_loader_validation.spec.js

@@ -0,0 +1,174 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { loadPrompt } from './prompt_loader.js';
+
+vi.mock( './load_content.js', () => ( {
+  loadContent: vi.fn()
+} ) );
+
+import { loadContent } from './load_content.js';
+import { ValidationError } from '@output.ai/core';
+
+describe( 'loadPrompt - validation with real schema', () => {
+  beforeEach( () => {
+    vi.clearAllMocks();
+  } );
+
+  it( 'should accept valid camelCase format with providerOptions and budgetTokens', () => {
+    // Frontmatter uses canonical format: providerOptions.thinking.budgetTokens
+    const promptContent = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+temperature: 0.7
+maxTokens: 64000
+providerOptions:
+  thinking:
+    type: enabled
+    budgetTokens: 1500
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    const result = loadPrompt( 'test', {} );
+
+    // Validation passes with canonical schema format
+    expect( result.config.providerOptions ).toBeDefined();
+    expect( result.config.providerOptions.thinking.type ).toBe( 'enabled' );
+    expect( result.config.providerOptions.thinking.budgetTokens ).toBe( 1500 );
+  } );
+
+  it( 'should REJECT snake_case max_tokens (not canonical)', () => {
+    // Using snake_case max_tokens should fail validation
+    const promptContent = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+max_tokens: 64000
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    // loadPrompt should throw ValidationError because schema should reject max_tokens.
+    // Other tests check the actual message, but this one specifically checks that we throw
+    // the right kind of error, just to check that as well
+    expect( () => {
+      loadPrompt( 'test', {} );
+    } ).toThrow( ValidationError );
+    expect( () => {
+      loadPrompt( 'test', {} );
+    } ).toThrow( /max_tokens/ );
+  } );
+
+  it( 'should REJECT deprecated "options" field (use providerOptions)', () => {
+    // Using 'options' instead of 'providerOptions' should fail validation
+    const promptContent = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+options:
+  thinking:
+    type: enabled
+    budgetTokens: 1500
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    // loadPrompt should throw ValidationError because schema should reject 'options'
+    expect( () => {
+      loadPrompt( 'test', {} );
+    } ).toThrow( /Invalid prompt file/ );
+  } );
+
+  it( 'should REJECT snake_case budget_tokens in providerOptions', () => {
+    // Using snake_case budget_tokens should fail validation
+    const promptContent = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+providerOptions:
+  thinking:
+    type: enabled
+    budget_tokens: 1500
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    // loadPrompt should throw ValidationError because schema rejects budget_tokens
+    expect( () => {
+      loadPrompt( 'test', {} );
+    } ).toThrow( /Invalid prompt file/ );
+  } );
+
+  it( 'end-to-end: should accept canonical camelCase format', () => {
+    // End-to-end test: frontmatter uses canonical providerOptions.thinking.budgetTokens
+    const promptContent = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+temperature: 0.7
+maxTokens: 64000
+providerOptions:
+  thinking:
+    type: enabled
+    budgetTokens: 1500
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    const result = loadPrompt( 'test', {} );
+
+    // After loadPrompt, uses canonical format
+    expect( result.config.providerOptions ).toBeDefined();
+    expect( result.config.providerOptions.thinking.type ).toBe( 'enabled' );
+    expect( result.config.providerOptions.thinking.budgetTokens ).toBe( 1500 );
+  } );
+
+  it( 'end-to-end: should FAIL validation with snake_case budget_tokens', () => {
+    // End-to-end test: using snake_case should fail validation
+    const promptContent = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+providerOptions:
+  thinking:
+    type: enabled
+    budget_tokens: 1500
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    // loadPrompt should throw ValidationError because schema rejects budget_tokens
+    expect( () => {
+      loadPrompt( 'test', {} );
+    } ).toThrow( /Invalid prompt file/ );
+  } );
+
+  it( 'end-to-end: should FAIL validation with snake_case max_tokens', () => {
+    // End-to-end test: using snake_case should fail validation
+    const promptContent = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+max_tokens: 4000
+providerOptions:
+  thinking:
+    type: enabled
+    budget_tokens: 1500
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    // loadPrompt should throw ValidationError because schema rejects max_tokens
+    expect( () => {
+      loadPrompt( 'test', {} );
+    } ).toThrow( /Invalid prompt file/ );
+  } );
+
+} );

package/src/prompt_validations.js

@@ -11,20 +11,40 @@ export const promptSchema = z.object( {
       thinking: z.object( {
         type: z.literal( 'enabled' ),
         budgetTokens: z.number()
-      } ).optional()
-    } ).optional()
-  } ),
+      } ).strict().optional()
+    } ).strict().optional()
+  } ).strict(),
   messages: z.array(
     z.object( {
       role: z.string(),
       content: z.string()
-    } )
+    } ).strict()
   )
-} );
+} ).strict();
+
+const getHintForError = errorMessage => {
+  if ( errorMessage.includes( 'max_tokens' ) ) {
+    return '\nHint: Use "maxTokens" (camelCase) instead of "max_tokens".';
+  }
+  if ( errorMessage.includes( 'budget_tokens' ) ) {
+    return '\nHint: Use "budgetTokens" (camelCase) instead of "budget_tokens".';
+  }
+  if ( errorMessage.includes( '"options"' ) ) {
+    return '\nHint: Use "providerOptions" instead of "options".';
+  }
+  return '';
+};
 
 export function validatePrompt( prompt ) {
   const result = promptSchema.safeParse( prompt );
   if ( !result.success ) {
-    throw new ValidationError( `Invalid prompt file: ${z.prettifyError( result.error )}` );
+    const promptIdentifier = prompt?.name ? `"${prompt.name}"` : '(unnamed)';
+    const errorMessage = z.prettifyError( result.error );
+    const hint = getHintForError( errorMessage );
+
+    throw new ValidationError(
+      `Invalid prompt file ${promptIdentifier}: ${errorMessage}${hint}`,
+      { cause: result.error }
+    );
   }
 }

package/src/prompt_validations.spec.js

@@ -0,0 +1,121 @@
+import { describe, it, expect } from 'vitest';
+import { ValidationError } from '@output.ai/core';
+import { validatePrompt } from './prompt_validations.js';
+
+describe( 'validatePrompt', () => {
+  it( 'should validate a correct prompt with all required fields', () => {
+    const validPrompt = {
+      name: 'test-prompt',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-3-opus-20240229',
+        temperature: 0.7,
+        maxTokens: 1000
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Hello, world!'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( validPrompt ) ).not.toThrow();
+  } );
+
+  it( 'should validate a minimal prompt with only required fields', () => {
+    const minimalPrompt = {
+      name: 'minimal-prompt',
+      config: {
+        provider: 'openai',
+        model: 'gpt-4'
+      },
+      messages: [
+        {
+          role: 'system',
+          content: 'You are a helpful assistant.'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( minimalPrompt ) ).not.toThrow();
+  } );
+
+  it( 'should validate a prompt with thinking providerOptions', () => {
+    const promptWithThinking = {
+      name: 'thinking-prompt',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-3-5-sonnet-20241022',
+        providerOptions: {
+          thinking: {
+            type: 'enabled',
+            budgetTokens: 5000
+          }
+        }
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Solve this problem.'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( promptWithThinking ) ).not.toThrow();
+  } );
+
+  it( 'should throw ValidationError when provider is invalid', () => {
+    const invalidPrompt = {
+      name: 'invalid-provider',
+      config: {
+        provider: 'invalid-provider',
+        model: 'some-model'
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Test'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( invalidPrompt ) ).toThrow( ValidationError );
+  } );
+
+  it( 'should throw ValidationError when required fields are missing', () => {
+    const missingNamePrompt = {
+      config: {
+        provider: 'anthropic',
+        model: 'claude-3-opus-20240229'
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Test'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( missingNamePrompt ) ).toThrow( ValidationError );
+  } );
+
+  it( 'should throw ValidationError when max tokens is snake_case', () => {
+    const maxTokensSnakeCase = {
+      name: 'test-prompt',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-3-opus-20240229',
+        max_tokens: 4000
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Test'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( maxTokensSnakeCase ) ).toThrow( ValidationError );
+  } );
+} );