@output.ai/llm 0.2.2 → 0.2.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@output.ai/llm",
-  "version": "0.2.2",
+  "version": "0.2.4",
   "description": "Framework abstraction to interact with LLM models",
   "type": "module",
   "main": "src/index.js",

package/src/ai_sdk.js

@@ -61,6 +61,8 @@ const extraAiSdkOptionsFromPrompt = prompt => {
  * @param {object} args - Generation arguments
  * @param {string} args.prompt - Prompt file name
  * @param {Record<string, string | number>} [args.variables] - Variables to interpolate
+ * @throws {ValidationError} If the prompt config is invalid (e.g., snake_case fields)
+ * @throws {FatalError} If the prompt file is not found or template rendering fails
  * @returns {Promise<string>} Generated text
  */
 export async function generateText( { prompt, variables } ) {
@@ -83,6 +85,8 @@ export async function generateText( { prompt, variables } ) {
  * @param {z.ZodObject} args.schema - Output schema
  * @param {string} [args.schemaName] - Output schema name
  * @param {string} [args.schemaDescription] - Output schema description
+ * @throws {ValidationError} If the prompt config is invalid (e.g., snake_case fields)
+ * @throws {FatalError} If the prompt file is not found or template rendering fails
  * @returns {Promise<object>} Object matching the provided schema
  */
 export async function generateObject( args ) {
@@ -112,6 +116,8 @@ export async function generateObject( args ) {
  * @param {z.ZodType} args.schema - Output schema (array item)
  * @param {string} [args.schemaName] - Output schema name
  * @param {string} [args.schemaDescription] - Output schema description
+ * @throws {ValidationError} If the prompt config is invalid (e.g., snake_case fields)
+ * @throws {FatalError} If the prompt file is not found or template rendering fails
  * @returns {Promise<object>} Array where each element matches the schema
  */
 export async function generateArray( args ) {
@@ -139,6 +145,8 @@ export async function generateArray( args ) {
  * @param {string} args.prompt - Prompt file name
  * @param {Record<string, string | number>} [args.variables] - Variables to interpolate
  * @param {string[]} args.enum - Allowed values for the generation
+ * @throws {ValidationError} If the prompt config is invalid (e.g., snake_case fields)
+ * @throws {FatalError} If the prompt file is not found or template rendering fails
  * @returns {Promise<string>} One of the provided enum values
  */
 export async function generateEnum( args ) {

package/src/index.d.ts

@@ -47,9 +47,6 @@ export type Prompt = {
     /** Maximum tokens in the response */
     maxTokens?: number;
 
-    /** Additional provider-specific options */
-    options?: Record<string, Record<string, JSONValue>>;
-
     /** Provider-specific configurations */
     providerOptions?: Record<string, unknown>;
   };

package/src/load_content.js

@@ -1,12 +1,13 @@
 import { join } from 'path';
 import { readdirSync, readFileSync } from 'node:fs';
 import { resolveInvocationDir } from '@output.ai/core/utils';
+import { FatalError } from '@output.ai/core';
 
 const scanDir = dir => {
   try {
     return readdirSync( dir, { withFileTypes: true } );
   } catch ( error ) {
-    throw new Error( `Error scanning directory "${dir}"`, { cause: error } );
+    throw new FatalError( `Error scanning directory "${dir}"`, { cause: error } );
   }
 };
 
@@ -14,7 +15,7 @@ const loadFile = path => {
   try {
     return readFileSync( path, 'utf-8' );
   } catch ( error ) {
-    throw new Error( `Error reading file "${path}"`, { cause: error } );
+    throw new FatalError( `Error reading file "${path}"`, { cause: error } );
   }
 };
 

package/src/parser.js

@@ -1,10 +1,11 @@
 import matter from 'gray-matter';
+import { FatalError } from '@output.ai/core';
 
 export function parsePrompt( raw ) {
-  const { data, content } = matter( raw );
+  const { data: config, content } = matter( raw );
 
   if ( !content || content.trim() === '' ) {
-    throw new Error( 'Prompt file has no content after frontmatter' );
+    throw new FatalError( 'Prompt file has no content after frontmatter' );
   }
 
   const infoExtractor = /<(system|user|assistant|tool)>([\s\S]*?)<\/\1>/gm;
@@ -16,12 +17,12 @@ export function parsePrompt( raw ) {
     const contentPreview = content.substring( 0, 200 );
     const ellipsis = content.length > 200 ? '...' : '';
 
-    throw new Error(
-      `No valid message blocks found in prompt file. 
-Expected format: <system>...</system>, <user>...</user>, etc. 
+    throw new FatalError(
+      `No valid message blocks found in prompt file.
+Expected format: <system>...</system>, <user>...</user>, etc.
 Content preview: ${contentPreview}${ellipsis}`
     );
   }
 
-  return { data, messages };
+  return { config, messages };
 }

package/src/parser.spec.js

@@ -13,7 +13,7 @@ model: claude-3-5-sonnet-20241022
 
     const result = parsePrompt( raw );
 
-    expect( result.data ).toEqual( {
+    expect( result.config ).toEqual( {
       provider: 'anthropic',
       model: 'claude-3-5-sonnet-20241022'
     } );
@@ -56,4 +56,67 @@ This is just plain text without any message tags.`;
       parsePrompt( raw );
     } ).toThrow( /Expected format/ );
   } );
+
+  it( 'should use providerOptions with budgetTokens in camelCase', () => {
+    // Frontmatter uses canonical format: providerOptions.thinking.budgetTokens
+    const raw = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+temperature: 0.7
+maxTokens: 64000
+providerOptions:
+  thinking:
+    type: enabled
+    budgetTokens: 1500
+---
+
+<user>Test</user>`;
+
+    const result = parsePrompt( raw );
+
+    expect( result.config.provider ).toBe( 'anthropic' );
+    expect( result.config.model ).toBe( 'claude-sonnet-4-20250514' );
+    expect( result.config.temperature ).toBe( 0.7 );
+    expect( result.config.maxTokens ).toBe( 64000 );
+
+    // Now expects canonical schema format in front-matter
+    expect( result.config.providerOptions ).toBeDefined();
+    expect( result.config.providerOptions.thinking ).toBeDefined();
+    expect( result.config.providerOptions.thinking.type ).toBe( 'enabled' );
+    expect( result.config.providerOptions.thinking.budgetTokens ).toBe( 1500 );
+  } );
+
+  it( 'should parse snake_case fields as-is (validation catches them later)', () => {
+    // Parser extracts fields as-is from frontmatter; validation happens later
+    const raw = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+temperature: 0.7
+max_tokens: 64000
+providerOptions:
+  thinking:
+    type: enabled
+    budget_tokens: 1500
+---
+
+<user>Test</user>`;
+
+    const result = parsePrompt( raw );
+
+    // Parser extracts snake_case fields as-is
+    expect( result.config.provider ).toBe( 'anthropic' );
+    expect( result.config.model ).toBe( 'claude-sonnet-4-20250514' );
+    expect( result.config.temperature ).toBe( 0.7 );
+    // snake_case preserved here because we're only calling parsePrompt, not validatePrompt
+    expect( result.config.max_tokens ).toBe( 64000 );
+    // camelCase not set because we only call parsePrompt
+    expect( result.config.maxTokens ).toBeUndefined();
+
+    // Snake_case fields in nested objects also preserved
+    expect( result.config.providerOptions ).toBeDefined();
+    expect( result.config.providerOptions.thinking ).toBeDefined();
+    expect( result.config.providerOptions.thinking.type ).toBe( 'enabled' );
+    expect( result.config.providerOptions.thinking.budget_tokens ).toBe( 1500 ); // snake_case preserved
+    expect( result.config.providerOptions.thinking.budgetTokens ).toBeUndefined(); // camelCase not set
+  } );
 } );

package/src/prompt_loader.js

@@ -2,30 +2,15 @@ import { parsePrompt } from './parser.js';
 import { Liquid } from 'liquidjs';
 import { loadContent } from './load_content.js';
 import { validatePrompt } from './prompt_validations.js';
+import { FatalError } from '@output.ai/core';
 
 const liquid = new Liquid();
 
-/**
- * Render a single message with template variables.
- *
- * @param {string} role - The message role
- * @param {string} content - The message content template
- * @param {Record<string, string | number>} values - Variables to interpolate
- * @param {string} promptName - Name of the prompt (for error messages)
- * @param {number} index - Message index (for error messages)
- * @returns {{role: string, content: string}} Rendered message object
- */
-const renderMessage = ( role, content, values, promptName, index ) => {
+const renderPrompt = ( name, content, values ) => {
   try {
-    return {
-      role,
-      content: liquid.parseAndRenderSync( content, values )
-    };
+    return liquid.parseAndRenderSync( content, values );
   } catch ( error ) {
-    throw new Error(
-      `Failed to render template in message ${index + 1} (role: ${role}) of prompt "${promptName}": ${error.message}`,
-      { cause: error }
-    );
+    throw new FatalError( `Failed to render template in prompt "${name}": ${error.message}`, { cause: error } );
   }
 };
 
@@ -36,20 +21,20 @@ const renderMessage = ( role, content, values, promptName, index ) => {
  * @param {Record<string, string | number>} [values] - Variables to interpolate
  * @returns {Prompt} Loaded and rendered prompt object
  */
-export const loadPrompt = ( name, values ) => {
+export const loadPrompt = ( name, values = {} ) => {
   const promptContent = loadContent( `${name}.prompt` );
   if ( !promptContent ) {
-    throw new Error( `Prompt ${name} not found.` );
+    throw new FatalError( `Prompt ${name} not found.` );
   }
 
-  const { data: config, messages } = parsePrompt( promptContent );
+  const renderedContent = renderPrompt( name, promptContent, values );
+
+  const { config, messages } = parsePrompt( renderedContent );
 
   const prompt = {
     name,
     config,
-    messages: messages.map( ( { role, content }, index ) =>
-      renderMessage( role, content, values, name, index )
-    )
+    messages
   };
 
   validatePrompt( prompt );

package/src/prompt_loader.spec.js

@@ -1,21 +1,16 @@
 import { describe, it, expect, vi, beforeEach } from 'vitest';
 import { loadPrompt } from './prompt_loader.js';
 
-// Mock dependencies
+// Mock dependencies that perform I/O or validation
 vi.mock( './load_content.js', () => ( {
   loadContent: vi.fn()
 } ) );
 
-vi.mock( './parser.js', () => ( {
-  parsePrompt: vi.fn()
-} ) );
-
 vi.mock( './prompt_validations.js', () => ( {
   validatePrompt: vi.fn()
 } ) );
 
 import { loadContent } from './load_content.js';
-import { parsePrompt } from './parser.js';
 import { validatePrompt } from './prompt_validations.js';
 
 describe( 'loadPrompt', () => {
@@ -31,10 +26,6 @@ model: claude-3-5-sonnet-20241022
 <user>Hello {{ name }}!</user>`;
 
     loadContent.mockReturnValue( promptContent );
-    parsePrompt.mockReturnValue( {
-      data: { provider: 'anthropic', model: 'claude-3-5-sonnet-20241022' },
-      messages: [ { role: 'user', content: 'Hello {{ name }}!' } ]
-    } );
 
     const result = loadPrompt( 'test', { name: 'World' } );
 
@@ -54,3 +45,105 @@ model: claude-3-5-sonnet-20241022
   } );
 
 } );
+
+describe( 'loadPrompt - template hydration in headers (integration tests)', () => {
+  beforeEach( () => {
+    vi.clearAllMocks();
+  } );
+
+  it( 'should hydrate template variables in YAML headers', () => {
+    const promptContent = `---
+provider: {{ provider_name }}
+model: {{ model_id }}
+temperature: 0.7
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    const result = loadPrompt( 'test', {
+      provider_name: 'anthropic',
+      model_id: 'claude-sonnet-4'
+    } );
+
+    expect( result.config.provider ).toBe( 'anthropic' );
+    expect( result.config.model ).toBe( 'claude-sonnet-4' );
+    expect( result.config.temperature ).toBe( 0.7 );
+  } );
+
+  it( 'should hydrate template variables in both headers and messages', () => {
+    const promptContent = `---
+provider: {{ provider }}
+model: {{ model }}
+---
+
+<user>Tell me about {{ topic }}</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    const result = loadPrompt( 'test', {
+      provider: 'openai',
+      model: 'gpt-4',
+      topic: 'testing'
+    } );
+
+    expect( result.config.provider ).toBe( 'openai' );
+    expect( result.config.model ).toBe( 'gpt-4' );
+    expect( result.messages[0].content ).toBe( 'Tell me about testing' );
+  } );
+
+  it( 'should render undefined template variables as null', () => {
+    const promptContent = `---
+provider: {{ undefined_var }}
+model: claude-3-5-sonnet-20241022
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    const result = loadPrompt( 'test', {} );
+
+    // Liquid renders undefined variables as empty, which becomes null in YAML
+    expect( result.config.provider ).toBe( null );
+    expect( result.config.model ).toBe( 'claude-3-5-sonnet-20241022' );
+  } );
+
+  it( 'should handle complex template expressions in headers', () => {
+    const promptContent = `---
+provider: anthropic
+model: {{ base_model }}-{{ version }}
+temperature: 0.7
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    const result = loadPrompt( 'test', {
+      base_model: 'claude-sonnet',
+      version: '4'
+    } );
+
+    expect( result.config.model ).toBe( 'claude-sonnet-4' );
+  } );
+
+  it( 'should use camelCase config keys', () => {
+    const promptContent = `---
+provider: anthropic
+model: claude-3-5-sonnet-20241022
+maxTokens: 1024
+temperature: 0.7
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    const result = loadPrompt( 'test', {} );
+
+    expect( result.config.maxTokens ).toBe( 1024 );
+  } );
+
+} );

package/src/prompt_loader_validation.spec.js

@@ -0,0 +1,174 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { loadPrompt } from './prompt_loader.js';
+
+vi.mock( './load_content.js', () => ( {
+  loadContent: vi.fn()
+} ) );
+
+import { loadContent } from './load_content.js';
+import { ValidationError } from '@output.ai/core';
+
+describe( 'loadPrompt - validation with real schema', () => {
+  beforeEach( () => {
+    vi.clearAllMocks();
+  } );
+
+  it( 'should accept valid camelCase format with providerOptions and budgetTokens', () => {
+    // Frontmatter uses canonical format: providerOptions.thinking.budgetTokens
+    const promptContent = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+temperature: 0.7
+maxTokens: 64000
+providerOptions:
+  thinking:
+    type: enabled
+    budgetTokens: 1500
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    const result = loadPrompt( 'test', {} );
+
+    // Validation passes with canonical schema format
+    expect( result.config.providerOptions ).toBeDefined();
+    expect( result.config.providerOptions.thinking.type ).toBe( 'enabled' );
+    expect( result.config.providerOptions.thinking.budgetTokens ).toBe( 1500 );
+  } );
+
+  it( 'should REJECT snake_case max_tokens (not canonical)', () => {
+    // Using snake_case max_tokens should fail validation
+    const promptContent = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+max_tokens: 64000
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    // loadPrompt should throw ValidationError because schema should reject max_tokens.
+    // Other tests check the actual message, but this one specifically checks that we throw
+    // the right kind of error, just to check that as well
+    expect( () => {
+      loadPrompt( 'test', {} );
+    } ).toThrow( ValidationError );
+    expect( () => {
+      loadPrompt( 'test', {} );
+    } ).toThrow( /max_tokens/ );
+  } );
+
+  it( 'should REJECT deprecated "options" field (use providerOptions)', () => {
+    // Using 'options' instead of 'providerOptions' should fail validation
+    const promptContent = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+options:
+  thinking:
+    type: enabled
+    budgetTokens: 1500
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    // loadPrompt should throw ValidationError because schema should reject 'options'
+    expect( () => {
+      loadPrompt( 'test', {} );
+    } ).toThrow( /Invalid prompt file/ );
+  } );
+
+  it( 'should REJECT snake_case budget_tokens in providerOptions', () => {
+    // Using snake_case budget_tokens should fail validation
+    const promptContent = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+providerOptions:
+  thinking:
+    type: enabled
+    budget_tokens: 1500
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    // loadPrompt should throw ValidationError because schema rejects budget_tokens
+    expect( () => {
+      loadPrompt( 'test', {} );
+    } ).toThrow( /Invalid prompt file/ );
+  } );
+
+  it( 'end-to-end: should accept canonical camelCase format', () => {
+    // End-to-end test: frontmatter uses canonical providerOptions.thinking.budgetTokens
+    const promptContent = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+temperature: 0.7
+maxTokens: 64000
+providerOptions:
+  thinking:
+    type: enabled
+    budgetTokens: 1500
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    const result = loadPrompt( 'test', {} );
+
+    // After loadPrompt, uses canonical format
+    expect( result.config.providerOptions ).toBeDefined();
+    expect( result.config.providerOptions.thinking.type ).toBe( 'enabled' );
+    expect( result.config.providerOptions.thinking.budgetTokens ).toBe( 1500 );
+  } );
+
+  it( 'end-to-end: should FAIL validation with snake_case budget_tokens', () => {
+    // End-to-end test: using snake_case should fail validation
+    const promptContent = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+providerOptions:
+  thinking:
+    type: enabled
+    budget_tokens: 1500
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    // loadPrompt should throw ValidationError because schema rejects budget_tokens
+    expect( () => {
+      loadPrompt( 'test', {} );
+    } ).toThrow( /Invalid prompt file/ );
+  } );
+
+  it( 'end-to-end: should FAIL validation with snake_case max_tokens', () => {
+    // End-to-end test: using snake_case should fail validation
+    const promptContent = `---
+provider: anthropic
+model: claude-sonnet-4-20250514
+max_tokens: 4000
+providerOptions:
+  thinking:
+    type: enabled
+    budget_tokens: 1500
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    // loadPrompt should throw ValidationError because schema rejects max_tokens
+    expect( () => {
+      loadPrompt( 'test', {} );
+    } ).toThrow( /Invalid prompt file/ );
+  } );
+
+} );

package/src/prompt_validations.js

@@ -11,20 +11,40 @@ export const promptSchema = z.object( {
       thinking: z.object( {
         type: z.literal( 'enabled' ),
         budgetTokens: z.number()
-      } ).optional()
-    } ).optional()
-  } ),
+      } ).strict().optional()
+    } ).strict().optional()
+  } ).strict(),
   messages: z.array(
     z.object( {
       role: z.string(),
       content: z.string()
-    } )
+    } ).strict()
   )
-} );
+} ).strict();
+
+const getHintForError = errorMessage => {
+  if ( errorMessage.includes( 'max_tokens' ) ) {
+    return '\nHint: Use "maxTokens" (camelCase) instead of "max_tokens".';
+  }
+  if ( errorMessage.includes( 'budget_tokens' ) ) {
+    return '\nHint: Use "budgetTokens" (camelCase) instead of "budget_tokens".';
+  }
+  if ( errorMessage.includes( '"options"' ) ) {
+    return '\nHint: Use "providerOptions" instead of "options".';
+  }
+  return '';
+};
 
 export function validatePrompt( prompt ) {
   const result = promptSchema.safeParse( prompt );
   if ( !result.success ) {
-    throw new ValidationError( `Invalid prompt file: ${z.prettifyError( result.error )}` );
+    const promptIdentifier = prompt?.name ? `"${prompt.name}"` : '(unnamed)';
+    const errorMessage = z.prettifyError( result.error );
+    const hint = getHintForError( errorMessage );
+
+    throw new ValidationError(
+      `Invalid prompt file ${promptIdentifier}: ${errorMessage}${hint}`,
+      { cause: result.error }
+    );
   }
 }

package/src/prompt_validations.spec.js

@@ -0,0 +1,121 @@
+import { describe, it, expect } from 'vitest';
+import { ValidationError } from '@output.ai/core';
+import { validatePrompt } from './prompt_validations.js';
+
+describe( 'validatePrompt', () => {
+  it( 'should validate a correct prompt with all required fields', () => {
+    const validPrompt = {
+      name: 'test-prompt',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-3-opus-20240229',
+        temperature: 0.7,
+        maxTokens: 1000
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Hello, world!'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( validPrompt ) ).not.toThrow();
+  } );
+
+  it( 'should validate a minimal prompt with only required fields', () => {
+    const minimalPrompt = {
+      name: 'minimal-prompt',
+      config: {
+        provider: 'openai',
+        model: 'gpt-4'
+      },
+      messages: [
+        {
+          role: 'system',
+          content: 'You are a helpful assistant.'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( minimalPrompt ) ).not.toThrow();
+  } );
+
+  it( 'should validate a prompt with thinking providerOptions', () => {
+    const promptWithThinking = {
+      name: 'thinking-prompt',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-3-5-sonnet-20241022',
+        providerOptions: {
+          thinking: {
+            type: 'enabled',
+            budgetTokens: 5000
+          }
+        }
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Solve this problem.'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( promptWithThinking ) ).not.toThrow();
+  } );
+
+  it( 'should throw ValidationError when provider is invalid', () => {
+    const invalidPrompt = {
+      name: 'invalid-provider',
+      config: {
+        provider: 'invalid-provider',
+        model: 'some-model'
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Test'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( invalidPrompt ) ).toThrow( ValidationError );
+  } );
+
+  it( 'should throw ValidationError when required fields are missing', () => {
+    const missingNamePrompt = {
+      config: {
+        provider: 'anthropic',
+        model: 'claude-3-opus-20240229'
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Test'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( missingNamePrompt ) ).toThrow( ValidationError );
+  } );
+
+  it( 'should throw ValidationError when max tokens is snake_case', () => {
+    const maxTokensSnakeCase = {
+      name: 'test-prompt',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-3-opus-20240229',
+        max_tokens: 4000
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Test'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( maxTokensSnakeCase ) ).toThrow( ValidationError );
+  } );
+} );