@output.ai/llm 0.2.1 → 0.2.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@output.ai/llm",
-  "version": "0.2.1",
+  "version": "0.2.3",
   "description": "Framework abstraction to interact with LLM models",
   "type": "module",
   "main": "src/index.js",

package/src/ai_sdk.js

@@ -1,13 +1,27 @@
 import { Tracing } from '@output.ai/core/tracing';
+import { z } from '@output.ai/core';
 import { loadModel } from './ai_model.js';
 import * as AI from 'ai';
 import { validateGenerateTextArgs, validateGenerateObjectArgs, validateGenerateArrayArgs, validateGenerateEnumArgs } from './validations.js';
 import { loadPrompt } from './prompt_loader.js';
 
-const traceWrapper = async ( traceId, fn ) => {
+/*
+  Word of wisdom:
+  We could retrieve the result object using the rest operator:
+  ```js
+  const { usage, providerMetadata, ...rest } = response;
+  const result = rest[resultProperty];
+  ```
+  But we CAN'T because the response of the generateText is an instance of `DefaultGenerateTextResult`
+  and 'text' is a getter (`get text()`).
+  Be aware of this when refactoring.
+*/
+const traceWrapper = async ( { traceId, resultProperty, fn } ) => {
   try {
-    const result = await fn();
-    Tracing.addEventEnd( { id: traceId, details: result } );
+    const response = await fn();
+    const { usage, providerMetadata } = response;
+    const result = response[resultProperty];
+    Tracing.addEventEnd( { id: traceId, details: { result, usage, providerMetadata } } );
     return result;
   } catch ( error ) {
     Tracing.addEventError( { id: traceId, details: error } );
@@ -52,11 +66,12 @@ const extraAiSdkOptionsFromPrompt = prompt => {
 export async function generateText( { prompt, variables } ) {
   validateGenerateTextArgs( { prompt, variables } );
   const loadedPrompt = loadPrompt( prompt, variables );
-  const traceId = startTrace( 'generateText', { prompt: loadedPrompt } );
+  const traceId = startTrace( 'generateText', { prompt, variables, loadedPrompt } );
 
-  return traceWrapper( traceId, async () =>
-    AI.generateText( extraAiSdkOptionsFromPrompt( loadedPrompt ) ).then( r => r.text )
-  );
+  return traceWrapper( {
+    traceId, resultProperty: 'text', fn: async () =>
+      AI.generateText( extraAiSdkOptionsFromPrompt( loadedPrompt ) )
+  } );
 }
 
 /**
@@ -74,17 +89,18 @@ export async function generateObject( args ) {
   validateGenerateObjectArgs( args );
   const { prompt, variables, schema, schemaName, schemaDescription } = args;
   const loadedPrompt = loadPrompt( prompt, variables );
-  const traceId = startTrace( 'generateObject', { ...args, prompt: loadedPrompt } );
+  const traceId = startTrace( 'generateObject', { ...args, schema: z.toJSONSchema( schema ), loadedPrompt } );
 
-  return traceWrapper( traceId, async () =>
-    AI.generateObject( {
-      output: 'object',
-      schema,
-      schemaName,
-      schemaDescription,
-      ...extraAiSdkOptionsFromPrompt( loadedPrompt )
-    } ).then( r => r.object )
-  );
+  return traceWrapper( {
+    traceId, resultProperty: 'object', fn: async () =>
+      AI.generateObject( {
+        output: 'object',
+        schema,
+        schemaName,
+        schemaDescription,
+        ...extraAiSdkOptionsFromPrompt( loadedPrompt )
+      } )
+  } );
 }
 
 /**
@@ -102,17 +118,18 @@ export async function generateArray( args ) {
   validateGenerateArrayArgs( args );
   const { prompt, variables, schema, schemaName, schemaDescription } = args;
   const loadedPrompt = loadPrompt( prompt, variables );
-  const traceId = startTrace( 'generateArray', { ...args, prompt: loadedPrompt } );
+  const traceId = startTrace( 'generateArray', { ...args, schema: z.toJSONSchema( schema ), loadedPrompt } );
 
-  return traceWrapper( traceId, async () =>
-    AI.generateObject( {
-      output: 'array',
-      schema,
-      schemaName,
-      schemaDescription,
-      ...extraAiSdkOptionsFromPrompt( loadedPrompt )
-    } ).then( r => r.object )
-  );
+  return traceWrapper( {
+    traceId, resultProperty: 'object', fn: async () =>
+      AI.generateObject( {
+        output: 'array',
+        schema,
+        schemaName,
+        schemaDescription,
+        ...extraAiSdkOptionsFromPrompt( loadedPrompt )
+      } )
+  } );
 }
 
 /**
@@ -128,13 +145,14 @@ export async function generateEnum( args ) {
   validateGenerateEnumArgs( args );
   const { prompt, variables, enum: _enum } = args;
   const loadedPrompt = loadPrompt( prompt, variables );
-  const traceId = startTrace( 'generateEnum', { ...args, prompt: loadedPrompt } );
+  const traceId = startTrace( 'generateEnum', { ...args, loadedPrompt } );
 
-  return traceWrapper( traceId, async () =>
-    AI.generateObject( {
-      output: 'enum',
-      enum: _enum,
-      ...extraAiSdkOptionsFromPrompt( loadedPrompt )
-    } ).then( r => r.object )
-  );
+  return traceWrapper( {
+    traceId, resultProperty: 'object', fn: async () =>
+      AI.generateObject( {
+        output: 'enum',
+        enum: _enum,
+        ...extraAiSdkOptionsFromPrompt( loadedPrompt )
+      } )
+  } );
 }

package/src/load_content.js

@@ -1,121 +1,43 @@
-import { join, dirname } from 'path';
-import { readFileSync, readdirSync } from 'node:fs';
-import { fileURLToPath } from 'node:url';
+import { join } from 'path';
+import { readdirSync, readFileSync } from 'node:fs';
+import { resolveInvocationDir } from '@output.ai/core/utils';
+import { FatalError } from '@output.ai/core';
 
-function extractFilePath( fileUrlMatch, parenMatch, directMatch ) {
-  if ( fileUrlMatch ) {
-    return fileURLToPath( 'file://' + fileUrlMatch[1] );
-  }
-  if ( parenMatch ) {
-    return parenMatch[1];
-  }
-  if ( directMatch ) {
-    return directMatch[1];
-  }
-  return null;
-}
-
-export const getInvocationDir = _ => {
-  const stack = new Error().stack;
-
-  if ( !stack ) {
-    throw new Error( 'Stack trace is unavailable - cannot determine invocation directory' );
+const scanDir = dir => {
+  try {
+    return readdirSync( dir, { withFileTypes: true } );
+  } catch ( error ) {
+    throw new FatalError( `Error scanning directory "${dir}"`, { cause: error } );
   }
+};
 
-  const lines = stack.split( '\n' );
-
-  // Search through stack to find first valid file path that is NOT in SDK
-  // Skip first 2 lines (Error message and getInvocationDir itself)
-  for ( const line of lines.slice( 2 ) ) {
-    // Match file:// URLs (ESM) or regular file paths
-    // Pattern 1: file:///path/to/file.js:line:col
-    // Pattern 2: at func (/path/to/file.js:line:col)
-    // Pattern 3: /path/to/file.js:line:col
-    const fileUrlMatch = line.match( /file:\/\/([^:]+\.(?:js|mjs|ts|tsx|jsx))/ );
-    const parenMatch = line.match( /\(([^:)]+\.(?:js|mjs|ts|tsx|jsx)):\d+:\d+\)/ );
-    const directMatch = line.match( /^\s*at\s+[^(]*([^:]+\.(?:js|mjs|ts|tsx|jsx)):\d+:\d+/ );
-
-    // Determine the file path from different stack trace formats
-    const filePath = extractFilePath( fileUrlMatch, parenMatch, directMatch );
-
-    if ( filePath ) {
-      // Skip internal Node.js paths
-      if ( filePath.includes( 'node:' ) || filePath.includes( 'internal/' ) ) {
-        continue;
-      }
-
-      // Skip SDK internal files - we want the actual caller, not the SDK
-      // This includes @output.ai packages and sdk/ directory
-      if ( filePath.includes( '/sdk/' ) || filePath.includes( 'node_modules/@output.ai/' ) ) {
-        continue;
-      }
-
-      // Found a valid caller file outside the SDK
-      return dirname( filePath );
-    }
+const loadFile = path => {
+  try {
+    return readFileSync( path, 'utf-8' );
+  } catch ( error ) {
+    throw new FatalError( `Error reading file "${path}"`, { cause: error } );
   }
-
-  throw new Error(
-    'Unable to determine invocation directory from stack trace. ' +
-    `Stack preview:\n${lines.slice( 0, 10 ).join( '\n' )}`
-  );
 };
 
 /**
  * Recursively search for a file by its name and load its content.
  *
  * @param {string} name - Name of the file load its content
- * @param {string} [dir=<invocation directory>] - The directory to search for the file
- * @param {number} [depth=0] - Current recursion depth
- * @param {number} [maxDepth=5] - Maximum recursion depth
+ * @param {string} [dir] - The directory to search for the file, defaults to invocation directory
  * @returns {string | null} - File content or null if not found
  */
-export const loadContent = ( name, dir = getInvocationDir(), depth = 0, maxDepth = 5 ) => {
-  // Stop recursion if max depth exceeded
-  if ( depth > maxDepth ) {
-    return null;
-  }
-
-  // Validate filename doesn't contain path separators (prevent directory traversal)
-  if ( name.includes( '..' ) || name.includes( '/' ) || name.includes( '\\' ) ) {
-    throw new Error( `Invalid file name "${name}" - must not contain path separators or ".."` );
-  }
-
-  try {
-    const entries = readdirSync( dir, { withFileTypes: true } );
-
-    for ( const entry of entries ) {
-      if ( entry.name === name ) {
-        try {
-          return readFileSync( join( dir, entry.name ), 'utf-8' );
-        } catch ( error ) {
-          throw new Error(
-            `Found file "${name}" in "${dir}" but failed to read it: ${error.message}`,
-            { cause: error }
-          );
-        }
-      }
-
-      // Recurse into subdirectories, but skip symlinks to prevent infinite loops
-      if ( entry.isDirectory() && !entry.isSymbolicLink() ) {
-        const content = loadContent( name, join( dir, entry.name ), depth + 1, maxDepth );
-        if ( content !== null ) {
-          return content;
-        }
-      }
+export const loadContent = ( name, dir = resolveInvocationDir() ) => {
+  for ( const entry of scanDir( dir ) ) {
+    if ( entry.name === name ) {
+      return loadFile( join( dir, entry.name ) );
     }
 
-    return null;
-  } catch ( error ) {
-    // Only suppress ENOENT (directory doesn't exist) during recursion
-    // This is expected when searching through nested directories
-    if ( error.code === 'ENOENT' ) {
-      return null;
+    if ( entry.isDirectory() && !entry.isSymbolicLink() ) {
+      const content = loadContent( name, join( dir, entry.name ) );
+      if ( content ) {
+        return content;
+      }
     }
-    // Propagate all other errors (permission denied, I/O errors, etc.)
-    throw new Error(
-      `Failed to read directory "${dir}" while searching for "${name}": ${error.message}`,
-      { cause: error }
-    );
   }
+  return null;
 };

package/src/load_content.spec.js

@@ -1,97 +1,81 @@
-import { describe, it, expect } from 'vitest';
-import { getInvocationDir, loadContent } from './load_content.js';
+import { describe, it, expect, vi, beforeEach } from 'vitest';
 import { join } from 'path';
 import { mkdtempSync, writeFileSync, mkdirSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 
-describe( 'getInvocationDir', () => {
-  it( 'extracts directory from stack trace', () => {
-    const dir = getInvocationDir();
-    expect( dir ).toBeTruthy();
-    expect( typeof dir ).toBe( 'string' );
-    // Should not contain filename, only directory
-    expect( dir ).not.toContain( 'load_content.spec.js' );
-  } );
+// Hoisted state so mocks can read dynamic values set in tests
+const state = vi.hoisted( () => ( { dir: '', entries: {} } ) );
+
+// Mock core utils to control resolveInvocationDir
+vi.mock( '@output.ai/core/utils', () => ( {
+  resolveInvocationDir: () => state.dir
+} ) );
+
+// Mock node:fs.readFileSync for directory scans while delegating file reads
+vi.mock( 'node:fs', async importOriginal => {
+  const actual = await importOriginal();
+  return {
+    ...actual,
+    readFileSync: vi.fn( ( path, options ) => {
+      if ( options && typeof options === 'object' && options.withFileTypes ) {
+        return state.entries[path] || [];
+      }
+      return actual.readFileSync( path, options );
+    } )
+  };
+} );
 
-  it( 'returns consistent directory when called multiple times', () => {
-    const dir1 = getInvocationDir();
-    const dir2 = getInvocationDir();
-    expect( dir1 ).toBe( dir2 );
-  } );
+const dirEntry = ( name, { isDir = false, isLink = false } = {} ) => ( {
+  name,
+  isDirectory: () => isDir,
+  isSymbolicLink: () => isLink
+} );
 
-  it( 'returns a valid filesystem path', () => {
-    const dir = getInvocationDir();
-    // Should be a non-empty string that looks like a directory path
-    expect( dir ).toBeTruthy();
-    expect( typeof dir ).toBe( 'string' );
-    expect( dir.length ).toBeGreaterThan( 0 );
-  } );
+beforeEach( () => {
+  state.dir = '';
+  state.entries = {};
 } );
 
 describe( 'loadContent', () => {
-  it( 'loads file from root directory', () => {
+  it( 'loads file from root directory using mocked resolveInvocationDir', async () => {
     const tempDir = mkdtempSync( join( tmpdir(), 'load-content-test-' ) );
+    state.dir = tempDir;
     const testContent = 'test file content';
     writeFileSync( join( tempDir, 'test.txt' ), testContent );
+    state.entries[tempDir] = [ dirEntry( 'test.txt' ) ];
 
-    const content = loadContent( 'test.txt', tempDir );
+    const { loadContent } = await import( './load_content.js' );
+    const content = loadContent( 'test.txt' );
 
     expect( content ).toBe( testContent );
   } );
 
-  it( 'loads file from nested subdirectory', () => {
+  it( 'loads file from nested subdirectory via recursion', async () => {
     const tempDir = mkdtempSync( join( tmpdir(), 'load-content-test-' ) );
     const subDir = join( tempDir, 'subdir' );
     mkdirSync( subDir );
+    state.dir = tempDir;
 
     const testContent = 'nested file content';
     writeFileSync( join( subDir, 'nested.txt' ), testContent );
 
-    const content = loadContent( 'nested.txt', tempDir );
+    state.entries[tempDir] = [ dirEntry( 'subdir', { isDir: true } ) ];
+    state.entries[subDir] = [ dirEntry( 'nested.txt' ) ];
+
+    const { loadContent } = await import( './load_content.js' );
+    const content = loadContent( 'nested.txt' );
 
     expect( content ).toBe( testContent );
   } );
 
-  it( 'returns null when file does not exist', () => {
+  it( 'returns null when file does not exist', async () => {
     const tempDir = mkdtempSync( join( tmpdir(), 'load-content-test-' ) );
+    state.dir = tempDir;
+    state.entries[tempDir] = [ dirEntry( 'other.txt' ) ];
 
-    const content = loadContent( 'nonexistent.txt', tempDir );
+    const { loadContent } = await import( './load_content.js' );
+    const content = loadContent( 'nonexistent.txt' );
 
     expect( content ).toBeNull();
   } );
-
-  it( 'returns null when starting directory does not exist', () => {
-    // ENOENT is suppressed and returns null (expected behavior)
-    const content = loadContent( 'test.txt', '/nonexistent-root-path-12345' );
-    expect( content ).toBeNull();
-  } );
-
-  it( 'respects depth limit', () => {
-    const tempDir = mkdtempSync( join( tmpdir(), 'load-content-test-' ) );
-
-    // Create deeply nested structure: dir/1/2/3/4/5/6/deep.txt (7 levels)
-    const levels = [ 1, 2, 3, 4, 5, 6 ];
-    const deepPath = levels.reduce( ( acc, level ) => {
-      const newPath = join( acc, `level${level}` );
-      mkdirSync( newPath );
-      return newPath;
-    }, tempDir );
-    writeFileSync( join( deepPath, 'deep.txt' ), 'deeply nested' );
-
-    // Should NOT find it with default maxDepth of 5
-    const content = loadContent( 'deep.txt', tempDir );
-    expect( content ).toBeNull();
-  } );
-
-  it( 'throws error when filename contains path separators', () => {
-    const tempDir = mkdtempSync( join( tmpdir(), 'load-content-test-' ) );
-
-    expect( () => {
-      loadContent( '../test.txt', tempDir );
-    } ).toThrow( /Invalid file name/ );
-
-    expect( () => {
-      loadContent( 'foo/bar.txt', tempDir );
-    } ).toThrow( /Invalid file name/ );
-  } );
 } );

package/src/parser.js

@@ -1,10 +1,11 @@
 import matter from 'gray-matter';
+import { FatalError } from '@output.ai/core';
 
 export function parsePrompt( raw ) {
-  const { data, content } = matter( raw );
+  const { data: config, content } = matter( raw );
 
   if ( !content || content.trim() === '' ) {
-    throw new Error( 'Prompt file has no content after frontmatter' );
+    throw new FatalError( 'Prompt file has no content after frontmatter' );
   }
 
   const infoExtractor = /<(system|user|assistant|tool)>([\s\S]*?)<\/\1>/gm;
@@ -16,12 +17,12 @@ export function parsePrompt( raw ) {
     const contentPreview = content.substring( 0, 200 );
     const ellipsis = content.length > 200 ? '...' : '';
 
-    throw new Error(
-      `No valid message blocks found in prompt file. 
-Expected format: <system>...</system>, <user>...</user>, etc. 
+    throw new FatalError(
+      `No valid message blocks found in prompt file.
+Expected format: <system>...</system>, <user>...</user>, etc.
 Content preview: ${contentPreview}${ellipsis}`
     );
   }
 
-  return { data, messages };
+  return { config, messages };
 }

package/src/parser.spec.js

@@ -13,7 +13,7 @@ model: claude-3-5-sonnet-20241022
 
     const result = parsePrompt( raw );
 
-    expect( result.data ).toEqual( {
+    expect( result.config ).toEqual( {
       provider: 'anthropic',
       model: 'claude-3-5-sonnet-20241022'
     } );

package/src/prompt_loader.js

@@ -2,30 +2,15 @@ import { parsePrompt } from './parser.js';
 import { Liquid } from 'liquidjs';
 import { loadContent } from './load_content.js';
 import { validatePrompt } from './prompt_validations.js';
+import { FatalError } from '@output.ai/core';
 
 const liquid = new Liquid();
 
-/**
- * Render a single message with template variables.
- *
- * @param {string} role - The message role
- * @param {string} content - The message content template
- * @param {Record<string, string | number>} values - Variables to interpolate
- * @param {string} promptName - Name of the prompt (for error messages)
- * @param {number} index - Message index (for error messages)
- * @returns {{role: string, content: string}} Rendered message object
- */
-const renderMessage = ( role, content, values, promptName, index ) => {
+const renderPrompt = ( name, content, values ) => {
   try {
-    return {
-      role,
-      content: liquid.parseAndRenderSync( content, values )
-    };
+    return liquid.parseAndRenderSync( content, values );
   } catch ( error ) {
-    throw new Error(
-      `Failed to render template in message ${index + 1} (role: ${role}) of prompt "${promptName}": ${error.message}`,
-      { cause: error }
-    );
+    throw new FatalError( `Failed to render template in prompt "${name}": ${error.message}`, { cause: error } );
   }
 };
 
@@ -36,20 +21,20 @@ const renderMessage = ( role, content, values, promptName, index ) => {
  * @param {Record<string, string | number>} [values] - Variables to interpolate
  * @returns {Prompt} Loaded and rendered prompt object
  */
-export const loadPrompt = ( name, values ) => {
+export const loadPrompt = ( name, values = {} ) => {
   const promptContent = loadContent( `${name}.prompt` );
   if ( !promptContent ) {
-    throw new Error( `Prompt ${name} not found.` );
+    throw new FatalError( `Prompt ${name} not found.` );
   }
 
-  const { data: config, messages } = parsePrompt( promptContent );
+  const renderedContent = renderPrompt( name, promptContent, values );
+
+  const { config, messages } = parsePrompt( renderedContent );
 
   const prompt = {
     name,
     config,
-    messages: messages.map( ( { role, content }, index ) =>
-      renderMessage( role, content, values, name, index )
-    )
+    messages
   };
 
   validatePrompt( prompt );

package/src/prompt_loader.spec.js

@@ -1,21 +1,16 @@
 import { describe, it, expect, vi, beforeEach } from 'vitest';
 import { loadPrompt } from './prompt_loader.js';
 
-// Mock dependencies
+// Mock dependencies that perform I/O or validation
 vi.mock( './load_content.js', () => ( {
   loadContent: vi.fn()
 } ) );
 
-vi.mock( './parser.js', () => ( {
-  parsePrompt: vi.fn()
-} ) );
-
 vi.mock( './prompt_validations.js', () => ( {
   validatePrompt: vi.fn()
 } ) );
 
 import { loadContent } from './load_content.js';
-import { parsePrompt } from './parser.js';
 import { validatePrompt } from './prompt_validations.js';
 
 describe( 'loadPrompt', () => {
@@ -31,10 +26,6 @@ model: claude-3-5-sonnet-20241022
 <user>Hello {{ name }}!</user>`;
 
     loadContent.mockReturnValue( promptContent );
-    parsePrompt.mockReturnValue( {
-      data: { provider: 'anthropic', model: 'claude-3-5-sonnet-20241022' },
-      messages: [ { role: 'user', content: 'Hello {{ name }}!' } ]
-    } );
 
     const result = loadPrompt( 'test', { name: 'World' } );
 
@@ -54,3 +45,88 @@ model: claude-3-5-sonnet-20241022
   } );
 
 } );
+
+describe( 'loadPrompt - template hydration in headers (integration tests)', () => {
+  beforeEach( () => {
+    vi.clearAllMocks();
+  } );
+
+  it( 'should hydrate template variables in YAML headers', () => {
+    const promptContent = `---
+provider: {{ provider_name }}
+model: {{ model_id }}
+temperature: 0.7
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    const result = loadPrompt( 'test', {
+      provider_name: 'anthropic',
+      model_id: 'claude-sonnet-4'
+    } );
+
+    expect( result.config.provider ).toBe( 'anthropic' );
+    expect( result.config.model ).toBe( 'claude-sonnet-4' );
+    expect( result.config.temperature ).toBe( 0.7 );
+  } );
+
+  it( 'should hydrate template variables in both headers and messages', () => {
+    const promptContent = `---
+provider: {{ provider }}
+model: {{ model }}
+---
+
+<user>Tell me about {{ topic }}</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    const result = loadPrompt( 'test', {
+      provider: 'openai',
+      model: 'gpt-4',
+      topic: 'testing'
+    } );
+
+    expect( result.config.provider ).toBe( 'openai' );
+    expect( result.config.model ).toBe( 'gpt-4' );
+    expect( result.messages[0].content ).toBe( 'Tell me about testing' );
+  } );
+
+  it( 'should render undefined template variables as null', () => {
+    const promptContent = `---
+provider: {{ undefined_var }}
+model: claude-3-5-sonnet-20241022
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    const result = loadPrompt( 'test', {} );
+
+    // Liquid renders undefined variables as empty, which becomes null in YAML
+    expect( result.config.provider ).toBe( null );
+    expect( result.config.model ).toBe( 'claude-3-5-sonnet-20241022' );
+  } );
+
+  it( 'should handle complex template expressions in headers', () => {
+    const promptContent = `---
+provider: anthropic
+model: {{ base_model }}-{{ version }}
+temperature: 0.7
+---
+
+<user>Hello</user>`;
+
+    loadContent.mockReturnValue( promptContent );
+
+    const result = loadPrompt( 'test', {
+      base_model: 'claude-sonnet',
+      version: '4'
+    } );
+
+    expect( result.config.model ).toBe( 'claude-sonnet-4' );
+  } );
+
+} );