@outputai/llm 0.1.0

package/src/prompt_validations.spec.js

@@ -0,0 +1,346 @@
+import { describe, it, expect, vi } from 'vitest';
+import { ValidationError } from '@outputai/core';
+import { validatePrompt } from './prompt_validations.js';
+
+describe( 'validatePrompt', () => {
+  it( 'should validate a correct prompt with all required fields', () => {
+    const validPrompt = {
+      name: 'test-prompt',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-3-opus-20240229',
+        temperature: 0.7,
+        maxTokens: 1000
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Hello, world!'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( validPrompt ) ).not.toThrow();
+  } );
+
+  it( 'should validate a minimal prompt with only required fields', () => {
+    const minimalPrompt = {
+      name: 'minimal-prompt',
+      config: {
+        provider: 'openai',
+        model: 'gpt-4'
+      },
+      messages: [
+        {
+          role: 'system',
+          content: 'You are a helpful assistant.'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( minimalPrompt ) ).not.toThrow();
+  } );
+
+  it( 'should validate a prompt with thinking providerOptions', () => {
+    const promptWithThinking = {
+      name: 'thinking-prompt',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-3-5-sonnet-20241022',
+        providerOptions: {
+          thinking: {
+            type: 'enabled',
+            budgetTokens: 5000
+          }
+        }
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Solve this problem.'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( promptWithThinking ) ).not.toThrow();
+  } );
+
+  it( 'should validate a prompt with thinking type disabled', () => {
+    const promptWithThinkingDisabled = {
+      name: 'thinking-disabled-prompt',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-3-5-sonnet-20241022',
+        providerOptions: {
+          thinking: {
+            type: 'disabled'
+          }
+        }
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Simple task.'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( promptWithThinkingDisabled ) ).not.toThrow();
+  } );
+
+  it( 'should validate a prompt with thinking without budgetTokens', () => {
+    const promptWithThinkingNoBudget = {
+      name: 'thinking-no-budget',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-3-5-sonnet-20241022',
+        providerOptions: {
+          thinking: {
+            type: 'enabled'
+          }
+        }
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Think about this.'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( promptWithThinkingNoBudget ) ).not.toThrow();
+  } );
+
+  it( 'should validate a prompt with anthropic-specific providerOptions', () => {
+    const promptWithAnthropicOptions = {
+      name: 'anthropic-options-prompt',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-sonnet-4-20250514',
+        providerOptions: {
+          thinking: {
+            type: 'enabled',
+            budgetTokens: 5000
+          },
+          anthropic: {
+            effort: 'medium',
+            customOption: 'value'
+          }
+        }
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Solve this problem.'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( promptWithAnthropicOptions ) ).not.toThrow();
+  } );
+
+  it( 'should validate a prompt with openai-specific providerOptions', () => {
+    const promptWithOpenAIOptions = {
+      name: 'openai-options-prompt',
+      config: {
+        provider: 'openai',
+        model: 'o3-mini',
+        providerOptions: {
+          openai: {
+            reasoningEffort: 'high',
+            reasoningSummary: 'detailed',
+            customParameter: 'test'
+          }
+        }
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Analyze this data.'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( promptWithOpenAIOptions ) ).not.toThrow();
+  } );
+
+  it( 'should validate a prompt with azure-specific providerOptions', () => {
+    const promptWithAzureOptions = {
+      name: 'azure-options-prompt',
+      config: {
+        provider: 'azure',
+        model: 'gpt-4',
+        providerOptions: {
+          azure: {
+            deploymentName: 'my-deployment',
+            customConfig: { key: 'value' }
+          }
+        }
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Process this request.'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( promptWithAzureOptions ) ).not.toThrow();
+  } );
+
+  it( 'should validate a prompt with mixed providerOptions including unknown fields', () => {
+    const promptWithMixedOptions = {
+      name: 'mixed-options-prompt',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-3-opus-20240229',
+        providerOptions: {
+          thinking: {
+            type: 'enabled',
+            budgetTokens: 3000
+          },
+          anthropic: {
+            effort: 'high'
+          },
+          customProviderField: 'should-be-allowed',
+          anotherCustomField: {
+            nested: 'value',
+            array: [ 1, 2, 3 ]
+          }
+        }
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Complex request with multiple options.'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( promptWithMixedOptions ) ).not.toThrow();
+  } );
+
+  it( 'should accept custom provider names for dynamic providers', () => {
+    const customProviderPrompt = {
+      name: 'custom-provider-prompt',
+      config: {
+        provider: 'my-custom-provider',
+        model: 'custom-model-v1'
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Test'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( customProviderPrompt ) ).not.toThrow();
+  } );
+
+  it( 'should accept extra config fields via passthrough', () => {
+    const extraFieldsPrompt = {
+      name: 'extra-fields-prompt',
+      config: {
+        provider: 'openai',
+        model: 'gpt-4',
+        topP: 0.9,
+        seed: 42,
+        stopSequences: [ 'END' ]
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Test'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( extraFieldsPrompt ) ).not.toThrow();
+  } );
+
+  it( 'should throw ValidationError when provider is empty string', () => {
+    const emptyProviderPrompt = {
+      name: 'empty-provider',
+      config: {
+        provider: '',
+        model: 'some-model'
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Test'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( emptyProviderPrompt ) ).toThrow( ValidationError );
+  } );
+
+  it( 'should throw ValidationError when required fields are missing', () => {
+    const missingNamePrompt = {
+      config: {
+        provider: 'anthropic',
+        model: 'claude-3-opus-20240229'
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Test'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( missingNamePrompt ) ).toThrow( ValidationError );
+  } );
+
+  it( 'should pass through budget_tokens in thinking and warn about snake_case', () => {
+    const warnSpy = vi.spyOn( console, 'warn' ).mockImplementation( () => {} );
+
+    const promptWithBudgetTokensSnake = {
+      name: 'thinking-budget-snake',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-sonnet-4-20250514',
+        providerOptions: {
+          thinking: {
+            type: 'enabled',
+            budget_tokens: 10000
+          }
+        }
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Think hard.'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( promptWithBudgetTokensSnake ) ).not.toThrow();
+    expect( warnSpy ).toHaveBeenCalledWith(
+      '[output-llm] "budget_tokens" found in providerOptions.thinking. Did you mean "budgetTokens"?'
+    );
+
+    warnSpy.mockRestore();
+  } );
+
+  it( 'should allow snake_case fields in config via passthrough (no longer strict)', () => {
+    const maxTokensSnakeCase = {
+      name: 'test-prompt',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-3-opus-20240229',
+        max_tokens: 4000
+      },
+      messages: [
+        {
+          role: 'user',
+          content: 'Test'
+        }
+      ]
+    };
+
+    expect( () => validatePrompt( maxTokensSnakeCase ) ).not.toThrow();
+  } );
+} );

package/src/source_extraction.js

@@ -0,0 +1,49 @@
+import { createHash } from 'node:crypto';
+
+/**
+ * Checks whether a tool result looks like a search response (has a `results` array with `url` strings).
+ */
+const isSearchResult = result => !!result?.results?.[0]?.url;
+
+const toSource = ( { url, title } ) => ( {
+  type: 'source',
+  sourceType: 'url',
+  id: createHash( 'sha256' ).update( url ).digest( 'hex' ).slice( 0, 16 ),
+  url,
+  title: title ?? ''
+} );
+
+/**
+ * Extracts source URLs from search tool results embedded in AI SDK step data.
+ *
+ * Detects any tool result containing a `results[]` array whose items have a `url` string field.
+ * This covers perplexitySearch, tavilySearch, exaSearch, and any future tool with the same shape.
+ *
+ * Best-effort: returns empty array on any error rather than throwing.
+ *
+ * @param {Array} steps - AI SDK response steps (response.steps)
+ * @returns {Array<{ type: string, sourceType: string, id: string, url: string, title: string }>}
+ */
+export function extractSourcesFromSteps( steps ) {
+  try {
+    if ( !Array.isArray( steps ) || steps.length === 0 ) {
+      return [];
+    }
+
+    const seen = new Set();
+    return steps
+      .flatMap( step => Array.isArray( step.toolResults ) ? step.toolResults : [] )
+      .flatMap( toolResult => isSearchResult( toolResult.output ) ? toolResult.output.results : [] )
+      .filter( item => {
+        if ( !item.url || seen.has( item.url ) ) {
+          return false;
+        }
+        seen.add( item.url );
+        return true;
+      } )
+      .map( toSource );
+  } catch ( error ) {
+    console.warn( '[output-llm] source extraction failed, returning empty sources', error );
+    return [];
+  }
+}

package/src/source_extraction.spec.js

@@ -0,0 +1,169 @@
+import { describe, it, expect } from 'vitest';
+import { extractSourcesFromSteps } from './source_extraction.js';
+
+describe( 'extractSourcesFromSteps', () => {
+  it( 'returns empty array for undefined/null/empty steps', () => {
+    expect( extractSourcesFromSteps( undefined ) ).toEqual( [] );
+    expect( extractSourcesFromSteps( null ) ).toEqual( [] );
+    expect( extractSourcesFromSteps( [] ) ).toEqual( [] );
+  } );
+
+  it( 'skips steps with no toolResults', () => {
+    const steps = [ { text: 'hello' } ];
+    expect( extractSourcesFromSteps( steps ) ).toEqual( [] );
+  } );
+
+  it( 'skips non-search tool results', () => {
+    const steps = [ {
+      toolResults: [ {
+        output: { answer: 'some text', confidence: 0.9 }
+      } ]
+    } ];
+    expect( extractSourcesFromSteps( steps ) ).toEqual( [] );
+  } );
+
+  it( 'skips tool results with empty results array', () => {
+    const steps = [ {
+      toolResults: [ { output: { results: [] } } ]
+    } ];
+    expect( extractSourcesFromSteps( steps ) ).toEqual( [] );
+  } );
+
+  it( 'skips tool results where results items lack url', () => {
+    const steps = [ {
+      toolResults: [ {
+        output: { results: [ { title: 'no url', content: 'text' } ] }
+      } ]
+    } ];
+    expect( extractSourcesFromSteps( steps ) ).toEqual( [] );
+  } );
+
+  it( 'extracts from perplexity-shaped results', () => {
+    const steps = [ {
+      toolResults: [ {
+        output: {
+          id: 'pplx-123',
+          results: [
+            { url: 'https://example.com/1', title: 'Example 1', snippet: 'text' },
+            { url: 'https://example.com/2', title: 'Example 2', snippet: 'text' }
+          ]
+        }
+      } ]
+    } ];
+
+    const sources = extractSourcesFromSteps( steps );
+    expect( sources ).toHaveLength( 2 );
+    expect( sources[0] ).toEqual( {
+      type: 'source',
+      sourceType: 'url',
+      id: expect.any( String ),
+      url: 'https://example.com/1',
+      title: 'Example 1'
+    } );
+    expect( sources[1].url ).toBe( 'https://example.com/2' );
+  } );
+
+  it( 'extracts from tavily-shaped results', () => {
+    const steps = [ {
+      toolResults: [ {
+        output: {
+          query: 'test query',
+          results: [
+            { url: 'https://tavily.com/a', title: 'Tavily A', content: 'stuff', score: 0.95 }
+          ]
+        }
+      } ]
+    } ];
+
+    const sources = extractSourcesFromSteps( steps );
+    expect( sources ).toHaveLength( 1 );
+    expect( sources[0].url ).toBe( 'https://tavily.com/a' );
+    expect( sources[0].title ).toBe( 'Tavily A' );
+  } );
+
+  it( 'extracts from exa-shaped results', () => {
+    const steps = [ {
+      toolResults: [ {
+        output: {
+          results: [
+            { url: 'https://exa.ai/r', title: 'Exa Result', text: 'content', summary: 'summary' }
+          ]
+        }
+      } ]
+    } ];
+
+    const sources = extractSourcesFromSteps( steps );
+    expect( sources ).toHaveLength( 1 );
+    expect( sources[0].url ).toBe( 'https://exa.ai/r' );
+    expect( sources[0].title ).toBe( 'Exa Result' );
+  } );
+
+  it( 'deduplicates by URL across multiple steps', () => {
+    const steps = [
+      {
+        toolResults: [ {
+          output: { results: [ { url: 'https://dup.com', title: 'First' } ] }
+        } ]
+      },
+      {
+        toolResults: [ {
+          output: { results: [ { url: 'https://dup.com', title: 'Second' } ] }
+        } ]
+      }
+    ];
+
+    const sources = extractSourcesFromSteps( steps );
+    expect( sources ).toHaveLength( 1 );
+    expect( sources[0].title ).toBe( 'First' );
+  } );
+
+  it( 'deduplicates by URL within the same step', () => {
+    const steps = [ {
+      toolResults: [
+        { output: { results: [ { url: 'https://same.com', title: 'A' } ] } },
+        { output: { results: [ { url: 'https://same.com', title: 'B' } ] } }
+      ]
+    } ];
+
+    const sources = extractSourcesFromSteps( steps );
+    expect( sources ).toHaveLength( 1 );
+  } );
+
+  it( 'handles mixed search and non-search tool results', () => {
+    const steps = [ {
+      toolResults: [
+        { output: { calculation: 42 } },
+        { output: { results: [ { url: 'https://real.com', title: 'Real' } ] } },
+        { output: 'plain string' }
+      ]
+    } ];
+
+    const sources = extractSourcesFromSteps( steps );
+    expect( sources ).toHaveLength( 1 );
+    expect( sources[0].url ).toBe( 'https://real.com' );
+  } );
+
+  it( 'defaults title to empty string when missing', () => {
+    const steps = [ {
+      toolResults: [ {
+        output: { results: [ { url: 'https://notitle.com' } ] }
+      } ]
+    } ];
+
+    const sources = extractSourcesFromSteps( steps );
+    expect( sources[0].title ).toBe( '' );
+  } );
+
+  it( 'generates deterministic id from URL', () => {
+    const steps = [ {
+      toolResults: [ {
+        output: { results: [ { url: 'https://stable.com/path', title: 'S' } ] }
+      } ]
+    } ];
+
+    const s1 = extractSourcesFromSteps( steps );
+    const s2 = extractSourcesFromSteps( steps );
+    expect( s1[0].id ).toBe( s2[0].id );
+    expect( s1[0].id ).toHaveLength( 16 );
+  } );
+} );

package/src/validations.js

@@ -0,0 +1,21 @@
+import { ValidationError, z } from '@outputai/core';
+
+const generateTextArgsSchema = z.object( {
+  prompt: z.string(),
+  variables: z.any().optional()
+} );
+
+function validateSchema( schema, input, errorPrefix ) {
+  const result = schema.safeParse( input );
+  if ( !result.success ) {
+    throw new ValidationError( `${errorPrefix}: ${z.prettifyError( result.error )}` );
+  }
+}
+
+export function validateGenerateTextArgs( args ) {
+  validateSchema( generateTextArgsSchema, args, 'Invalid generateText() arguments' );
+}
+
+export function validateStreamTextArgs( args ) {
+  validateSchema( generateTextArgsSchema, args, 'Invalid streamText() arguments' );
+}