@outputai/core 0.15.1-next.4a437bf.0 → 0.15.1-next.8ef04e2.0

package/src/llm/utils/skills.js

@@ -0,0 +1,74 @@
+import { existsSync, lstatSync, readFileSync, readdirSync } from 'node:fs';
+import { resolve, basename } from 'node:path';
+import matter from 'gray-matter';
+import { ValidationError } from '#errors';
+
+/**
+ * @typedef Skill
+ * @property {string} name
+ * @property {string} description
+ * @property {string} instructions
+ */
+
+/**
+ * Recursively load skill paths. Explicit files may use any extension; directories load `.md` files only.
+ * Frontmatter may provide `name` and `description`; body becomes the instructions.
+ *
+ * @param {string[]} paths
+ * @param {boolean} markdownFilesOnly
+ * @returns {Skills[]}
+ */
+const recursiveLoadSkillPaths = ( paths, markdownFilesOnly ) => {
+  const loaded = [];
+  for ( const path of paths ) {
+    if ( !existsSync( path ) ) {
+      throw new ValidationError( `Skill path "${path}" not found` );
+    }
+    const stats = lstatSync( path );
+    if ( stats.isSymbolicLink() ) {
+      continue;
+    }
+    if ( stats.isDirectory() ) {
+      loaded.push( ...recursiveLoadSkillPaths(
+        readdirSync( path ).sort().map( f => resolve( path, f ) ),
+        true
+      ) );
+      continue;
+    }
+    if ( markdownFilesOnly && !path.endsWith( '.md' ) ) {
+      continue;
+    }
+
+    const raw = readFileSync( path, 'utf-8' );
+    const { data, content } = matter( raw );
+    loaded.push( {
+      name: data.name ?? basename( path, '.md' ),
+      description: data.description ?? basename( path, '.md' ),
+      instructions: content.trim()
+    } );
+  }
+  return loaded;
+};
+
+/**
+ * Recursively load skill files without following symbolic links.
+ *
+ * @param {string[]} paths
+ * @returns {Skills[]}
+ */
+export const recursiveLoadSkillFile = paths => recursiveLoadSkillPaths( paths, false );
+
+/**
+ * Resolve all skills available to a prompt.
+ *
+ * @param {object} prompt - Loaded prompt object
+ * @returns {{ name: string, description: string, instructions: string }[]}
+ */
+export const loadSkills = prompt => {
+  const baseDir = prompt.fileDir;
+
+  // skills referenced in the prompt config
+  const promptSkillPaths = prompt.config.skills.map( f => resolve( baseDir, f ) );
+
+  return recursiveLoadSkillFile( promptSkillPaths );
+};

package/src/llm/utils/sources.js

@@ -0,0 +1,73 @@
+import { createHash } from 'node:crypto';
+import { Logger } from '#runtime_logger';
+
+/** Builds the final source shape */
+const buildSource = ( { url, title } ) => {
+  const trimmedUrl = url.trim();
+  return {
+    type: 'source',
+    sourceType: 'url',
+    id: createHash( 'sha256' ).update( trimmedUrl ).digest( 'hex' ).slice( 0, 16 ),
+    url: trimmedUrl,
+    title: title ?? ''
+  };
+};
+
+/** Return value it is array, otherwise return [] */
+const asArray = v => Array.isArray( v ) ? v : [];
+
+const isNonBlankUrl = url => typeof url === 'string' && url.trim().length > 0;
+
+/**
+ * Extracts source from search tool results embedded in AI SDK step data.
+ *
+ * Detects any tool result containing a `results[]` array whose items have a `url` string field.
+ * This covers perplexitySearch, tavilySearch, exaSearch, and any future tool with the same shape.
+ * Deduplicate key is "url"
+ *
+ * @param {Array} steps - AI SDK response steps (response.steps)
+ * @returns {Array<{ key: string, source: object }>}
+ */
+const extractSourcesFromSteps = steps =>
+  asArray( steps )
+    .flatMap( step => asArray( step?.toolResults ) )
+    .flatMap( toolResult => asArray( toolResult?.output?.results ) )
+    .filter( item => isNonBlankUrl( item?.url ) )
+    .map( item => {
+      const source = buildSource( item );
+      return { key: source.url, source };
+    } );
+
+/**
+ * Extract only the valid sources from the response (has url or id).
+ * Deduplicate key is `url` when set, otherwise `id` (document sources have `id` and no `url`).
+ *
+ * @param {Array} sources
+ * @returns {Array<{ key: string, source: object }>}
+ */
+const extractValidSourcesFromResponse = sources =>
+  asArray( sources )
+    .filter( item => item !== null && typeof item === 'object' && ( isNonBlankUrl( item.url ) || typeof item.id === 'string' ) )
+    .map( item => ( { key: isNonBlankUrl( item.url ) ? item.url.trim() : item.id, source: item } ) );
+
+/** Deduplicate wrapped sources by their key */
+const deduplicateSources = sources => new Map( sources.map( item => [ item.key, item.source ] ) ).values().toArray();
+
+/**
+ * Extract sources from tools usage and final response, deduplicate, and return.
+ * Response sources are preferred over tools when the key matches.
+ *
+ * @param {object} response AI SDK response
+ * @returns {object[]} Merged sources
+ */
+export const extractSources = response => {
+  try {
+    const { steps, sources: sourcesFromResponse } = response ?? {};
+    const sourcesFromTools = extractSourcesFromSteps( asArray( steps ) );
+    const validSourcesFromResponse = extractValidSourcesFromResponse( sourcesFromResponse );
+    return deduplicateSources( sourcesFromTools.concat( validSourcesFromResponse ) );
+  } catch ( error ) {
+    Logger.error( 'Sources extraction failed', { namespace: 'LLM', error: error?.message || String( error ) } );
+    return [];
+  }
+};

package/src/llm/utils/stream.js

@@ -0,0 +1,75 @@
+import { setTimeout as delay } from 'node:timers/promises';
+
+/** How long a failed stream may go quiet before the drain gives up on its remaining parts */
+const FAILURE_DRAIN_TIMEOUT_MS = 250;
+
+const TIMED_OUT = Symbol( 'drain-timed-out' );
+
+/** Extracts the error an abort or error part should surface, or null for every other part */
+const extractError = ( part, abortSignal ) => {
+  if ( part?.type === 'abort' ) {
+    const reason = abortSignal?.reason;
+    return reason instanceof Error ?
+      reason :
+      new Error( part.reason ?? 'Streaming aborted.', { cause: reason } );
+  }
+
+  if ( part?.type === 'error' ) {
+    return part.error instanceof Error ?
+      part.error :
+      new Error( part.error ? String( part.error ) : 'Streaming failed.', { cause: part.error } );
+  }
+
+  if ( abortSignal?.aborted ) {
+    return abortSignal.reason instanceof Error ?
+      abortSignal.reason :
+      new Error( 'Streaming aborted.', { cause: abortSignal.reason } );
+  }
+
+  return null;
+};
+
+/** Reads the next part, resolving with `TIMED_OUT` when the stream goes quiet for longer than `timeoutMs` */
+const nextWithin = async ( iterator, timeoutMs ) =>
+  Promise.race( [ iterator.next(), delay( timeoutMs, TIMED_OUT, { ref: false } ) ] );
+
+/**
+ * Consumes a streaming result until it completes, then throws the first abort or error part it saw.
+ * Callers must not throw from the AI SDK `onError` callback; that errors the stream before these parts are delivered.
+ *
+ * The draining continues past a failure on purpose: the AI SDK fires its lifecycle callbacks as parts flow through the
+ * stream, so `onStepEnd` and `onEnd` - and with them the usage of everything already spent - only arrive if something
+ * keeps reading. A stalled provider can leave the stream open forever, so once a failure is captured every read waits
+ * at most `FAILURE_DRAIN_TIMEOUT_MS` for the next part, and the drain gives up and throws anyway when none arrives.
+ * A healthy stream ends immediately after a failure, so the timeout is not part of the normal path; a successful
+ * stream is never timed out.
+ *
+ * @param {object} stream - AI SDK stream result with `stream`
+ * @param {AbortSignal} [abortSignal] - Used to recover the original abort reason
+ */
+export const drainStream = async ( stream, abortSignal ) => {
+  const iterator = stream.stream[Symbol.asyncIterator]();
+  const state = { error: null };
+
+  while ( true ) {
+    const hasDeadline = state.error || abortSignal?.aborted;
+    const result = await ( hasDeadline ? nextWithin( iterator, FAILURE_DRAIN_TIMEOUT_MS ) : iterator.next() );
+
+    if ( result.done ) {
+      break;
+    }
+
+    if ( result?.value || abortSignal?.aborted ) {
+      state.error ??= extractError( result.value, abortSignal );
+    }
+
+    if ( result === TIMED_OUT ) {
+      iterator.return?.()?.catch( () => {} );
+      break;
+    }
+  }
+
+  if ( state.error ) {
+    throw state.error;
+  }
+};

package/src/llm/utils/tools.js

@@ -0,0 +1,48 @@
+import { tool as createTool } from 'ai';
+import { z } from 'zod';
+import { ValidationError } from '#errors';
+import { getProvider } from '../ai_provider.js';
+
+/**
+ * Build the `load_skill` AI SDK tool that the LLM calls to retrieve full skill instructions.
+ *
+ * @param {Skill[]} skills
+ * @returns {import('ai').Tool}
+ */
+export const buildLoadSkillTool = skills => createTool( {
+  description: 'Get detailed instructions for a named skill',
+  inputSchema: z.object( { name: z.string().describe( 'Name of the skill to load' ) } ),
+  execute: ( { name } ) => {
+    const sk = skills.find( s => s.name === name );
+    if ( !sk ) {
+      return `Skill "${name}" not found. Available: ${skills.map( s => s.name ).join( ', ' )}`;
+    }
+    return sk.instructions;
+  }
+} );
+
+const buildProviderError = ( prompt, suffix ) => `Provider "${prompt.config.provider}" used by prompt "${prompt.name}" does not support ${suffix}.`;
+
+/** Load tools from a prompt provider. */
+export const loadPromptTools = prompt => {
+  const promptToolNames = Object.keys( prompt.config.tools ?? {} );
+  if ( promptToolNames.length === 0 ) {
+    return null;
+  }
+
+  const { tools } = getProvider( prompt.config.provider );
+  if ( !tools ) {
+    throw new ValidationError( buildProviderError( prompt, 'provider-specific tools' ) );
+  }
+
+  const missingTools = promptToolNames.filter( t => !Object.hasOwn( tools, t ) || typeof tools[t] !== 'function' );
+  if ( missingTools.length > 0 ) {
+    throw new ValidationError( buildProviderError( prompt, `these tools: ${missingTools.join( ', ' )}` ) );
+  }
+
+  const loadedTools = {};
+  for ( const name of promptToolNames ) {
+    loadedTools[name] = tools[name]( prompt.config.tools[name] );
+  }
+  return loadedTools;
+};

package/src/llm/utils/usage.js

@@ -0,0 +1,156 @@
+import Decimal from 'decimal.js';
+import { BaseAttribute } from '#trace_attribute';
+import { parseGroundingUsage } from './grounding.js';
+import { extractUsageFromSteps } from './usage_tools.js';
+
+const exists = v => Number.isSafeInteger( v ) && v >= 0;
+
+const safeSum = ( ...values ) => values.filter( exists ).reduce( ( t, v ) => t + v, 0 );
+
+export class LLMGenerationUsageItem {
+  static Group = { INPUT: 'input', OUTPUT: 'output', TOOLS: 'tools' };
+
+  group;
+  label;
+  amount;
+
+  constructor( group, label, amount ) {
+    this.group = group;
+    this.label = label;
+    this.amount = amount;
+  }
+}
+
+export class LLMGenerationUsage extends BaseAttribute {
+  static TYPE = 'llm:generation:usage';
+
+  static Status = Object.freeze( {
+    COMPLETE: 'complete',
+    INCOMPLETE: 'incomplete'
+  } );
+
+  providerId;
+  modelId;
+  status = LLMGenerationUsage.Status.INCOMPLETE;
+  input = null;
+  output = null;
+  total = null;
+  items = [];
+
+  constructor( modelId, providerId, items ) {
+    super( LLMGenerationUsage.TYPE );
+    this.providerId = providerId;
+    this.modelId = modelId;
+    this.items = items;
+    const inputItems = items.filter( p => p.group === LLMGenerationUsageItem.Group.INPUT );
+    if ( inputItems.length > 0 ) {
+      this.input = inputItems.reduce( ( s, p ) => s.add( p.amount ), Decimal( 0 ) ).toNumber();
+    }
+    const outputItems = items.filter( p => p.group === LLMGenerationUsageItem.Group.OUTPUT );
+    if ( outputItems.length > 0 ) {
+      this.output = outputItems.reduce( ( s, p ) => s.add( p.amount ), Decimal( 0 ) ).toNumber();
+    }
+
+    if ( exists( this.input ) || exists( this.output ) ) {
+      this.total = ( this.input ?? 0 ) + ( this.output ?? 0 );
+    }
+    this.status = LLMGenerationUsage.Status[exists( this.input ) && exists( this.output ) ? 'COMPLETE' : 'INCOMPLETE'];
+  }
+}
+
+/**
+ * Returns usage from response or reconstruct it from steps.
+ *
+ * The token check is load-bearing: aborted and truncated calls still report an aggregate, but with
+ * every token count undefined, and the steps are then the only record of what was already spent.
+ */
+const resolveUsage = ( { usage, steps } ) => {
+  if ( usage && ( exists( usage.inputTokens ) || exists( usage.outputTokens ) ) ) {
+    return usage;
+  }
+  return Array.isArray( steps ) && steps.length > 0 ? extractUsageFromSteps( steps ) : null;
+};
+
+/**
+ * Converts raw AI SDK usage to LLMGenerationUsage data.
+ *
+ * Reads `response.usage`, falling back to the sum of `response.steps` usage when the aggregate is
+ * missing tokens (truncated or failed streams). Grounding is aggregated from `response.steps`, so a
+ * partial `{ steps }` response is accepted when only the steps are known.
+ *
+ * @param {object} args
+ * @param {object} args.prompt - Output prompt with model configuration
+ * @param {object} [args.usage] - AI SDK response usage
+ * @param {object} [args.steps] - AI SDK response steps
+ *
+ * @returns {LLMGenerationUsage | null} LLM generation usage with input, output, total and detailed breakdown
+ */
+export const parseLLMUsage = ( { prompt, usage: usageArg = null, steps = [] } ) => {
+  const usage = resolveUsage( { usage: usageArg, steps } );
+  if ( !usage ) {
+    return null;
+  }
+  const { provider: providerId, model: modelId } = prompt.config;
+  const { inputTokens, inputTokenDetails, outputTokens, outputTokenDetails } = usage;
+  const { noCacheTokens, cacheReadTokens, cacheWriteTokens } = inputTokenDetails ?? {};
+  const { textTokens, reasoningTokens } = outputTokenDetails ?? {};
+
+  const items = [];
+
+  if ( exists( inputTokens ) ) {
+    // the sum of the components must be equal to the tokens
+    const useDetails = inputTokens > 0 && safeSum( noCacheTokens, cacheReadTokens, cacheWriteTokens ) === inputTokens;
+    const group = LLMGenerationUsageItem.Group.INPUT;
+
+    if ( useDetails ) {
+      if ( exists( noCacheTokens ) ) {
+        items.push( new LLMGenerationUsageItem( group, 'no_cache', noCacheTokens ) );
+      }
+      if ( exists( cacheReadTokens ) ) {
+        items.push( new LLMGenerationUsageItem( group, 'cache_read', cacheReadTokens ) );
+      }
+      if ( exists( cacheWriteTokens ) ) {
+        items.push( new LLMGenerationUsageItem( group, 'cache_write', cacheWriteTokens ) );
+      }
+    } else {
+      items.push( new LLMGenerationUsageItem( group, null, inputTokens ) );
+    }
+  }
+
+  if ( exists( outputTokens ) ) {
+    // the sum of the components must be equal to the tokens
+    const useDetails = outputTokens > 0 && safeSum( textTokens, reasoningTokens ) === outputTokens;
+    const group = LLMGenerationUsageItem.Group.OUTPUT;
+
+    if ( useDetails ) {
+      if ( exists( textTokens ) ) {
+        items.push( new LLMGenerationUsageItem( group, 'text', textTokens ) );
+      }
+      if ( exists( reasoningTokens ) ) {
+        items.push( new LLMGenerationUsageItem( group, 'reasoning', reasoningTokens ) );
+      }
+    } else {
+      items.push( new LLMGenerationUsageItem( group, null, outputTokens ) );
+    }
+  }
+
+  // Grounding is billed per step, so aggregate across every step rather than only the final one.
+  // Per-query families (Gemini 3) return queries.length per step; per-prompt families (Gemini 2.x)
+  // return 1 per grounded step. Summing yields total queries and grounded-step count respectively.
+  if ( Array.isArray( steps ) ) {
+    const grounding = steps
+      .filter( step => step?.providerMetadata )
+      .map( step => parseGroundingUsage( modelId, step.providerMetadata ) )
+      .filter( Boolean );
+    if ( grounding.length > 0 ) {
+      const amount = grounding.reduce( ( sum, g ) => sum + g.amount, 0 );
+      items.push( new LLMGenerationUsageItem( LLMGenerationUsageItem.Group.TOOLS, grounding[0].label, amount ) );
+    }
+  }
+
+  if ( items.length === 0 ) {
+    return null;
+  }
+
+  return new LLMGenerationUsage( modelId, providerId, items );
+};

package/src/llm/utils/usage_tools.js

@@ -0,0 +1,23 @@
+const sum = ( a, b ) => {
+  if ( Number.isFinite( a ) ) {
+    return a + ( Number.isFinite( b ) ? b : 0 );
+  } else {
+    return Number.isFinite( b ) ? b : undefined;
+  }
+};
+
+const isPlainObject = value => value !== null && typeof value === 'object';
+
+/** Deep numeric merge of two AI SDK usage objects; `raw` is provider-shaped and never summed */
+const mergeUsage = ( total, usage ) => Object.entries( usage )
+  .filter( ( [ key ] ) => key !== 'raw' )
+  .reduce( ( merged, [ key, value ] ) => ( {
+    ...merged,
+    [key]: isPlainObject( value ) ? mergeUsage( merged[key] ?? {}, value ) : sum( merged[key], value )
+  } ), total );
+
+/** Sums per-step AI SDK usage */
+export const extractUsageFromSteps = steps => steps
+  .map( step => step?.usage )
+  .filter( Boolean )
+  .reduce( ( total, usage ) => mergeUsage( total, usage ), {} );