@outputai/llm 0.7.0 → 0.7.1-next.2a4105c.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@outputai/llm",
-  "version": "0.7.0",
+  "version": "0.7.1-next.2a4105c.0",
   "description": "Framework abstraction to interact with LLM models",
   "type": "module",
   "main": "src/index.js",
@@ -13,7 +13,7 @@
     "gray-matter": "4.0.3",
     "liquidjs": "10.25.7",
     "undici": "8.1.0",
-    "@outputai/core": "0.7.0"
+    "@outputai/core": "0.7.1-next.2a4105c.0"
   },
   "devDependencies": {
     "ai": "6.0.168",

package/src/ai_sdk_options.js

@@ -1,4 +1,5 @@
 import { loadImageModel, loadTextModel, loadTools } from './ai_model.js';
+import { resolveMessageProviderOptions } from './prompt/block_options.js';
 import { FatalError } from '@outputai/core';
 
 /**
@@ -13,7 +14,7 @@ export const loadAiSdkTextOptions = prompt => {
   }
   const options = {
     model: loadTextModel( prompt ),
-    messages: prompt.messages,
+    messages: resolveMessageProviderOptions( prompt ),
     providerOptions: prompt.config.providerOptions
   };
 

package/src/ai_sdk_options.spec.js

@@ -161,4 +161,32 @@ describe( 'ai_sdk_options', () => {
     );
     expect( loadImageModelImpl ).not.toHaveBeenCalled();
   } );
+
+  it( 'resolves block attributes into per-message providerOptions', async () => {
+    const prompt = {
+      name: 'cache@v1',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-sonnet-4-5',
+        messageOptions: { cached: { anthropic: { cacheControl: { type: 'ephemeral', ttl: '1h' } } } }
+      },
+      messages: [
+        { role: 'system', content: 'Static', attributes: { options: 'cached' } },
+        { role: 'user', content: 'Hello' }
+      ],
+      instructions: null
+    };
+
+    const { loadAiSdkTextOptions } = await importSut();
+    const result = loadAiSdkTextOptions( prompt );
+
+    expect( result.messages ).toEqual( [
+      {
+        role: 'system',
+        content: 'Static',
+        providerOptions: { anthropic: { cacheControl: { type: 'ephemeral', ttl: '1h' } } }
+      },
+      { role: 'user', content: 'Hello' }
+    ] );
+  } );
 } );

package/src/cost/index.js

@@ -32,8 +32,13 @@ export const calculateLLMCallCost = async ( { modelId, usage } ) => {
     if ( Number.isFinite( pricing.input ) && Number.isFinite( nonCachedTokens ) ) {
       llmUsage.addUsage( { type: 'input', ppm: pricing.input, amount: nonCachedTokens } );
     }
-    if ( Number.isFinite( pricing.cache_read ) && Number.isFinite( cachedInputTokens ) ) {
-      llmUsage.addUsage( { type: 'input_cached', ppm: pricing.cache_read, amount: cachedInputTokens } );
+    // Surface cached input tokens whenever the provider reports them, even if the model's
+    // pricing lacks a cache_read rate — otherwise caching savings vanish from the token
+    // aggregation (these tokens are already excluded from the input line above). Price at
+    // cache_read when available, otherwise at 0.
+    if ( Number.isFinite( cachedInputTokens ) ) {
+      const cacheReadPpm = Number.isFinite( pricing.cache_read ) ? pricing.cache_read : 0;
+      llmUsage.addUsage( { type: 'input_cached', ppm: cacheReadPpm, amount: cachedInputTokens } );
     }
     if ( Number.isFinite( pricing.output ) && Number.isFinite( outputTokens ) ) {
       llmUsage.addUsage( { type: 'output', ppm: pricing.output, amount: outputTokens } );

package/src/cost/index.spec.js

@@ -132,7 +132,7 @@ describe( 'calculateLLMCallCost', () => {
     } );
   } );
 
-  it( 'omits cached usage when model has no cache_read rate', async () => {
+  it( 'still counts cached tokens when the model has no cache_read rate', async () => {
     mockFetchModelsPricing.mockResolvedValue( new Map( [ [ 'no-cache', { input: 2, output: 10 } ] ] ) );
 
     const result = await calculateLLMCallCost( {
@@ -140,14 +140,17 @@ describe( 'calculateLLMCallCost', () => {
       usage: { inputTokens: 1_000_000, cachedInputTokens: 200_000, outputTokens: 0 }
     } );
 
+    // Cached tokens are surfaced (priced at 0 without a cache_read rate) so caching is
+    // visible in the aggregation; cost is unchanged since they are excluded from `input`.
     expectLLMUsage( result, {
       modelId: 'no-cache',
       usage: [
         { type: 'input', ppm: 2, amount: 800_000, total: 1.6 },
+        { type: 'input_cached', ppm: 0, amount: 200_000, total: 0 },
         { type: 'output', ppm: 10, amount: 0, total: 0 }
       ],
       total: 1.6,
-      tokensUsed: 800_000
+      tokensUsed: 1_000_000
     } );
   } );
 

package/src/index.d.ts

@@ -57,6 +57,12 @@ export type PromptMessage = {
   role: string;
   /** The content of the message */
   content: string;
+  /**
+   * Parsed opening-tag attributes for the block. Currently `options` — a space-separated list of
+   * frontmatter `messageOptions` set names — which is resolved into per-message `providerOptions`
+   * at call time and stripped before the request is sent. Authored as `<system options="set_a set_b">`.
+   */
+  attributes?: Record<string, string | boolean>;
 };
 
 /**
@@ -139,6 +145,13 @@ export type Prompt = {
 
     /** Provider-specific options */
     providerOptions?: Record<string, unknown>;
+
+    /**
+     * Named, reusable per-message `providerOptions` sets, referenced from message blocks via the
+     * `options="<name>"` attribute. Each value is a provider-namespaced options object, e.g.
+     * `{ anthropic: { cacheControl: { type: 'ephemeral' } } }`.
+     */
+    messageOptions?: Record<string, Record<string, Record<string, unknown>>>;
   };
 
   /** Array of messages in the conversation */

package/src/prompt/block_options.js

@@ -0,0 +1,58 @@
+import { FatalError, z } from '@outputai/core';
+
+/** Shallow-merge two providerOptions objects, combining keys within each provider namespace. */
+const mergeProviderOptions = ( base = {}, extra = {} ) => {
+  const merged = { ...base };
+  for ( const [ namespace, options ] of Object.entries( extra ) ) {
+    merged[namespace] = { ...merged[namespace], ...options };
+  }
+  return merged;
+};
+
+/** Merge the named `messageOptions` sets referenced by a block's `options` attribute. */
+const resolveOptions = ( value, { name, config } ) => {
+  const sets = config.messageOptions ?? {};
+  return value.trim().split( /\s+/ ).reduce( ( acc, setName ) => {
+    if ( !sets[setName] ) {
+      throw new FatalError( `Prompt "${name}" references unknown messageOptions set "${setName}"` );
+    }
+    return mergeProviderOptions( acc, sets[setName] );
+  }, {} );
+};
+
+/**
+ * Registry of supported block attributes. Each entry declares how the attribute is validated
+ * (`schema`) and how it contributes to a message's per-message `providerOptions` (`resolve`).
+ * Add an entry to support a new block option — validation ({@link attributesSchema}) and
+ * resolution ({@link resolveMessageProviderOptions}) both derive from this table.
+ */
+const BLOCK_OPTIONS = {
+  options: {
+    schema: z.string().min( 1 ),
+    resolve: resolveOptions
+  }
+};
+
+/** Zod schema for a block's `attributes` object, derived from the option registry. */
+export const attributesSchema = z.object(
+  Object.fromEntries(
+    Object.entries( BLOCK_OPTIONS ).map( ( [ name, def ] ) => [ name, def.schema.optional() ] )
+  )
+).strict();
+
+/**
+ * Resolve each message's authoring `attributes` into AI SDK per-message `providerOptions`,
+ * returning clean messages with the `attributes` helper stripped.
+ *
+ * @param {object} prompt - Loaded prompt object (`{ name, config, messages }`)
+ * @returns {Array<object>} Messages with resolved `providerOptions`
+ */
+export const resolveMessageProviderOptions = ( { name, config, messages } ) =>
+  messages.map( ( { attributes, providerOptions, ...message } ) => {
+    const resolved = Object.entries( attributes ?? {} ).reduce( ( acc, [ key, value ] ) => {
+      const option = BLOCK_OPTIONS[key];
+      return option ? mergeProviderOptions( acc, option.resolve( value, { name, config } ) ) : acc;
+    }, providerOptions ?? {} );
+
+    return Object.keys( resolved ).length > 0 ? { ...message, providerOptions: resolved } : message;
+  } );

package/src/prompt/block_options.spec.js

@@ -0,0 +1,71 @@
+import { describe, it, expect } from 'vitest';
+import { FatalError } from '@outputai/core';
+import { attributesSchema, resolveMessageProviderOptions } from './block_options.js';
+
+const textPrompt = ( { config = {}, messages } ) => ( {
+  name: 'test@v1',
+  config: { provider: 'anthropic', model: 'claude-sonnet-4-5', ...config },
+  messages
+} );
+
+describe( 'attributesSchema', () => {
+  it( 'accepts the options attribute', () => {
+    expect( attributesSchema.safeParse( { options: 'cached' } ).success ).toBe( true );
+    expect( attributesSchema.safeParse( { options: 'cached fast' } ).success ).toBe( true );
+    expect( attributesSchema.safeParse( {} ).success ).toBe( true );
+  } );
+
+  it( 'rejects unknown attributes, including the removed cache shorthand', () => {
+    expect( attributesSchema.safeParse( { cache: true } ).success ).toBe( false );
+    expect( attributesSchema.safeParse( { unknown: 'x' } ).success ).toBe( false );
+  } );
+} );
+
+describe( 'resolveMessageProviderOptions', () => {
+  it( 'merges a referenced messageOptions set into per-message providerOptions', () => {
+    const result = resolveMessageProviderOptions( textPrompt( {
+      config: { messageOptions: { cached: { anthropic: { cacheControl: { type: 'ephemeral' } } } } },
+      messages: [
+        { role: 'system', content: 'Docs', attributes: { options: 'cached' } },
+        { role: 'user', content: 'Hello' }
+      ]
+    } ) );
+
+    expect( result ).toEqual( [
+      {
+        role: 'system',
+        content: 'Docs',
+        providerOptions: { anthropic: { cacheControl: { type: 'ephemeral' } } }
+      },
+      { role: 'user', content: 'Hello' }
+    ] );
+  } );
+
+  it( 'merges multiple referenced sets onto one block', () => {
+    const [ system ] = resolveMessageProviderOptions( textPrompt( {
+      config: {
+        messageOptions: {
+          cached: { anthropic: { cacheControl: { type: 'ephemeral', ttl: '1h' } } },
+          openaiKey: { openai: { promptCacheKey: 'enrich-v1' } }
+        }
+      },
+      messages: [ { role: 'system', content: 'Docs', attributes: { options: 'cached openaiKey' } } ]
+    } ) );
+
+    expect( system.providerOptions ).toEqual( {
+      anthropic: { cacheControl: { type: 'ephemeral', ttl: '1h' } },
+      openai: { promptCacheKey: 'enrich-v1' }
+    } );
+  } );
+
+  it( 'throws when the options attribute references an unknown set', () => {
+    expect( () => resolveMessageProviderOptions( textPrompt( {
+      messages: [ { role: 'user', content: 'Hello', attributes: { options: 'missing' } } ]
+    } ) ) ).toThrow( FatalError );
+  } );
+
+  it( 'leaves messages without attributes unchanged', () => {
+    const messages = [ { role: 'user', content: 'Hello' } ];
+    expect( resolveMessageProviderOptions( textPrompt( { messages } ) ) ).toEqual( messages );
+  } );
+} );

package/src/prompt/blocks.js

@@ -0,0 +1,47 @@
+/**
+ * Roles that introduce a message block. Add a role here to support a new
+ * `<role>...</role>` block — the tokenizer pattern is derived from this set,
+ * so no other parser change is required.
+ */
+export const BLOCK_ROLES = new Set( [ 'system', 'user', 'assistant', 'tool' ] );
+
+const BLOCK_PATTERN = new RegExp(
+  `<(${[ ...BLOCK_ROLES ].join( '|' )})((?:\\s[^>]*)?)>([\\s\\S]*?)<\\/\\1>`,
+  'gm'
+);
+
+const ATTRIBUTE_PATTERN = /([a-zA-Z][\w-]*)(?:=(?:"([^"]*)"|'([^']*)'|(\S+)))?/g;
+
+/**
+ * Parse a raw opening-tag attribute string into a plain object. Supports bare booleans
+ * (`cache`), double/single-quoted values, and unquoted values:
+ * `cache options="a b" ttl='1h'` → `{ cache: true, options: 'a b', ttl: '1h' }`.
+ *
+ * @param {string} [raw] - Raw attribute text between the role and the closing `>`
+ * @returns {Record<string, string | true>} Parsed attributes
+ */
+export const parseAttributes = ( raw = '' ) =>
+  Object.fromEntries(
+    [ ...raw.matchAll( ATTRIBUTE_PATTERN ) ].map(
+      ( [ _, key, doubleQuoted, singleQuoted, bare ] ) =>
+        [ key, doubleQuoted ?? singleQuoted ?? bare ?? true ]
+    )
+  );
+
+/**
+ * Tokenize a rendered prompt body into message blocks. Each block is `{ role, content }`,
+ * plus `attributes` when the opening tag carried any. Content between role tags is treated
+ * as opaque text, so prompt bodies may freely contain other angle-bracket markup.
+ *
+ * @param {string} content - Rendered prompt body (after frontmatter is stripped)
+ * @returns {Array<{ role: string, content: string, attributes?: Record<string, string | true> }>}
+ */
+export const tokenizeBlocks = content =>
+  [ ...content.matchAll( BLOCK_PATTERN ) ].map( ( [ _, role, rawAttributes, text ] ) => {
+    const attributes = parseAttributes( rawAttributes.trim() );
+    return {
+      role,
+      content: text.trim(),
+      ...( Object.keys( attributes ).length > 0 && { attributes } )
+    };
+  } );

package/src/prompt/blocks.spec.js

@@ -0,0 +1,63 @@
+import { describe, it, expect } from 'vitest';
+import { parseAttributes, tokenizeBlocks, BLOCK_ROLES } from './blocks.js';
+
+describe( 'parseAttributes', () => {
+  it( 'parses a bare attribute as boolean true', () => {
+    expect( parseAttributes( 'pinned' ) ).toEqual( { pinned: true } );
+  } );
+
+  it( 'parses double- and single-quoted values', () => {
+    expect( parseAttributes( 'ttl="1h" mode=\'fast\'' ) ).toEqual( { ttl: '1h', mode: 'fast' } );
+  } );
+
+  it( 'parses unquoted values', () => {
+    expect( parseAttributes( 'ttl=1h' ) ).toEqual( { ttl: '1h' } );
+  } );
+
+  it( 'parses multiple attributes and preserves spaces inside quotes', () => {
+    expect( parseAttributes( 'pinned options="cached fast"' ) ).toEqual( {
+      pinned: true,
+      options: 'cached fast'
+    } );
+  } );
+
+  it( 'returns an empty object for blank input', () => {
+    expect( parseAttributes( '' ) ).toEqual( {} );
+    expect( parseAttributes() ).toEqual( {} );
+  } );
+} );
+
+describe( 'tokenizeBlocks', () => {
+  it( 'tokenizes plain blocks without an attributes key', () => {
+    const blocks = tokenizeBlocks( '<system>Hi</system>\n<user>Yo</user>' );
+    expect( blocks ).toEqual( [
+      { role: 'system', content: 'Hi' },
+      { role: 'user', content: 'Yo' }
+    ] );
+  } );
+
+  it( 'attaches parsed attributes to the block', () => {
+    const blocks = tokenizeBlocks( '<system options="a b" pinned>Hi</system>' );
+    expect( blocks[0] ).toEqual( {
+      role: 'system',
+      content: 'Hi',
+      attributes: { options: 'a b', pinned: true }
+    } );
+  } );
+
+  it( 'captures unknown attributes generically (validation rejects them later)', () => {
+    const blocks = tokenizeBlocks( '<user data="x">Hi</user>' );
+    expect( blocks[0].attributes ).toEqual( { data: 'x' } );
+  } );
+
+  it( 'treats angle-bracket markup inside a block as opaque content', () => {
+    const blocks = tokenizeBlocks( '<user>Compare <div> and <span> tags</user>' );
+    expect( blocks[0] ).toEqual( { role: 'user', content: 'Compare <div> and <span> tags' } );
+  } );
+
+  it( 'tokenizes every registered role', () => {
+    const body = [ ...BLOCK_ROLES ].map( role => `<${role}>${role} body</${role}>` ).join( '\n' );
+    const blocks = tokenizeBlocks( body );
+    expect( blocks.map( block => block.role ) ).toEqual( [ ...BLOCK_ROLES ] );
+  } );
+} );

package/src/prompt/parser.js

@@ -1,5 +1,6 @@
 import matter from 'gray-matter';
 import { FatalError } from '@outputai/core';
+import { tokenizeBlocks } from './blocks.js';
 
 export function parsePrompt( { name, raw } ) {
   const { data: config, content } = matter( raw );
@@ -8,11 +9,7 @@ export function parsePrompt( { name, raw } ) {
     throw new FatalError( `Prompt "${name}" has no content after frontmatter` );
   }
 
-  const infoExtractor = /<(system|user|assistant|tool)>([\s\S]*?)<\/\1>/gm;
-  const messages = [ ...content.matchAll( infoExtractor ) ].map(
-    ( [ _, role, text ] ) => ( { role, content: text.trim() } )
-  );
-
+  const messages = tokenizeBlocks( content );
   const instructions = messages.length === 0 ? content.trim() : null;
 
   return { config, messages, instructions };

package/src/prompt/parser.spec.js

@@ -164,4 +164,23 @@ model: claude-3-5-sonnet-20241022
     ] );
     expect( result.instructions ).toBeNull();
   } );
+
+  it( 'surfaces block opening-tag attributes as an attributes object', () => {
+    const raw = `---
+provider: anthropic
+model: claude-sonnet-4-5
+---
+
+<system options="cached">Static.</system>
+<user>Question</user>`;
+
+    const result = parsePrompt( { name: 'test', raw } );
+
+    expect( result.messages[0] ).toEqual( {
+      role: 'system',
+      content: 'Static.',
+      attributes: { options: 'cached' }
+    } );
+    expect( result.messages[1] ).toEqual( { role: 'user', content: 'Question' } );
+  } );
 } );

package/src/prompt/validations.js

@@ -1,8 +1,12 @@
 import { ValidationError, z } from '@outputai/core';
+import { attributesSchema } from './block_options.js';
 
 const toolConfigSchema = z.record( z.string(), z.unknown() );
 const toolsConfigSchema = z.record( z.string(), toolConfigSchema );
 
+// A provider-namespaced options object, e.g. { anthropic: { cacheControl: { type: 'ephemeral' } } }
+const providerOptionsSchema = z.record( z.string(), z.record( z.string(), z.unknown() ) );
+
 export const promptSchema = z.object( {
   name: z.string(),
   config: z.object( {
@@ -22,12 +26,14 @@ export const promptSchema = z.object( {
         type: z.enum( [ 'enabled', 'disabled' ] ),
         budgetTokens: z.number().optional()
       } ).loose().optional()
-    } ).loose().optional()
+    } ).loose().optional(),
+    messageOptions: z.record( z.string(), providerOptionsSchema ).optional()
   } ).loose(),
   messages: z.array(
     z.object( {
       role: z.string(),
-      content: z.string()
+      content: z.string(),
+      attributes: attributesSchema.optional()
     } ).strict()
   ),
   instructions: z.string().trim().min( 1 ).nullable().optional()

package/src/prompt/validations.spec.js

@@ -596,4 +596,53 @@ describe( 'validatePrompt', () => {
 
     expect( () => validatePrompt( maxTokensSnakeCase ) ).not.toThrow();
   } );
+
+  it( 'should validate the options attribute referencing messageOptions sets', () => {
+    const promptWithMessageOptions = {
+      name: 'message-options-prompt',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-sonnet-4-5',
+        messageOptions: {
+          cached: { anthropic: { cacheControl: { type: 'ephemeral' } } }
+        }
+      },
+      messages: [
+        { role: 'system', content: 'Docs.', attributes: { options: 'cached' } },
+        { role: 'user', content: 'Question' }
+      ]
+    };
+
+    expect( () => validatePrompt( promptWithMessageOptions ) ).not.toThrow();
+  } );
+
+  it( 'should reject the removed cache shorthand as an unknown block attribute', () => {
+    const cacheShorthandPrompt = {
+      name: 'cache-shorthand-prompt',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-sonnet-4-5'
+      },
+      messages: [
+        { role: 'system', content: 'Static.', attributes: { cache: true } }
+      ]
+    };
+
+    expect( () => validatePrompt( cacheShorthandPrompt ) ).toThrow( ValidationError );
+  } );
+
+  it( 'should throw ValidationError for unknown top-level message fields', () => {
+    const unknownFieldPrompt = {
+      name: 'unknown-field-prompt',
+      config: {
+        provider: 'anthropic',
+        model: 'claude-sonnet-4-5'
+      },
+      messages: [
+        { role: 'user', content: 'Hi', options: 'cached' }
+      ]
+    };
+
+    expect( () => validatePrompt( unknownFieldPrompt ) ).toThrow( ValidationError );
+  } );
 } );