@outputai/core 0.4.0 → 0.4.1-dev.92bc2fb.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@outputai/core",
-  "version": "0.4.0",
+  "version": "0.4.1-dev.92bc2fb.0",
   "description": "The core module of the output framework",
   "type": "module",
   "exports": {
@@ -19,6 +19,9 @@
     "./sdk_utils": {
       "types": "./src/utils/index.d.ts",
       "import": "./src/utils/index.js"
+    },
+    "./sdk_tracing_tools": {
+      "import": "./src/tracing/tools/index.js"
     }
   },
   "files": [

package/src/activity_integration/tracing.d.ts

@@ -42,4 +42,5 @@ export declare function addEventAttribute( args: { eventId: string; name: string
  */
 export declare const Attribute: {
   COST: 'cost';
+  TOKEN_USAGE: 'token_usage';
 };

package/src/activity_integration/tracing.js

@@ -49,5 +49,6 @@ export const addEventAttribute = ( { eventId, name, value } ) =>
  * Known attributes
  */
 export const Attribute = {
-  COST: 'cost'
+  COST: 'cost',
+  TOKEN_USAGE: 'token_usage'
 };

package/src/tracing/tools/aggregate_trace_attributes.js

@@ -0,0 +1,118 @@
+/**
+ * Aggregate `attributes.cost` and `attributes.token_usage` across an entire trace tree.
+ *
+ * Walks every node in the tree, sums `attributes.cost.total` grouped by the emitting
+ * event name (inferred from node `kind` — see `eventNameForKind`), and sums
+ * `attributes.token_usage` across LLM nodes. Falls back to `output.usage` on
+ * legacy llm trace nodes that predate the `attributes.token_usage` write
+ * (see overview §1.2).
+ *
+ * @typedef {object} TraceAttributes
+ * @property {{ total: number, components: Array<{ name: string, value: number }> }} cost
+ * @property {{ inputTokens: number, outputTokens: number, cachedInputTokens: number, totalTokens: number }} tokenUsage
+ */
+
+const COST_EVENT_LLM = 'cost:llm:request';
+const COST_EVENT_HTTP = 'cost:http:request';
+const COST_EVENT_OTHER = 'other';
+
+/**
+ * Map a trace node `kind` to the canonical cost event name that would emit it.
+ * Unknown kinds bucket into `other` so future event sources still roll up cleanly.
+ *
+ * @param {string} kind
+ * @returns {string}
+ */
+const eventNameForKind = kind => {
+  if ( kind === 'llm' ) {
+    return COST_EVENT_LLM;
+  }
+  if ( kind === 'http' ) {
+    return COST_EVENT_HTTP;
+  }
+  return COST_EVENT_OTHER;
+};
+
+const isNumber = value => typeof value === 'number' && Number.isFinite( value );
+
+/**
+ * Pull token usage off an llm node, preferring the new attribute over the legacy
+ * `output.usage` fallback. Returns `null` when neither shape is present.
+ */
+const readTokenUsage = node => {
+  const attrUsage = node.attributes?.token_usage;
+  if ( attrUsage && typeof attrUsage === 'object' ) {
+    return attrUsage;
+  }
+  const legacyUsage = node.output?.usage;
+  if ( legacyUsage && typeof legacyUsage === 'object' ) {
+    return legacyUsage;
+  }
+  return null;
+};
+
+/**
+ * Recursively walk a trace tree depth-first, applying `visit` to each node.
+ */
+const walk = ( node, visit ) => {
+  if ( !node ) {
+    return;
+  }
+  visit( node );
+  for ( const child of node.children ?? [] ) {
+    walk( child, visit );
+  }
+};
+
+/**
+ * Build the aggregated `attributes` payload returned by `/trace-attributes`.
+ * Component buckets always appear in a stable order so callers can index them
+ * positionally if they want to.
+ *
+ * @param {object|null} root - The root NodeEntry returned by `buildTraceTree`.
+ * @returns {TraceAttributes}
+ */
+export default function aggregateTraceAttributes( root ) {
+  const costByEvent = new Map( [
+    [ COST_EVENT_LLM, 0 ],
+    [ COST_EVENT_HTTP, 0 ],
+    [ COST_EVENT_OTHER, 0 ]
+  ] );
+  const tokenUsage = { inputTokens: 0, outputTokens: 0, cachedInputTokens: 0, totalTokens: 0 };
+
+  walk( root, node => {
+    const cost = node.attributes?.cost;
+    if ( cost && isNumber( cost.total ) ) {
+      const eventName = eventNameForKind( node.kind );
+      costByEvent.set( eventName, ( costByEvent.get( eventName ) ?? 0 ) + cost.total );
+    }
+
+    if ( node.kind === 'llm' ) {
+      const usage = readTokenUsage( node );
+      if ( usage ) {
+        if ( isNumber( usage.inputTokens ) ) {
+          tokenUsage.inputTokens += usage.inputTokens;
+        }
+        if ( isNumber( usage.outputTokens ) ) {
+          tokenUsage.outputTokens += usage.outputTokens;
+        }
+        if ( isNumber( usage.cachedInputTokens ) ) {
+          tokenUsage.cachedInputTokens += usage.cachedInputTokens;
+        }
+        if ( isNumber( usage.totalTokens ) ) {
+          tokenUsage.totalTokens += usage.totalTokens;
+        }
+      }
+    }
+  } );
+
+  const components = Array.from( costByEvent, ( [ name, value ] ) => ( { name, value } ) );
+  const total = components.reduce( ( sum, { value } ) => sum + value, 0 );
+
+  return {
+    cost: { total, components },
+    tokenUsage
+  };
+}
+
+export { COST_EVENT_LLM, COST_EVENT_HTTP, COST_EVENT_OTHER };

package/src/tracing/tools/aggregate_trace_attributes.spec.js

@@ -0,0 +1,231 @@
+import { describe, it, expect } from 'vitest';
+import aggregateTraceAttributes, {
+  COST_EVENT_LLM,
+  COST_EVENT_HTTP,
+  COST_EVENT_OTHER
+} from './aggregate_trace_attributes.js';
+
+const node = ( { id, kind = 'step', attributes = {}, output, children = [] } ) => ( {
+  id,
+  kind,
+  name: id,
+  startedAt: 0,
+  endedAt: 0,
+  input: undefined,
+  output,
+  attributes,
+  children
+} );
+
+describe( 'aggregate_trace_attributes', () => {
+  it( 'returns zeros for a null root', () => {
+    const result = aggregateTraceAttributes( null );
+    expect( result.cost.total ).toBe( 0 );
+    expect( result.cost.components ).toEqual( [
+      { name: COST_EVENT_LLM, value: 0 },
+      { name: COST_EVENT_HTTP, value: 0 },
+      { name: COST_EVENT_OTHER, value: 0 }
+    ] );
+    expect( result.tokenUsage ).toEqual( {
+      inputTokens: 0, outputTokens: 0, cachedInputTokens: 0, totalTokens: 0
+    } );
+  } );
+
+  it( 'returns zeros for a tree with no cost or usage attributes', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      children: [ node( { id: 's1' } ), node( { id: 's2' } ) ]
+    } );
+    const result = aggregateTraceAttributes( root );
+    expect( result.cost.total ).toBe( 0 );
+    expect( result.tokenUsage.totalTokens ).toBe( 0 );
+  } );
+
+  it( 'buckets cost by node kind into llm / http / other components', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      children: [
+        node( { id: 'llm-1', kind: 'llm', attributes: { cost: { total: 0.20 } } } ),
+        node( { id: 'llm-2', kind: 'llm', attributes: { cost: { total: 0.10 } } } ),
+        node( { id: 'http-1', kind: 'http', attributes: { cost: { total: 0.50 } } } ),
+        // Unknown kind falls into the catch-all bucket
+        node( { id: 'step-1', kind: 'step', attributes: { cost: { total: 0.07 } } } )
+      ]
+    } );
+    const result = aggregateTraceAttributes( root );
+
+    const byName = Object.fromEntries( result.cost.components.map( c => [ c.name, c.value ] ) );
+    expect( byName[COST_EVENT_LLM] ).toBeCloseTo( 0.30, 10 );
+    expect( byName[COST_EVENT_HTTP] ).toBeCloseTo( 0.50, 10 );
+    expect( byName[COST_EVENT_OTHER] ).toBeCloseTo( 0.07, 10 );
+    expect( result.cost.total ).toBeCloseTo( 0.87, 10 );
+  } );
+
+  it( 'total equals the sum of all components', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      children: [
+        node( { id: 'llm-1', kind: 'llm', attributes: { cost: { total: 0.1234 } } } ),
+        node( { id: 'http-1', kind: 'http', attributes: { cost: { total: 0.0011 } } } )
+      ]
+    } );
+    const { cost } = aggregateTraceAttributes( root );
+    const sum = cost.components.reduce( ( s, c ) => s + c.value, 0 );
+    expect( cost.total ).toBeCloseTo( sum, 10 );
+  } );
+
+  it( 'sums token_usage across llm nodes from the attribute path', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      children: [
+        node( {
+          id: 'llm-1', kind: 'llm', attributes: {
+            token_usage: { inputTokens: 100, outputTokens: 20, cachedInputTokens: 5, totalTokens: 125 }
+          }
+        } ),
+        node( {
+          id: 'llm-2', kind: 'llm', attributes: {
+            token_usage: { inputTokens: 50, outputTokens: 10, cachedInputTokens: 1, totalTokens: 61 }
+          }
+        } )
+      ]
+    } );
+    const { tokenUsage } = aggregateTraceAttributes( root );
+    expect( tokenUsage ).toEqual( {
+      inputTokens: 150,
+      outputTokens: 30,
+      cachedInputTokens: 6,
+      totalTokens: 186
+    } );
+  } );
+
+  it( 'falls back to output.usage on legacy llm nodes that lack attributes.token_usage', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      children: [
+        // Legacy shape — usage lives on output.usage, no attributes.token_usage
+        node( {
+          id: 'llm-legacy',
+          kind: 'llm',
+          output: { result: '...', usage: { inputTokens: 200, outputTokens: 40, totalTokens: 240 } }
+        } )
+      ]
+    } );
+    const { tokenUsage } = aggregateTraceAttributes( root );
+    expect( tokenUsage.inputTokens ).toBe( 200 );
+    expect( tokenUsage.outputTokens ).toBe( 40 );
+    expect( tokenUsage.totalTokens ).toBe( 240 );
+    expect( tokenUsage.cachedInputTokens ).toBe( 0 );
+  } );
+
+  it( 'prefers attributes.token_usage over output.usage when both are present', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      children: [
+        node( {
+          id: 'llm-1',
+          kind: 'llm',
+          attributes: { token_usage: { inputTokens: 10, outputTokens: 2, totalTokens: 12 } },
+          output: { usage: { inputTokens: 999, outputTokens: 999, totalTokens: 999 } }
+        } )
+      ]
+    } );
+    const { tokenUsage } = aggregateTraceAttributes( root );
+    expect( tokenUsage.inputTokens ).toBe( 10 );
+    expect( tokenUsage.totalTokens ).toBe( 12 );
+  } );
+
+  it( 'ignores token_usage shapes on non-llm nodes', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      // attributes.token_usage on a non-llm node is intentionally ignored —
+      // only llm nodes contribute to the token-usage rollup today.
+      children: [
+        node( {
+          id: 'step-1', kind: 'step', attributes: {
+            token_usage: { inputTokens: 999, outputTokens: 999, totalTokens: 999 }
+          }
+        } )
+      ]
+    } );
+    const { tokenUsage } = aggregateTraceAttributes( root );
+    expect( tokenUsage.totalTokens ).toBe( 0 );
+  } );
+
+  it( 'aggregates a mixed tree with cost on http nodes and usage on llm nodes', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      children: [
+        node( {
+          id: 'llm-1',
+          kind: 'llm',
+          attributes: {
+            cost: { total: 0.0038 },
+            token_usage: { inputTokens: 2264, outputTokens: 411, cachedInputTokens: 100, totalTokens: 2775 }
+          }
+        } ),
+        node( {
+          id: 'http-1',
+          kind: 'http',
+          attributes: { cost: { total: 0.50 } }
+        } )
+      ]
+    } );
+    const result = aggregateTraceAttributes( root );
+
+    const byName = Object.fromEntries( result.cost.components.map( c => [ c.name, c.value ] ) );
+    expect( byName[COST_EVENT_LLM] ).toBeCloseTo( 0.0038, 10 );
+    expect( byName[COST_EVENT_HTTP] ).toBeCloseTo( 0.50, 10 );
+    expect( byName[COST_EVENT_OTHER] ).toBe( 0 );
+    expect( result.cost.total ).toBeCloseTo( 0.5038, 10 );
+
+    expect( result.tokenUsage ).toEqual( {
+      inputTokens: 2264,
+      outputTokens: 411,
+      cachedInputTokens: 100,
+      totalTokens: 2775
+    } );
+  } );
+
+  it( 'recurses through nested children', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      children: [
+        node( {
+          id: 's1',
+          kind: 'step',
+          children: [
+            node( {
+              id: 'llm-1', kind: 'llm', attributes: {
+                cost: { total: 0.01 },
+                token_usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }
+              }
+            } )
+          ]
+        } )
+      ]
+    } );
+    const result = aggregateTraceAttributes( root );
+    expect( result.cost.total ).toBeCloseTo( 0.01, 10 );
+    expect( result.tokenUsage.totalTokens ).toBe( 15 );
+  } );
+
+  it( 'keeps the canonical component ordering: llm, http, other', () => {
+    const root = node( { id: 'wf', kind: 'workflow' } );
+    const { cost } = aggregateTraceAttributes( root );
+    expect( cost.components.map( c => c.name ) ).toEqual( [
+      COST_EVENT_LLM,
+      COST_EVENT_HTTP,
+      COST_EVENT_OTHER
+    ] );
+  } );
+} );

package/src/tracing/tools/index.js

@@ -0,0 +1,7 @@
+export { default as buildTraceTree } from './build_trace_tree.js';
+export {
+  default as aggregateTraceAttributes,
+  COST_EVENT_LLM,
+  COST_EVENT_HTTP,
+  COST_EVENT_OTHER
+} from './aggregate_trace_attributes.js';