@outputai/core 0.3.3-next.e8eff63.0 → 0.4.1-dev.06c2b50.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@outputai/core",
-  "version": "0.3.3-next.e8eff63.0",
+  "version": "0.4.1-dev.06c2b50.0",
   "description": "The core module of the output framework",
   "type": "module",
   "exports": {
@@ -19,6 +19,9 @@
     "./sdk_utils": {
       "types": "./src/utils/index.d.ts",
       "import": "./src/utils/index.js"
+    },
+    "./sdk_tracing_tools": {
+      "import": "./src/tracing/tools/index.js"
     }
   },
   "files": [
@@ -45,7 +48,12 @@
     "redis": "5.12.1",
     "stacktrace-parser": "0.1.11",
     "undici": "8.1.0",
-    "winston": "3.19.0",
+    "winston": "3.19.0"
+  },
+  "peerDependencies": {
+    "zod": "^4.3.6"
+  },
+  "devDependencies": {
     "zod": "4.3.6"
   },
   "license": "Apache-2.0",

package/src/activity_integration/tracing.d.ts

@@ -42,4 +42,5 @@ export declare function addEventAttribute( args: { eventId: string; name: string
  */
 export declare const Attribute: {
   COST: 'cost';
+  TOKEN_USAGE: 'token_usage';
 };

package/src/activity_integration/tracing.js

@@ -49,5 +49,6 @@ export const addEventAttribute = ( { eventId, name, value } ) =>
  * Known attributes
  */
 export const Attribute = {
-  COST: 'cost'
+  COST: 'cost',
+  TOKEN_USAGE: 'token_usage'
 };

package/src/interface/workflow.d.ts

@@ -139,9 +139,9 @@ export type WorkflowFunction<
  */
 export type WorkflowFunctionWrapper<WorkflowFunction> =
   [Parameters<WorkflowFunction>[0]] extends [undefined | null] ?
-    ( input?: undefined | null, config?: WorkflowInvocationConfiguration<Parameters<WorkflowFunction>[1]> ) =>
+    ( input?: undefined | null, config?: WorkflowInvocationConfiguration ) =>
     ReturnType<WorkflowFunction> :
-    ( input: Parameters<WorkflowFunction>[0], config?: WorkflowInvocationConfiguration<Parameters<WorkflowFunction>[1]> ) =>
+    ( input: Parameters<WorkflowFunction>[0], config?: WorkflowInvocationConfiguration ) =>
     ReturnType<WorkflowFunction>;
 
 /**

package/src/tracing/tools/aggregate_trace_attributes.js

@@ -0,0 +1,118 @@
+/**
+ * Aggregate `attributes.cost` and `attributes.token_usage` across an entire trace tree.
+ *
+ * Walks every node in the tree, sums `attributes.cost.total` grouped by the emitting
+ * event name (inferred from node `kind` — see `eventNameForKind`), and sums
+ * `attributes.token_usage` across LLM nodes. Falls back to `output.usage` on
+ * legacy llm trace nodes that predate the `attributes.token_usage` write
+ * (see overview §1.2).
+ *
+ * @typedef {object} TraceAttributes
+ * @property {{ total: number, components: Array<{ name: string, value: number }> }} cost
+ * @property {{ inputTokens: number, outputTokens: number, cachedInputTokens: number, totalTokens: number }} tokenUsage
+ */
+
+const COST_EVENT_LLM = 'cost:llm:request';
+const COST_EVENT_HTTP = 'cost:http:request';
+const COST_EVENT_OTHER = 'other';
+
+/**
+ * Map a trace node `kind` to the canonical cost event name that would emit it.
+ * Unknown kinds bucket into `other` so future event sources still roll up cleanly.
+ *
+ * @param {string} kind
+ * @returns {string}
+ */
+const eventNameForKind = kind => {
+  if ( kind === 'llm' ) {
+    return COST_EVENT_LLM;
+  }
+  if ( kind === 'http' ) {
+    return COST_EVENT_HTTP;
+  }
+  return COST_EVENT_OTHER;
+};
+
+const isNumber = value => typeof value === 'number' && Number.isFinite( value );
+
+/**
+ * Pull token usage off an llm node, preferring the new attribute over the legacy
+ * `output.usage` fallback. Returns `null` when neither shape is present.
+ */
+const readTokenUsage = node => {
+  const attrUsage = node.attributes?.token_usage;
+  if ( attrUsage && typeof attrUsage === 'object' ) {
+    return attrUsage;
+  }
+  const legacyUsage = node.output?.usage;
+  if ( legacyUsage && typeof legacyUsage === 'object' ) {
+    return legacyUsage;
+  }
+  return null;
+};
+
+/**
+ * Recursively walk a trace tree depth-first, applying `visit` to each node.
+ */
+const walk = ( node, visit ) => {
+  if ( !node ) {
+    return;
+  }
+  visit( node );
+  for ( const child of node.children ?? [] ) {
+    walk( child, visit );
+  }
+};
+
+/**
+ * Build the aggregated `attributes` payload returned by `/trace-attributes`.
+ * Component buckets always appear in a stable order so callers can index them
+ * positionally if they want to.
+ *
+ * @param {object|null} root - The root NodeEntry returned by `buildTraceTree`.
+ * @returns {TraceAttributes}
+ */
+export default function aggregateTraceAttributes( root ) {
+  const costByEvent = new Map( [
+    [ COST_EVENT_LLM, 0 ],
+    [ COST_EVENT_HTTP, 0 ],
+    [ COST_EVENT_OTHER, 0 ]
+  ] );
+  const tokenUsage = { inputTokens: 0, outputTokens: 0, cachedInputTokens: 0, totalTokens: 0 };
+
+  walk( root, node => {
+    const cost = node.attributes?.cost;
+    if ( cost && isNumber( cost.total ) ) {
+      const eventName = eventNameForKind( node.kind );
+      costByEvent.set( eventName, ( costByEvent.get( eventName ) ?? 0 ) + cost.total );
+    }
+
+    if ( node.kind === 'llm' ) {
+      const usage = readTokenUsage( node );
+      if ( usage ) {
+        if ( isNumber( usage.inputTokens ) ) {
+          tokenUsage.inputTokens += usage.inputTokens;
+        }
+        if ( isNumber( usage.outputTokens ) ) {
+          tokenUsage.outputTokens += usage.outputTokens;
+        }
+        if ( isNumber( usage.cachedInputTokens ) ) {
+          tokenUsage.cachedInputTokens += usage.cachedInputTokens;
+        }
+        if ( isNumber( usage.totalTokens ) ) {
+          tokenUsage.totalTokens += usage.totalTokens;
+        }
+      }
+    }
+  } );
+
+  const components = Array.from( costByEvent, ( [ name, value ] ) => ( { name, value } ) );
+  const total = components.reduce( ( sum, { value } ) => sum + value, 0 );
+
+  return {
+    cost: { total, components },
+    tokenUsage
+  };
+}
+
+export { COST_EVENT_LLM, COST_EVENT_HTTP, COST_EVENT_OTHER };

package/src/tracing/tools/aggregate_trace_attributes.spec.js

@@ -0,0 +1,231 @@
+import { describe, it, expect } from 'vitest';
+import aggregateTraceAttributes, {
+  COST_EVENT_LLM,
+  COST_EVENT_HTTP,
+  COST_EVENT_OTHER
+} from './aggregate_trace_attributes.js';
+
+const node = ( { id, kind = 'step', attributes = {}, output, children = [] } ) => ( {
+  id,
+  kind,
+  name: id,
+  startedAt: 0,
+  endedAt: 0,
+  input: undefined,
+  output,
+  attributes,
+  children
+} );
+
+describe( 'aggregate_trace_attributes', () => {
+  it( 'returns zeros for a null root', () => {
+    const result = aggregateTraceAttributes( null );
+    expect( result.cost.total ).toBe( 0 );
+    expect( result.cost.components ).toEqual( [
+      { name: COST_EVENT_LLM, value: 0 },
+      { name: COST_EVENT_HTTP, value: 0 },
+      { name: COST_EVENT_OTHER, value: 0 }
+    ] );
+    expect( result.tokenUsage ).toEqual( {
+      inputTokens: 0, outputTokens: 0, cachedInputTokens: 0, totalTokens: 0
+    } );
+  } );
+
+  it( 'returns zeros for a tree with no cost or usage attributes', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      children: [ node( { id: 's1' } ), node( { id: 's2' } ) ]
+    } );
+    const result = aggregateTraceAttributes( root );
+    expect( result.cost.total ).toBe( 0 );
+    expect( result.tokenUsage.totalTokens ).toBe( 0 );
+  } );
+
+  it( 'buckets cost by node kind into llm / http / other components', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      children: [
+        node( { id: 'llm-1', kind: 'llm', attributes: { cost: { total: 0.20 } } } ),
+        node( { id: 'llm-2', kind: 'llm', attributes: { cost: { total: 0.10 } } } ),
+        node( { id: 'http-1', kind: 'http', attributes: { cost: { total: 0.50 } } } ),
+        // Unknown kind falls into the catch-all bucket
+        node( { id: 'step-1', kind: 'step', attributes: { cost: { total: 0.07 } } } )
+      ]
+    } );
+    const result = aggregateTraceAttributes( root );
+
+    const byName = Object.fromEntries( result.cost.components.map( c => [ c.name, c.value ] ) );
+    expect( byName[COST_EVENT_LLM] ).toBeCloseTo( 0.30, 10 );
+    expect( byName[COST_EVENT_HTTP] ).toBeCloseTo( 0.50, 10 );
+    expect( byName[COST_EVENT_OTHER] ).toBeCloseTo( 0.07, 10 );
+    expect( result.cost.total ).toBeCloseTo( 0.87, 10 );
+  } );
+
+  it( 'total equals the sum of all components', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      children: [
+        node( { id: 'llm-1', kind: 'llm', attributes: { cost: { total: 0.1234 } } } ),
+        node( { id: 'http-1', kind: 'http', attributes: { cost: { total: 0.0011 } } } )
+      ]
+    } );
+    const { cost } = aggregateTraceAttributes( root );
+    const sum = cost.components.reduce( ( s, c ) => s + c.value, 0 );
+    expect( cost.total ).toBeCloseTo( sum, 10 );
+  } );
+
+  it( 'sums token_usage across llm nodes from the attribute path', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      children: [
+        node( {
+          id: 'llm-1', kind: 'llm', attributes: {
+            token_usage: { inputTokens: 100, outputTokens: 20, cachedInputTokens: 5, totalTokens: 125 }
+          }
+        } ),
+        node( {
+          id: 'llm-2', kind: 'llm', attributes: {
+            token_usage: { inputTokens: 50, outputTokens: 10, cachedInputTokens: 1, totalTokens: 61 }
+          }
+        } )
+      ]
+    } );
+    const { tokenUsage } = aggregateTraceAttributes( root );
+    expect( tokenUsage ).toEqual( {
+      inputTokens: 150,
+      outputTokens: 30,
+      cachedInputTokens: 6,
+      totalTokens: 186
+    } );
+  } );
+
+  it( 'falls back to output.usage on legacy llm nodes that lack attributes.token_usage', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      children: [
+        // Legacy shape — usage lives on output.usage, no attributes.token_usage
+        node( {
+          id: 'llm-legacy',
+          kind: 'llm',
+          output: { result: '...', usage: { inputTokens: 200, outputTokens: 40, totalTokens: 240 } }
+        } )
+      ]
+    } );
+    const { tokenUsage } = aggregateTraceAttributes( root );
+    expect( tokenUsage.inputTokens ).toBe( 200 );
+    expect( tokenUsage.outputTokens ).toBe( 40 );
+    expect( tokenUsage.totalTokens ).toBe( 240 );
+    expect( tokenUsage.cachedInputTokens ).toBe( 0 );
+  } );
+
+  it( 'prefers attributes.token_usage over output.usage when both are present', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      children: [
+        node( {
+          id: 'llm-1',
+          kind: 'llm',
+          attributes: { token_usage: { inputTokens: 10, outputTokens: 2, totalTokens: 12 } },
+          output: { usage: { inputTokens: 999, outputTokens: 999, totalTokens: 999 } }
+        } )
+      ]
+    } );
+    const { tokenUsage } = aggregateTraceAttributes( root );
+    expect( tokenUsage.inputTokens ).toBe( 10 );
+    expect( tokenUsage.totalTokens ).toBe( 12 );
+  } );
+
+  it( 'ignores token_usage shapes on non-llm nodes', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      // attributes.token_usage on a non-llm node is intentionally ignored —
+      // only llm nodes contribute to the token-usage rollup today.
+      children: [
+        node( {
+          id: 'step-1', kind: 'step', attributes: {
+            token_usage: { inputTokens: 999, outputTokens: 999, totalTokens: 999 }
+          }
+        } )
+      ]
+    } );
+    const { tokenUsage } = aggregateTraceAttributes( root );
+    expect( tokenUsage.totalTokens ).toBe( 0 );
+  } );
+
+  it( 'aggregates a mixed tree with cost on http nodes and usage on llm nodes', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      children: [
+        node( {
+          id: 'llm-1',
+          kind: 'llm',
+          attributes: {
+            cost: { total: 0.0038 },
+            token_usage: { inputTokens: 2264, outputTokens: 411, cachedInputTokens: 100, totalTokens: 2775 }
+          }
+        } ),
+        node( {
+          id: 'http-1',
+          kind: 'http',
+          attributes: { cost: { total: 0.50 } }
+        } )
+      ]
+    } );
+    const result = aggregateTraceAttributes( root );
+
+    const byName = Object.fromEntries( result.cost.components.map( c => [ c.name, c.value ] ) );
+    expect( byName[COST_EVENT_LLM] ).toBeCloseTo( 0.0038, 10 );
+    expect( byName[COST_EVENT_HTTP] ).toBeCloseTo( 0.50, 10 );
+    expect( byName[COST_EVENT_OTHER] ).toBe( 0 );
+    expect( result.cost.total ).toBeCloseTo( 0.5038, 10 );
+
+    expect( result.tokenUsage ).toEqual( {
+      inputTokens: 2264,
+      outputTokens: 411,
+      cachedInputTokens: 100,
+      totalTokens: 2775
+    } );
+  } );
+
+  it( 'recurses through nested children', () => {
+    const root = node( {
+      id: 'wf',
+      kind: 'workflow',
+      children: [
+        node( {
+          id: 's1',
+          kind: 'step',
+          children: [
+            node( {
+              id: 'llm-1', kind: 'llm', attributes: {
+                cost: { total: 0.01 },
+                token_usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }
+              }
+            } )
+          ]
+        } )
+      ]
+    } );
+    const result = aggregateTraceAttributes( root );
+    expect( result.cost.total ).toBeCloseTo( 0.01, 10 );
+    expect( result.tokenUsage.totalTokens ).toBe( 15 );
+  } );
+
+  it( 'keeps the canonical component ordering: llm, http, other', () => {
+    const root = node( { id: 'wf', kind: 'workflow' } );
+    const { cost } = aggregateTraceAttributes( root );
+    expect( cost.components.map( c => c.name ) ).toEqual( [
+      COST_EVENT_LLM,
+      COST_EVENT_HTTP,
+      COST_EVENT_OTHER
+    ] );
+  } );
+} );

package/src/tracing/tools/index.js

@@ -0,0 +1,7 @@
+export { default as buildTraceTree } from './build_trace_tree.js';
+export {
+  default as aggregateTraceAttributes,
+  COST_EVENT_LLM,
+  COST_EVENT_HTTP,
+  COST_EVENT_OTHER
+} from './aggregate_trace_attributes.js';

package/src/worker/bundler_options.js

@@ -1,9 +1,40 @@
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import {
+  findPackageRoot,
+  isPathDescendentFromNodeModules,
+  packageExposesWorkflows
+} from './loader_tools.js';
 
 const __dirname = dirname( fileURLToPath( import.meta.url ) );
 const workerDir = __dirname; // sdk/core/src/worker
 const interfaceDir = join( __dirname, '..', 'interface' );
+const packagesWithWorkflowsMap = new Map();
+
+/**
+ * Skip loaders for most of `node_modules`, except packages that expose workflows.
+ */
+const excludeUnlessPackageExposeWorkflows = resource => {
+  // internal parts: exclude
+  if ( resource.startsWith( workerDir ) || resource.startsWith( interfaceDir ) ) {
+    return true;
+  }
+  // not node_modules/: include
+  if ( !isPathDescendentFromNodeModules( resource ) ) {
+    return false;
+  }
+
+  const rootPath = findPackageRoot( resource );
+  if ( !rootPath ) {
+    return true;
+  }
+
+  if ( !packagesWithWorkflowsMap.has( rootPath ) ) {
+    packagesWithWorkflowsMap.set( rootPath, packageExposesWorkflows( join( rootPath, 'package.json' ) ) );
+  }
+
+  return !packagesWithWorkflowsMap.get( rootPath );
+};
 
 export const webpackConfigHook = config => {
   // Prefer the "output-workflow-bundle" export condition when resolving packages.
@@ -23,8 +54,7 @@ export const webpackConfigHook = config => {
   // Validation loader (runs first)
   config.module.rules.push( {
     test: /\.js$/,
-    // Exclude node_modules and internal core worker files
-    exclude: resource => /node_modules/.test( resource ) || resource.startsWith( workerDir ) || resource.startsWith( interfaceDir ),
+    exclude: excludeUnlessPackageExposeWorkflows,
     enforce: 'pre',
     use: {
       loader: join( __dirname, './webpack_loaders/workflow_validator/index.mjs' )
@@ -33,8 +63,7 @@ export const webpackConfigHook = config => {
   // Use AST-based loader for rewriting steps/workflows
   config.module.rules.push( {
     test: /\.js$/,
-    // Exclude node_modules and internal core worker files
-    exclude: resource => /node_modules/.test( resource ) || resource.startsWith( workerDir ) || resource.startsWith( interfaceDir ),
+    exclude: excludeUnlessPackageExposeWorkflows,
     use: {
       loader: join( __dirname, './webpack_loaders/workflow_rewriter/index.mjs' )
     }

package/src/worker/bundler_options.spec.js

@@ -0,0 +1,62 @@
+import { describe, it, expect, afterEach } from 'vitest';
+import { mkdirSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { webpackConfigHook } from './bundler_options.js';
+
+const __dirname = dirname( fileURLToPath( import.meta.url ) );
+const TEMP_BASE = join( process.cwd(), 'sdk/core/temp_test_bundler_options' );
+
+afterEach( () => {
+  rmSync( TEMP_BASE, { recursive: true, force: true } );
+} );
+
+const buildExcludes = () => {
+  const config = webpackConfigHook( {} );
+  return config.module.rules.map( rule => rule.exclude );
+};
+
+const writePackageResource = ( packagePath, pkgJson ) => {
+  const resource = join( packagePath, 'lib', 'index.js' );
+  mkdirSync( dirname( resource ), { recursive: true } );
+  writeFileSync( join( packagePath, 'package.json' ), JSON.stringify( pkgJson ) );
+  writeFileSync( resource, 'export const x = 1;\n' );
+  return resource;
+};
+
+describe( 'webpackConfigHook loader excludes', () => {
+  it( 'keeps loaders enabled for project files outside node_modules', () => {
+    for ( const exclude of buildExcludes() ) {
+      expect( exclude( join( TEMP_BASE, 'src', 'workflow.js' ) ) ).toBe( false );
+    }
+  } );
+
+  it( 'excludes worker and interface internals', () => {
+    for ( const exclude of buildExcludes() ) {
+      expect( exclude( join( __dirname, 'loader.js' ) ) ).toBe( true );
+      expect( exclude( join( __dirname, '..', 'interface', 'index.js' ) ) ).toBe( true );
+    }
+  } );
+
+  it( 'keeps loaders enabled for packages that expose workflows', () => {
+    const resource = writePackageResource(
+      join( TEMP_BASE, 'node_modules', '@acme', 'catalog' ),
+      { name: '@acme/catalog', outputai: { workflows: { expose: true } } }
+    );
+
+    for ( const exclude of buildExcludes() ) {
+      expect( exclude( resource ) ).toBe( false );
+    }
+  } );
+
+  it( 'excludes packages that do not expose workflows', () => {
+    const resource = writePackageResource(
+      join( TEMP_BASE, 'node_modules', 'plain_lib' ),
+      { name: 'plain_lib', dependencies: { '@outputai/core': '1.0.0' } }
+    );
+
+    for ( const exclude of buildExcludes() ) {
+      expect( exclude( resource ) ).toBe( true );
+    }
+  } );
+} );