@outputai/core 0.4.1-next.6f25d10.0 → 0.4.1-next.c0b98d8.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@outputai/core",
-  "version": "0.4.1-next.6f25d10.0",
+  "version": "0.4.1-next.c0b98d8.0",
   "description": "The core module of the output framework",
   "type": "module",
   "exports": {
@@ -33,6 +33,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "3.1038.0",
+    "@aws-sdk/lib-storage": "3.1038.0",
     "@babel/generator": "7.29.1",
     "@babel/parser": "7.29.2",
     "@babel/traverse": "7.29.0",
@@ -42,6 +43,7 @@
     "@temporalio/common": "1.17.0",
     "@temporalio/worker": "1.17.0",
     "@temporalio/workflow": "1.17.0",
+    "json-stream-stringify": "3.1.6",
     "redis": "5.12.1",
     "stacktrace-parser": "0.1.11",
     "undici": "8.1.0",

package/src/tracing/processors/local/index.js

@@ -1,9 +1,11 @@
-import { appendFileSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { appendFileSync, mkdirSync, readdirSync, readFileSync, rmSync, createWriteStream } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'url';
 import buildTraceTree from '../../tools/build_trace_tree.js';
-import { safeFormatJSON } from '../../tools/utils.js';
 import { EOL } from 'node:os';
+import { JsonStreamStringify } from 'json-stream-stringify';
+
+import { pipeline } from 'stream/promises';
 
 const __dirname = dirname( fileURLToPath( import.meta.url ) );
 
@@ -109,7 +111,7 @@ export const init = () => {
  * @param {object} args.executionContext - Execution info: workflowId, workflowName, startTime
  * @returns {void}
  */
-export const exec = ( { entry, executionContext } ) => {
+export const exec = async ( { entry, executionContext } ) => {
   const { workflowId, workflowName, startTime } = executionContext;
   const tempFilePath = createTempFilePath( executionContext );
   addEntry( entry, tempFilePath );
@@ -126,7 +128,11 @@ export const exec = ( { entry, executionContext } ) => {
   const path = join( dir, buildTraceFilename( { startTime, workflowId } ) );
 
   mkdirSync( dir, { recursive: true } );
-  writeFileSync( path, safeFormatJSON( content ) + EOL, 'utf-8' );
+
+  await pipeline(
+    new JsonStreamStringify( content ),
+    createWriteStream( path )
+  );
 };
 
 /**

package/src/tracing/processors/local/index.spec.js

@@ -1,5 +1,4 @@
 import { describe, it, expect, vi, beforeEach } from 'vitest';
-import { EOL } from 'node:os';
 
 // In-memory fs mock store
 const store = { files: new Map() };
@@ -12,6 +11,7 @@ const appendFileSyncMock = vi.fn( ( path, data ) => {
 const readFileSyncMock = vi.fn( path => store.files.get( path ) ?? '' );
 const readdirSyncMock = vi.fn( () => [] );
 const rmSyncMock = vi.fn();
+const createWriteStreamMock = vi.fn( path => ( { path } ) );
 
 vi.mock( 'node:fs', () => ( {
   mkdirSync: mkdirSyncMock,
@@ -19,9 +19,36 @@ vi.mock( 'node:fs', () => ( {
   appendFileSync: appendFileSyncMock,
   readFileSync: readFileSyncMock,
   readdirSync: readdirSyncMock,
-  rmSync: rmSyncMock
+  rmSync: rmSyncMock,
+  createWriteStream: createWriteStreamMock
 } ) );
 
+const pipelineMock = vi.fn( async ( source, destination ) => {
+  const chunks = [];
+  for await ( const chunk of source ) {
+    chunks.push( Buffer.isBuffer( chunk ) ? chunk : Buffer.from( chunk ) );
+  }
+  store.files.set( destination.path, Buffer.concat( chunks ).toString( 'utf8' ) );
+} );
+vi.mock( 'node:stream/promises', () => ( { pipeline: pipelineMock } ) );
+
+vi.mock( 'json-stream-stringify', async () => {
+  const { Readable } = await import( 'node:stream' );
+  return {
+    JsonStreamStringify: class extends Readable {
+      constructor( body ) {
+        super();
+        this.body = body;
+      }
+
+      _read() {
+        this.push( JSON.stringify( this.body ) );
+        this.push( null );
+      }
+    }
+  };
+} );
+
 const buildTraceTreeMock = vi.fn( entries => ( { count: entries.length } ) );
 vi.mock( '../../tools/build_trace_tree.js', () => ( { default: buildTraceTreeMock } ) );
 
@@ -58,17 +85,18 @@ describe( 'tracing/processors/local', () => {
     const workflowId = 'id1';
     const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
 
-    exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
-    exec( { ...ctx, entry: childTick( 'child-1', startTime + 1 ) } );
-    exec( { ...ctx, entry: rootEnd( workflowId, startTime + 2 ) } );
+    await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
+    await exec( { ...ctx, entry: childTick( 'child-1', startTime + 1 ) } );
+    await exec( { ...ctx, entry: rootEnd( workflowId, startTime + 2 ) } );
 
     expect( buildTraceTreeMock ).toHaveBeenCalledTimes( 1 );
     expect( buildTraceTreeMock.mock.calls[0][0] ).toHaveLength( 3 );
 
-    expect( writeFileSyncMock ).toHaveBeenCalledTimes( 1 );
-    const [ writtenPath, content ] = writeFileSyncMock.mock.calls[0];
+    expect( createWriteStreamMock ).toHaveBeenCalledTimes( 1 );
+    expect( pipelineMock ).toHaveBeenCalledTimes( 1 );
+    const [ writtenPath ] = createWriteStreamMock.mock.calls[0];
     expect( writtenPath ).toMatch( /\/tmp\/project\/logs\/runs\/WF\// );
-    expect( JSON.parse( content.trim() ).count ).toBe( 3 );
+    expect( JSON.parse( store.files.get( writtenPath ) ).count ).toBe( 3 );
   } );
 
   it( 'exec(): does not build or write on non-flush entries', async () => {
@@ -79,11 +107,13 @@ describe( 'tracing/processors/local', () => {
     const workflowId = 'id1';
     const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
 
-    exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
-    exec( { ...ctx, entry: childTick( 'child-1', startTime + 1 ) } );
+    await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
+    await exec( { ...ctx, entry: childTick( 'child-1', startTime + 1 ) } );
 
     expect( buildTraceTreeMock ).not.toHaveBeenCalled();
     expect( writeFileSyncMock ).not.toHaveBeenCalled();
+    expect( createWriteStreamMock ).not.toHaveBeenCalled();
+    expect( pipelineMock ).not.toHaveBeenCalled();
   } );
 
   it( 'exec(): flushes on error action before root end', async () => {
@@ -94,12 +124,13 @@ describe( 'tracing/processors/local', () => {
     const workflowId = 'id1';
     const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
 
-    exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
-    exec( { ...ctx, entry: { id: 'step-1', action: 'error', timestamp: startTime + 1 } } );
+    await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
+    await exec( { ...ctx, entry: { id: 'step-1', action: 'error', timestamp: startTime + 1 } } );
 
     expect( buildTraceTreeMock ).toHaveBeenCalledTimes( 1 );
     expect( buildTraceTreeMock.mock.calls[0][0] ).toHaveLength( 2 );
-    expect( writeFileSyncMock ).toHaveBeenCalledTimes( 1 );
+    expect( createWriteStreamMock ).toHaveBeenCalledTimes( 1 );
+    expect( pipelineMock ).toHaveBeenCalledTimes( 1 );
   } );
 
   it( 'getDestination(): returns absolute path under callerDir logs', async () => {
@@ -128,11 +159,11 @@ describe( 'tracing/processors/local', () => {
     const workflowId = 'id1';
     const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
 
-    exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
-    exec( { ...ctx, entry: rootEnd( workflowId, startTime + 1 ) } );
+    await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
+    await exec( { ...ctx, entry: rootEnd( workflowId, startTime + 1 ) } );
 
-    expect( writeFileSyncMock ).toHaveBeenCalledTimes( 1 );
-    const [ writtenPath ] = writeFileSyncMock.mock.calls[0];
+    expect( createWriteStreamMock ).toHaveBeenCalledTimes( 1 );
+    const [ writtenPath ] = createWriteStreamMock.mock.calls[0];
 
     expect( writtenPath ).not.toContain( '/host/path/logs' );
     expect( writtenPath ).toMatch( /\/tmp\/project\/logs\/runs\/WF\// );
@@ -164,15 +195,15 @@ describe( 'tracing/processors/local', () => {
     const workflowName = 'test-workflow';
     const ctx = { executionContext: { workflowId, workflowName, startTime } };
 
-    exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
-    exec( { ...ctx, entry: rootEnd( workflowId, startTime + 1 ) } );
+    await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
+    await exec( { ...ctx, entry: rootEnd( workflowId, startTime + 1 ) } );
 
     const destination = getDestination( { startTime, workflowId, workflowName } );
 
-    const [ writtenPath, payload ] = writeFileSyncMock.mock.calls[0];
+    const [ writtenPath ] = createWriteStreamMock.mock.calls[0];
     expect( writtenPath ).not.toContain( '/Users/ben/project' );
     expect( writtenPath ).toMatch( /\/tmp\/project\/logs\/runs\/test-workflow\// );
-    expect( payload.endsWith( EOL ) ).toBe( true );
+    expect( JSON.parse( store.files.get( writtenPath ) ).count ).toBe( 2 );
 
     expect( destination ).toBe( '/Users/ben/project/logs/runs/test-workflow/2020-01-02-03-04-05-678Z_workflow-id-123.json' );
   } );

package/src/tracing/processors/s3/index.js

@@ -1,10 +1,9 @@
 import { upload } from './s3_client.js';
 import { getRedisClient } from './redis_client.js';
 import buildTraceTree from '../../tools/build_trace_tree.js';
-import { EOL } from 'node:os';
 import { loadEnv, getVars } from './configs.js';
 import { createChildLogger } from '#logger';
-import { safeFormatJSON } from '../../tools/utils.js';
+import { JsonStreamStringify } from 'json-stream-stringify';
 
 const log = createChildLogger( 'S3 Processor' );
 
@@ -100,9 +99,10 @@ export const exec = async ( { entry, executionContext } ) => {
     log.warn( 'Incomplete trace file discarded', { workflowId, error: 'incomplete_trace_file' } );
     return;
   }
+
   await upload( {
     key: getS3Key( { workflowId, workflowName, startTime } ),
-    content: safeFormatJSON( content ) + EOL
+    content: new JsonStreamStringify( content )
   } );
   await bustEntries( cacheKey );
 };

package/src/tracing/processors/s3/index.spec.js

@@ -27,6 +27,31 @@ vi.mock( './s3_client.js', () => ( { upload: uploadMock } ) );
 const buildTraceTreeMock = vi.fn( entries => ( { count: entries.length } ) );
 vi.mock( '../../tools/build_trace_tree.js', () => ( { default: buildTraceTreeMock } ) );
 
+vi.mock( 'json-stream-stringify', async () => {
+  const { Readable } = await import( 'node:stream' );
+  return {
+    JsonStreamStringify: class extends Readable {
+      constructor( body ) {
+        super();
+        this.body = body;
+      }
+
+      _read() {
+        this.push( JSON.stringify( this.body ) );
+        this.push( null );
+      }
+    }
+  };
+} );
+
+const streamToString = async stream => {
+  const chunks = [];
+  for await ( const chunk of stream ) {
+    chunks.push( Buffer.isBuffer( chunk ) ? chunk : Buffer.from( chunk ) );
+  }
+  return Buffer.concat( chunks ).toString( 'utf8' );
+};
+
 describe( 'tracing/processors/s3', () => {
   beforeEach( () => {
     vi.useFakeTimers();
@@ -74,7 +99,7 @@ describe( 'tracing/processors/s3', () => {
     expect( uploadMock ).toHaveBeenCalledTimes( 1 );
     const { key, content } = uploadMock.mock.calls[0][0];
     expect( key ).toMatch( /^WF\/2020\/01\/02\// );
-    expect( JSON.parse( content.trim() ).count ).toBe( 3 );
+    expect( JSON.parse( await streamToString( content ) ).count ).toBe( 3 );
     expect( delMock ).toHaveBeenCalledTimes( 1 );
     expect( delMock ).toHaveBeenCalledWith( 'traces/WF/id1' );
   } );

package/src/tracing/processors/s3/s3_client.js

@@ -1,4 +1,5 @@
-import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
+import { S3Client } from '@aws-sdk/client-s3';
+import { Upload } from '@aws-sdk/lib-storage';
 import { getVars } from './configs.js';
 
 const state = { s3Client: null };
@@ -21,7 +22,14 @@ const getS3Client = () => {
  * Upload given file to S3
  * @param {object} args
  * @param {string} key - S3 file key
- * @param {string} content - File content
+ * @param {string|import('node:stream').Readable} content - File content
  */
 export const upload = ( { key, content } ) =>
-  getS3Client().send( new PutObjectCommand( { Bucket: getVars().remoteS3Bucket, Key: key, Body: content } ) );
+  new Upload( {
+    client: getS3Client(),
+    params: {
+      Bucket: getVars().remoteS3Bucket,
+      Key: key,
+      Body: content
+    }
+  } ).done();

package/src/tracing/processors/s3/s3_client.spec.js

@@ -9,24 +9,27 @@ vi.mock( '#utils', () => ( {
 const getVarsMock = vi.fn();
 vi.mock( './configs', () => ( { getVars: () => getVarsMock() } ) );
 
-const sendMock = vi.fn();
 const ctorState = { args: null };
 class S3ClientMock {
   constructor( args ) {
     ctorState.args = args;
-  } send = sendMock;
-}
-class PutObjectCommandMock {
-  constructor( input ) {
-    this.input = input;
   }
 }
-
 vi.mock( '@aws-sdk/client-s3', () => ( {
-  S3Client: S3ClientMock,
-  PutObjectCommand: PutObjectCommandMock
+  S3Client: S3ClientMock
 } ) );
 
+const uploadDoneMock = vi.fn();
+const uploadCtorState = { args: [] };
+class UploadMock {
+  constructor( args ) {
+    uploadCtorState.args.push( args );
+  }
+
+  done = uploadDoneMock;
+}
+vi.mock( '@aws-sdk/lib-storage', () => ( { Upload: UploadMock } ) );
+
 async function loadModule() {
   vi.resetModules();
   return import( './s3_client.js' );
@@ -35,6 +38,9 @@ async function loadModule() {
 describe( 'tracing/processors/s3/s3_client', () => {
   beforeEach( () => {
     vi.clearAllMocks();
+    ctorState.args = null;
+    uploadCtorState.args = [];
+    uploadDoneMock.mockResolvedValue( undefined );
     getVarsMock.mockReturnValue( {
       awsRegion: 'us-east-1',
       awsAccessKeyId: 'id',
@@ -48,15 +54,21 @@ describe( 'tracing/processors/s3/s3_client', () => {
 
     await upload( { key: 'wf/key.json', content: '{"a":1}' } );
 
-    expect( ctorState.args ).toEqual( { region: 'us-east-1', credentials: { secretAccessKey: 'sek', accessKeyId: 'id' } } );
-    expect( sendMock ).toHaveBeenCalledTimes( 1 );
-    const cmd = sendMock.mock.calls[0][0];
-    expect( cmd ).toBeInstanceOf( PutObjectCommandMock );
-    expect( cmd.input ).toEqual( { Bucket: 'bucket', Key: 'wf/key.json', Body: '{"a":1}' } );
+    expect( ctorState.args ).toEqual( {
+      region: 'us-east-1',
+      credentials: { secretAccessKey: 'sek', accessKeyId: 'id' }
+    } );
+    expect( uploadCtorState.args ).toHaveLength( 1 );
+    expect( uploadCtorState.args[0] ).toEqual( {
+      client: expect.any( S3ClientMock ),
+      params: { Bucket: 'bucket', Key: 'wf/key.json', Body: '{"a":1}' }
+    } );
+    expect( uploadDoneMock ).toHaveBeenCalledTimes( 1 );
 
     // subsequent upload uses cached client
     await upload( { key: 'wf/key2.json', content: '{}' } );
-    expect( sendMock ).toHaveBeenCalledTimes( 2 );
+    expect( uploadCtorState.args ).toHaveLength( 2 );
+    expect( uploadDoneMock ).toHaveBeenCalledTimes( 2 );
   } );
 } );
 

package/src/tracing/tools/utils.js

@@ -19,31 +19,3 @@ export const serializeError = error =>
     message: error.message,
     stack: error.stack
   };
-
-/**
- * Tries to stringify an object to an indented JSON string.
- * If its byte size is bigger than threshold returns a plain JSON string without formatting.
- *
- * @param {object|array} content
- * @param {*} [threshold] - The max allowed size to try to stringify with formatting (in bytes). Default is 50mb
- * @returns {string} String representation of the object
- */
-export const safeFormatJSON = ( content, threshold = 50 * 1024 * 1024 /* 50mb */ ) => {
-  const plainString = JSON.stringify( content );
-  const plainStringSize = Buffer.byteLength( plainString, 'utf8' );
-
-  if ( plainStringSize > threshold ) {
-    return plainString;
-  }
-  try {
-    return JSON.stringify( content, undefined, 2 );
-  } catch ( error ) {
-    // Only handles this specific error because other common parsing errors like:
-    // "TypeError: cyclic object value" and "RangeError: Maximum call stack size exceeded"
-    // would have been thrown on the first parsing.
-    if ( error instanceof RangeError && error.message === 'Invalid string length' ) {
-      return plainString;
-    }
-    throw error;
-  }
-};

package/src/tracing/tools/utils.spec.js

@@ -1,15 +1,5 @@
-import { describe, it, expect, vi } from 'vitest';
-import { safeFormatJSON, serializeError } from './utils.js';
-
-const isPrettyStringifyCall = args => args.length >= 3 && args[2] === 2;
-
-/** @param {number} targetBytes UTF-8 size of compact JSON.stringify( { a: "<xs>" } ) */
-const objectWithCompactByteLength = targetBytes => {
-  const sample = { a: '' };
-  const overhead = Buffer.byteLength( JSON.stringify( sample ), 'utf8' );
-  const repeat = Math.max( 0, targetBytes - overhead );
-  return { a: 'x'.repeat( repeat ) };
-};
+import { describe, it, expect } from 'vitest';
+import { serializeError } from './utils.js';
 
 describe( 'tracing/utils', () => {
   it( 'serializeError unwraps causes and keeps message/stack', () => {
@@ -21,126 +11,4 @@ describe( 'tracing/utils', () => {
     expect( out.message ).toBe( 'inner' );
     expect( typeof out.stack ).toBe( 'string' );
   } );
-
-  describe( 'safeFormatJSON', () => {
-    it( 'formats small objects with indentation when under threshold', () => {
-      const content = { a: 1, b: [ 2, 3 ] };
-      const out = safeFormatJSON( content, 10_000 );
-
-      expect( out ).toContain( '\n' );
-      expect( out ).toMatch( /^\{\n/ );
-      expect( JSON.parse( out ) ).toEqual( content );
-    } );
-
-    it( 'formats small arrays with indentation when under threshold', () => {
-      const content = [ 1, { nested: true } ];
-      const out = safeFormatJSON( content, 10_000 );
-
-      expect( out ).toContain( '\n' );
-      expect( out.trimStart() ).toMatch( /^\[/ );
-      expect( JSON.parse( out ) ).toEqual( content );
-    } );
-
-    it( 'returns compact JSON when compact UTF-8 size is strictly greater than threshold', () => {
-      const content = objectWithCompactByteLength( 40 );
-      const compact = JSON.stringify( content );
-      expect( Buffer.byteLength( compact, 'utf8' ) ).toBe( 40 );
-
-      const out = safeFormatJSON( content, 39 );
-      expect( out ).toBe( compact );
-      expect( out ).not.toContain( '\n  ' );
-      expect( JSON.parse( out ) ).toEqual( content );
-    } );
-
-    it( 'uses pretty JSON when compact UTF-8 size equals threshold', () => {
-      const content = objectWithCompactByteLength( 40 );
-      const compact = JSON.stringify( content );
-      expect( Buffer.byteLength( compact, 'utf8' ) ).toBe( 40 );
-
-      const out = safeFormatJSON( content, 40 );
-      expect( out ).not.toBe( compact );
-      expect( out ).toContain( '\n' );
-      expect( JSON.parse( out ) ).toEqual( content );
-    } );
-
-    it( 'uses UTF-8 byte length for threshold, not JavaScript string length', () => {
-      const content = { label: 'éclair' };
-      const compact = JSON.stringify( content );
-      expect( compact.length ).toBeLessThan( Buffer.byteLength( compact, 'utf8' ) );
-
-      const bytes = Buffer.byteLength( compact, 'utf8' );
-      const outCompact = safeFormatJSON( content, bytes - 1 );
-      expect( outCompact ).toBe( compact );
-
-      const outPretty = safeFormatJSON( content, bytes + 100 );
-      expect( outPretty ).toContain( '\n' );
-      expect( JSON.parse( outPretty ) ).toEqual( content );
-    } );
-
-    it( 'round-trips empty object and primitives for both branches', () => {
-      const tiny = {};
-      const pretty = safeFormatJSON( tiny, 100 );
-      expect( JSON.parse( pretty ) ).toEqual( tiny );
-
-      const forcedCompact = safeFormatJSON( tiny, 0 );
-      expect( JSON.parse( forcedCompact ) ).toEqual( tiny );
-    } );
-
-    it( 'returns compact JSON when pretty stringify throws Invalid string length', () => {
-      const content = { a: 1 };
-      const compact = JSON.stringify( content );
-      const origStringify = JSON.stringify.bind( JSON );
-
-      const spy = vi.spyOn( JSON, 'stringify' ).mockImplementation( ( ...args ) => {
-        if ( isPrettyStringifyCall( args ) ) {
-          throw new RangeError( 'Invalid string length' );
-        }
-        return origStringify( ...args );
-      } );
-
-      try {
-        const out = safeFormatJSON( content, 10_000 );
-        expect( out ).toBe( compact );
-        expect( JSON.parse( out ) ).toEqual( content );
-      } finally {
-        spy.mockRestore();
-      }
-    } );
-
-    it( 'rethrows RangeError when message is not Invalid string length', () => {
-      const content = { a: 1 };
-      const origStringify = JSON.stringify.bind( JSON );
-
-      const spy = vi.spyOn( JSON, 'stringify' ).mockImplementation( ( ...args ) => {
-        if ( isPrettyStringifyCall( args ) ) {
-          throw new RangeError( 'not the string length error' );
-        }
-        return origStringify( ...args );
-      } );
-
-      try {
-        expect( () => safeFormatJSON( content, 10_000 ) ).toThrow( RangeError );
-      } finally {
-        spy.mockRestore();
-      }
-    } );
-
-    it( 'rethrows non-RangeError from pretty stringify', () => {
-      const content = { a: 1 };
-      const origStringify = JSON.stringify.bind( JSON );
-
-      const spy = vi.spyOn( JSON, 'stringify' ).mockImplementation( ( ...args ) => {
-        if ( isPrettyStringifyCall( args ) ) {
-          throw new TypeError( 'cyclic structure' );
-        }
-        return origStringify( ...args );
-      } );
-
-      try {
-        expect( () => safeFormatJSON( content, 10_000 ) ).toThrow( TypeError );
-      } finally {
-        spy.mockRestore();
-      }
-    } );
-  } );
 } );