@output.ai/core 0.1.0 → 0.1.2

package/src/interface/zod_integration.spec.js

@@ -410,12 +410,12 @@ describe( 'Zod Schema Integration Tests', () => {
         inputSchema,
         fn: async input => {
           switch ( input.action ) {
-          case 'create':
-            return `Creating ${input.type}: ${input.name}`;
-          case 'delete':
-            return `Deleting item ${input.id}`;
-          default:
-            throw new Error( 'Unknown action' );
+            case 'create':
+              return `Creating ${input.type}: ${input.name}`;
+            case 'delete':
+              return `Deleting item ${input.id}`;
+            default:
+              throw new Error( 'Unknown action' );
           }
         }
       } );

package/src/internal_activities/index.js

@@ -1,11 +1,7 @@
 import { FatalError } from '#errors';
-import { existsSync, readFileSync, readdirSync } from 'node:fs';
-import { join } from 'node:path';
-import { setMetadata } from '../interface/metadata.js';
+import { setMetadata } from '#utils';
 import { ComponentType } from '#consts';
 
-const callerDir = process.argv[2];
-
 /**
  * Send a post to a given URL
  *
@@ -41,31 +37,3 @@ export const sendWebhook = async ( { url, workflowId, payload } ) => {
 };
 
 setMetadata( sendWebhook, { type: ComponentType.INTERNAL_STEP } );
-
-/**
- * Read the trace file of a given execution and returns the content
- *
- * @param {object} options
- * @param {string} options.traceId - The is of the trace
- * @param {string} options.traceHelm - The helm of the trace file
- * @returns {string[]} Each line of the trace file
- */
-export const readTraceFile = async ( { traceId, traceHelm } ) => {
-  const dir = join( callerDir, 'logs', 'runs', traceHelm );
-
-  if ( !existsSync( dir ) ) {
-    console.log( '[Core.ReadTraceFile]', 'Trace folder not found', dir );
-    return [];
-  }
-
-  const fileName = readdirSync( dir ).find( f => f.endsWith( `-${traceId}.raw` ) );
-
-  if ( !fileName ) {
-    console.log( '[Core.ReadTraceFile]', 'Trace file not found', { traceId, traceHelm } );
-    return [];
-  }
-
-  return readFileSync( join( dir, fileName ), 'utf-8' ).split( '\n' );
-};
-
-setMetadata( readTraceFile, { type: ComponentType.INTERNAL_STEP, skipTrace: true } );

package/src/tracing/index.d.ts

@@ -17,8 +17,8 @@ export declare const Tracing: {
    *
    * @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
    * @param {string} kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
-   * @param {string} name - The human friendly name of the Event: eg: query, request, create.
-   * @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
+   * @param {string} name - The human friendly name of the Event: query, request, create.
+   * @param {any} details - All details attached to this Event Phase. DB queried records, HTTP response body.
    * @returns {void}
    */
   addEventStart( args: { id: string; kind: string; name: string; details: any } ): void; // eslint-disable-line @typescript-eslint/no-explicit-any
@@ -29,7 +29,7 @@ export declare const Tracing: {
    * It needs to use the same id of the start phase.
    *
    * @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
-   * @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
+   * @param {any} details - All details attached to this Event Phase. DB queried records, HTTP response body.
    * @returns {void}
    */
   addEventEnd( args: { id: string; details: any } ): void; // eslint-disable-line @typescript-eslint/no-explicit-any
@@ -40,7 +40,7 @@ export declare const Tracing: {
    * It needs to use the same id of the start phase.
    *
    * @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
-   * @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
+   * @param {any} details - All details attached to this Event Phase. DB queried records, HTTP response body.
    * @returns {void}
    */
   addEventError( args: { id: string; details: any } ): void; // eslint-disable-line @typescript-eslint/no-explicit-any

package/src/tracing/index.js

@@ -1,116 +1,4 @@
-import { Storage } from '#async_storage';
-import { mkdirSync, existsSync, readdirSync, appendFileSync } from 'node:fs';
-import { join } from 'path';
-import { EOL } from 'os';
-import { buildLogTree } from './tracer_tree.js';
-import { serializeError } from './utils.js';
-
-const callerDir = process.argv[2];
-// It is is isolated here instead of #configs to allow this module to be exported without all other configs requirements
-const tracingEnabled = [ '1', 'true', 'on' ].includes( process.env.TRACING_ENABLED );
-
-/**
- * Trace nomenclature
- *
- * Trace - The collection of Events;
- * Event - Any entry in the Trace file, must have the two phases START and END or ERROR;
- * Phase - An specific part of an Event, either START or the conclusive END or ERROR;
- */
-
-/**
- * Adds the Trace Event Phase to the trace file
- *
- * @param {string} phase - The phase
- * @param {object} options - All the trace fields
- * @returns {void}
- */
-function addEventPhase( phase, { kind, details, id, name, parentId, traceId, traceHelm } ) {
-  if ( !tracingEnabled || name === 'catalog' ) {
-    return;
-  }
-
-  const parsedDetails = details instanceof Error ? serializeError( details ) : details;
-  const timestamp = Date.now();
-  const entry = { phase, kind, details: parsedDetails, id, name, timestamp, parentId };
-  const outputDir = join( callerDir, 'logs', 'runs', traceHelm );
-
-  if ( !existsSync( outputDir ) ) {
-    mkdirSync( outputDir, { recursive: true } );
-  }
-
-  const suffix = `-${traceId}.raw`;
-  const logFile = readdirSync( outputDir ).find( f => f.endsWith( suffix ) ) ?? `${new Date( timestamp ).toISOString()}-${suffix}`;
-  const logPath = join( outputDir, logFile );
-
-  appendFileSync( logPath, JSON.stringify( entry ) + EOL, 'utf-8' );
-  buildLogTree( logPath );
-};
-
-/**
- * Internal use only
- *
- * Adds the start phase of a new event at the default trace for the current workflow.
- *
- * @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
- * @param {string} kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
- * @param {string} name - The human friendly name of the Event: eg: query, request, create.
- * @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
- * @param {string} parentId - The parent Event, used to build a three.
- * @param {string} traceId - The traceId, which identifies from which trace this Event belongs.
- * @param {string} traceHelm - The trace helm, which is a taxonomical naming of the Trace.
- * @returns {void}
- */
-export const addEventStart = options => addEventPhase( 'start', options );
-
-/**
- * Internal use only
- *
- * Adds the end phase at an event at the default trace for the current workflow.
- *
- * It needs to use the same id of the start phase.
- *
- * @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
- * @param {string} kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
- * @param {string} name - The human friendly name of the Event: eg: query, request, create.
- * @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
- * @param {string} parentId - The parent Event, used to build a three.
- * @param {string} traceId - The traceId, which identifies from which trace this Event belongs.
- * @param {string} traceHelm - The trace helm, which is a taxonomical naming of the Trace.
- * @returns {void}
- */
-export const addEventEnd = options => addEventPhase( 'end', options );
-
-/**
- * Internal use only
- *
- * Adds the error phase at an event as error at the default trace for the current workflow.
- *
- * It needs to use the same id of the start phase.
- *
- * @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
- * @param {string} kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
- * @param {string} name - The human friendly name of the Event: eg: query, request, create.
- * @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
- * @param {string} parentId - The parent Event, used to build a three.
- * @param {string} traceId - The traceId, which identifies from which trace this Event belongs.
- * @param {string} traceHelm - The trace helm, which is a taxonomical naming of the Trace.
- * @returns {void}
- */
-export const addEventError = options => addEventPhase( 'error', options );
-
-/**
- * Adds an Event Phase reading traceId, traceHelm and parentId from the context.
- *
- * @param {object} options - The common trace configurations
- */
-function addEventPhaseWithContext( phase, options ) {
-  const storeContent = Storage.load();
-  if ( !storeContent ) { // means this was called from a unit test
-    return;
-  }
-  const { parentId, traceId, traceHelm } = storeContent;
-  addEventPhase( phase, { ...options, parentId, traceId, traceHelm } );
-};
+import { addEventPhaseWithContext } from './trace_engine.js';
 
 /**
  * The public namespace for tracing
@@ -122,10 +10,11 @@ export const Tracing = {
   /**
    * Adds the start phase of a new event at the default trace for the current workflow.
    *
-   * @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
-   * @param {string} kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
-   * @param {string} name - The human friendly name of the Event: eg: query, request, create.
-   * @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
+   * @param {object} args
+   * @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
+   * @param {string} args.kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
+   * @param {string} args.name - The human friendly name of the Event: query, request, create.
+   * @param {object} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
    * @returns {void}
    */
   addEventStart: ( { id, kind, name, details } ) => addEventPhaseWithContext( 'start', { kind, name, details, id } ),
@@ -135,8 +24,9 @@ export const Tracing = {
    *
    * It needs to use the same id of the start phase.
    *
-   * @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
-   * @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
+   * @param {object} args
+   * @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
+   * @param {object} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
    * @returns {void}
    */
   addEventEnd: ( { id, details } ) => addEventPhaseWithContext( 'end', { id, details } ),
@@ -146,8 +36,9 @@ export const Tracing = {
    *
    * It needs to use the same id of the start phase.
    *
-   * @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
-   * @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
+   * @param {object} args
+   * @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
+   * @param {object} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
    * @returns {void}
    */
   addEventError: ( { id, details } ) => addEventPhaseWithContext( 'error', { id, details } )

package/src/tracing/internal_interface.js

@@ -0,0 +1,66 @@
+import { addEventPhase, init } from './trace_engine.js';
+
+/**
+ * Init method, if not called, no processors are attached and trace functions are dummy
+ */
+export { init };
+
+/**
+ * Trace nomenclature
+ *
+ * Trace - The collection of Events;
+ * Event - Any entry in the Trace file, must have the two phases START and END or ERROR;
+ * Phase - An specific part of an Event, either START or the conclusive END or ERROR;
+ */
+
+/**
+ * Internal use only
+ *
+ * Adds the start phase of a new event at the default trace for the current workflow.
+ *
+ * @param {object} args
+ * @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
+ * @param {string} args.kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
+ * @param {string} args.name - The human friendly name of the Event: query, request, create.
+ * @param {any} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
+ * @param {string} args.parentId - The parent Event, used to build a three.
+ * @param {object} args.executionContext - The original execution context from the workflow
+ * @returns {void}
+ */
+export const addEventStart = options => addEventPhase( 'start', options );
+
+/**
+ * Internal use only
+ *
+ * Adds the end phase at an event at the default trace for the current workflow.
+ *
+ * It needs to use the same id of the start phase.
+ *
+ * @param {object} args
+ * @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
+ * @param {string} args.kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
+ * @param {string} args.name - The human friendly name of the Event: query, request, create.
+ * @param {any} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
+ * @param {string} args.parentId - The parent Event, used to build a three.
+ * @param {object} args.executionContext - The original execution context from the workflow
+ * @returns {void}
+ */
+export const addEventEnd = options => addEventPhase( 'end', options );
+
+/**
+ * Internal use only
+ *
+ * Adds the error phase at an event as error at the default trace for the current workflow.
+ *
+ * It needs to use the same id of the start phase.
+ *
+ * @param {object} args
+ * @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
+ * @param {string} args.kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
+ * @param {string} args.name - The human friendly name of the Event: query, request, create.
+ * @param {any} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
+ * @param {string} args.parentId - The parent Event, used to build a three.
+ * @param {object} args.executionContext - The original execution context from the workflow
+ * @returns {void}
+ */
+export const addEventError = options => addEventPhase( 'error', options );

package/src/tracing/processors/local/index.js

@@ -0,0 +1,50 @@
+import { appendFileSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'url';
+import buildTraceTree from '../../tools/build_trace_tree.js';
+import { EOL } from 'node:os';
+
+const oneWeekInMS = 1000 * 60 * 60 * 24 * 7;
+const __dirname = dirname( fileURLToPath( import.meta.url ) );
+const tempDir = join( __dirname, 'temp', 'traces' );
+
+const accumulate = ( { entry, executionContext: { workflowId, startTime } } ) => {
+  const path = join( tempDir, `${startTime}_${workflowId}.trace` );
+  appendFileSync( path, JSON.stringify( entry ) + EOL, 'utf-8' );
+  return readFileSync( path, 'utf-8' ).split( EOL ).slice( 0, -1 ).map( v => JSON.parse( v ) );
+};
+
+const cleanupOldTempFiles = ( threshold = Date.now() - oneWeekInMS ) =>
+  readdirSync( tempDir )
+    .filter( f => +f.split( '_' )[0] < threshold )
+    .forEach( f => rmSync( join( tempDir, f ) ) );
+
+/**
+ * Init this processor
+ */
+export const init = () => {
+  mkdirSync( tempDir, { recursive: true } );
+  cleanupOldTempFiles();
+};
+
+/**
+ * Execute this processor:
+ *
+ * Persist a trace tree file to local file system, updating upon each new entry
+ *
+ * @param {object} args
+ * @param {object} entry - Trace event phase
+ * @param {object} executionContext - Execution info: workflowId, workflowName, startTime
+ * @returns
+ */
+export const exec = ( { entry, executionContext } ) => {
+  const { workflowId, workflowName, startTime } = executionContext;
+  const content = buildTraceTree( accumulate( { entry, executionContext } ) );
+
+  const timestamp = new Date( startTime ).toISOString().replace( /[:T.]/g, '-' );
+  const dir = join( process.argv[2], 'logs', 'runs', workflowName );
+  const path = join( dir, `${timestamp}_${workflowId}.json` );
+
+  mkdirSync( dir, { recursive: true } );
+  writeFileSync( path, JSON.stringify( content, undefined, 2 ) + EOL, 'utf-8' );
+};

package/src/tracing/processors/local/index.spec.js

@@ -0,0 +1,67 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+// In-memory fs mock store
+const store = { files: new Map() };
+const mkdirSyncMock = vi.fn();
+const writeFileSyncMock = vi.fn();
+const appendFileSyncMock = vi.fn( ( path, data ) => {
+  const prev = store.files.get( path ) ?? '';
+  store.files.set( path, prev + data );
+} );
+const readFileSyncMock = vi.fn( path => store.files.get( path ) ?? '' );
+const readdirSyncMock = vi.fn( () => [] );
+const rmSyncMock = vi.fn();
+
+vi.mock( 'node:fs', () => ( {
+  mkdirSync: mkdirSyncMock,
+  writeFileSync: writeFileSyncMock,
+  appendFileSync: appendFileSyncMock,
+  readFileSync: readFileSyncMock,
+  readdirSync: readdirSyncMock,
+  rmSync: rmSyncMock
+} ) );
+
+const buildTraceTreeMock = vi.fn( entries => ( { count: entries.length } ) );
+vi.mock( '../../tools/build_trace_tree.js', () => ( { default: buildTraceTreeMock } ) );
+
+describe( 'tracing/processors/local', () => {
+  beforeEach( () => {
+    vi.clearAllMocks();
+    store.files.clear();
+    process.argv[2] = '/tmp/project';
+  } );
+
+  it( 'init(): creates temp dir and cleans up old files', async () => {
+    const { init } = await import( './index.js' );
+
+    const now = Date.now();
+    readdirSyncMock.mockReturnValue( [ `${now - ( 8 * 24 * 60 * 60 * 1000 )}_old.trace`, `${now}_new.trace` ] );
+
+    init();
+
+    expect( mkdirSyncMock ).toHaveBeenCalledWith( expect.stringMatching( /temp\/traces$/ ), { recursive: true } );
+    expect( rmSyncMock ).toHaveBeenCalledTimes( 1 );
+  } );
+
+  it( 'exec(): accumulates entries and writes aggregated tree', async () => {
+    const { exec, init } = await import( './index.js' );
+    init();
+
+    const startTime = Date.parse( '2020-01-02T03:04:05.678Z' );
+    const ctx = { executionContext: { workflowId: 'id1', workflowName: 'WF', startTime } };
+
+    exec( { ...ctx, entry: { name: 'A', phase: 'start', timestamp: startTime } } );
+    exec( { ...ctx, entry: { name: 'A', phase: 'tick', timestamp: startTime + 1 } } );
+    exec( { ...ctx, entry: { name: 'A', phase: 'end', timestamp: startTime + 2 } } );
+
+    // buildTraceTree called with 1, 2, 3 entries respectively
+    expect( buildTraceTreeMock ).toHaveBeenCalledTimes( 3 );
+    expect( buildTraceTreeMock.mock.calls.at( -1 )[0].length ).toBe( 3 );
+
+    expect( writeFileSyncMock ).toHaveBeenCalledTimes( 3 );
+    const [ writtenPath, content ] = writeFileSyncMock.mock.calls.at( -1 );
+    expect( writtenPath ).toMatch( /\/logs\/runs\/WF\// );
+    expect( JSON.parse( content.trim() ).count ).toBe( 3 );
+  } );
+} );
+

package/src/tracing/processors/s3/index.js

@@ -0,0 +1,51 @@
+import { upload } from './s3_client.js';
+import { getRedisClient } from './redis_client.js';
+import buildTraceTree from '../../tools/build_trace_tree.js';
+import { EOL } from 'node:os';
+
+const oneMonthInSeconds = 60 * 60 * 24 * 30;
+
+const accumulate = async ( { entry, executionContext: { workflowName, workflowId } } ) => {
+  const key = `traces/${workflowName}/${workflowId}`;
+  const transaction = ( await getRedisClient() ).multi();
+
+  transaction.zAdd( key, [
+    { score: entry.timestamp, value: JSON.stringify( entry ) }
+  ], { NX: true } );
+  transaction.expire( key, oneMonthInSeconds, 'GT' );
+  transaction.zRange( key, 0, -1 );
+  const [ ,, zList ] = await transaction.exec();
+  return zList.map( v => JSON.parse( v ) );
+};
+
+const getS3Key = ( { startTime, workflowId, workflowName } ) => {
+  const isoDate = new Date( startTime ).toISOString();
+  const [ year, month, day ] = isoDate.split( /\D/, 3 );
+  const timeStamp = isoDate.replace( /[:T.]/g, '-' );
+  return `${workflowName}/${year}/${month}/${day}/${timeStamp}_${workflowId}.json`;
+};
+
+/**
+ * Init this processor
+ */
+export const init = async () => {
+  await getRedisClient();
+};
+
+/**
+ * Execute this processor: send a complete trace tree file to S3 when the workflow finishes
+ *
+ * @param {object} args
+ * @param {object} entry - Trace event phase
+ * @param {object} executionContext - Execution info: workflowId, workflowName, startTime
+ */
+export const exec = async ( { entry, executionContext } ) => {
+  const { workflowName, workflowId, startTime } = executionContext;
+  const content = buildTraceTree( await accumulate( { entry, executionContext } ) );
+
+  const isRootWorkflowEnd = !entry.parentId && entry.phase !== 'start';
+  return isRootWorkflowEnd ? upload( {
+    key: getS3Key( { workflowId, workflowName, startTime } ),
+    content: JSON.stringify( content, undefined, 2 ) + EOL
+  } ) : 0;
+};

package/src/tracing/processors/s3/index.spec.js

@@ -0,0 +1,64 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+const redisMulti = {
+  zAdd: vi.fn().mockReturnThis(),
+  expire: vi.fn().mockReturnThis(),
+  zRange: vi.fn().mockReturnThis(),
+  exec: vi.fn()
+};
+const getRedisClientMock = vi.fn( async () => ( { multi: () => redisMulti } ) );
+vi.mock( './redis_client.js', () => ( { getRedisClient: getRedisClientMock } ) );
+
+const uploadMock = vi.fn();
+vi.mock( './s3_client.js', () => ( { upload: uploadMock } ) );
+
+const buildTraceTreeMock = vi.fn( entries => ( { count: entries.length } ) );
+vi.mock( '../../tools/build_trace_tree.js', () => ( { default: buildTraceTreeMock } ) );
+
+describe( 'tracing/processors/s3', () => {
+  beforeEach( () => {
+    vi.clearAllMocks();
+    process.env.TRACE_REMOTE_S3_BUCKET = 'bkt';
+  } );
+
+  it( 'init(): ensures redis client is created', async () => {
+    const { init } = await import( './index.js' );
+    await init();
+    expect( getRedisClientMock ).toHaveBeenCalledTimes( 1 );
+  } );
+
+  it( 'exec(): accumulates via redis, uploads only on root workflow end', async () => {
+    const { exec } = await import( './index.js' );
+    const startTime = Date.parse( '2020-01-02T03:04:05.678Z' );
+    const ctx = { executionContext: { workflowId: 'id1', workflowName: 'WF', startTime } };
+
+    // Redis will return progressively larger sorted sets
+    redisMulti.exec
+      .mockResolvedValueOnce( [ , , [ JSON.stringify( { name: 'A', phase: 'start', timestamp: startTime } ) ] ] )
+      .mockResolvedValueOnce( [ , , [
+        JSON.stringify( { name: 'A', phase: 'start', timestamp: startTime } ),
+        JSON.stringify( { name: 'A', phase: 'tick', timestamp: startTime + 1 } )
+      ] ] )
+      .mockResolvedValueOnce( [ , , [
+        JSON.stringify( { name: 'A', phase: 'start', timestamp: startTime } ),
+        JSON.stringify( { name: 'A', phase: 'tick', timestamp: startTime + 1 } ),
+        JSON.stringify( { name: 'A', phase: 'end', timestamp: startTime + 2 } )
+      ] ] );
+
+    await exec( { ...ctx, entry: { name: 'A', phase: 'start', timestamp: startTime, parentId: 'root' } } );
+    await exec( { ...ctx, entry: { name: 'A', phase: 'tick', timestamp: startTime + 1, parentId: 'root' } } );
+    // Root end: no parentId and not start
+    await exec( { ...ctx, entry: { name: 'A', phase: 'end', timestamp: startTime + 2 } } );
+
+    // Accumulation happened 3 times
+    expect( redisMulti.zAdd ).toHaveBeenCalledTimes( 3 );
+    expect( buildTraceTreeMock ).toHaveBeenCalledTimes( 3 );
+
+    // Only last call triggers upload
+    expect( uploadMock ).toHaveBeenCalledTimes( 1 );
+    const { key, content } = uploadMock.mock.calls[0][0];
+    expect( key ).toMatch( /^WF\/2020\/01\/02\// );
+    expect( JSON.parse( content.trim() ).count ).toBe( 3 );
+  } );
+} );
+

package/src/tracing/processors/s3/redis_client.js

@@ -0,0 +1,19 @@
+import { createClient } from 'redis';
+import { throws } from '#utils';
+
+const state = { client: null };
+
+/**
+ * Return a connected Redis instance
+ * @returns {redis.RedisClientType}
+ */
+export async function getRedisClient() {
+  const url = process.env.REDIS_URL ?? throws( new Error( 'Missing REDIS_URL environment variable' ) );
+  if ( await state.client?.ping().catch( _ => 0 ) === 'PONG' ) {
+    return state.client;
+  };
+
+  const client = createClient( { url, socket: { keepAlive: 15000 } } );
+  await client.connect();
+  return state.client = client;
+};

package/src/tracing/processors/s3/redis_client.spec.js

@@ -0,0 +1,50 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+vi.mock( '#utils', () => ( {
+  throws: e => {
+    throw e;
+  }
+} ) );
+
+const createClientImpl = vi.fn();
+vi.mock( 'redis', () => ( { createClient: opts => createClientImpl( opts ) } ) );
+
+async function loadModule() {
+  vi.resetModules();
+  return import( './redis_client.js' );
+}
+
+describe( 'tracing/processors/s3/redis_client', () => {
+  beforeEach( () => {
+    vi.clearAllMocks();
+    delete process.env.REDIS_URL;
+  } );
+
+  it( 'throws if REDIS_URL is missing', async () => {
+    const { getRedisClient } = await loadModule();
+    await expect( getRedisClient() ).rejects.toThrow( 'Missing REDIS_URL' );
+  } );
+
+  it( 'creates client with url, connects once, then reuses cached when ping is PONG', async () => {
+    process.env.REDIS_URL = 'redis://localhost:6379';
+
+    const pingMock = vi.fn().mockResolvedValue( 'PONG' );
+    const connectMock = vi.fn().mockResolvedValue();
+    const created = [];
+    createClientImpl.mockImplementation( opts => {
+      created.push( opts );
+      return { connect: connectMock, ping: pingMock };
+    } );
+
+    const { getRedisClient } = await loadModule();
+
+    const c1 = await getRedisClient();
+    const c2 = await getRedisClient();
+
+    expect( created ).toHaveLength( 1 );
+    expect( connectMock ).toHaveBeenCalledTimes( 1 );
+    expect( pingMock ).toHaveBeenCalledTimes( 1 );
+    expect( c1 ).toBe( c2 );
+    expect( created[0] ).toMatchObject( { url: 'redis://localhost:6379', socket: { keepAlive: 15000 } } );
+  } );
+} );

package/src/tracing/processors/s3/s3_client.js

@@ -0,0 +1,33 @@
+import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
+import { throws } from '#utils';
+
+const state = { s3Client: null };
+
+/**
+ * Return a S3 Client instance
+ * @returns {S3Client}
+ */
+const getS3Client = () => {
+  if ( state.s3Client ) {
+    return state.s3Client;
+  }
+
+  const region = process.env.AWS_REGION ?? throws( new Error( 'Missing AWS_REGION env var' ) );
+  const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY ?? throws( new Error( 'Missing AWS_SECRET_ACCESS_KEY env var' ) );
+  const accessKeyId = process.env.AWS_ACCESS_KEY_ID ?? throws( new Error( 'Missing AWS_ACCESS_KEY_ID env var' ) );
+
+  return state.s3Client = new S3Client( { region, secretAccessKey, accessKeyId } );
+};
+
+/**
+ * Upload given file to S3
+ * @param {object} args
+ * @param {string} key - S3 file key
+ * @param {string} content - File content
+ */
+export const upload = ( { key, content } ) =>
+  getS3Client().send( new PutObjectCommand( {
+    Bucket: process.env.TRACE_REMOTE_S3_BUCKET ?? throws( new Error( 'Missing TRACE_REMOTE_S3_BUCKET env var' ) ),
+    Key: key,
+    Body: content
+  } ) );