@outputai/core 0.4.1-dev.622e67b.0 → 0.4.1-dev.6555a2c.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/package.json +5 -1
  2. package/src/activity_integration/events.d.ts +6 -1
  3. package/src/activity_integration/events.js +2 -2
  4. package/src/activity_integration/events.spec.js +87 -0
  5. package/src/activity_integration/tracing.d.ts +5 -10
  6. package/src/activity_integration/tracing.js +4 -9
  7. package/src/consts.js +4 -0
  8. package/src/hooks/index.d.ts +40 -3
  9. package/src/hooks/index.js +6 -6
  10. package/src/interface/aggregations.js +24 -0
  11. package/src/interface/aggregations.spec.js +91 -0
  12. package/src/interface/workflow.d.ts +12 -1
  13. package/src/interface/workflow.js +44 -20
  14. package/src/interface/workflow.spec.js +183 -7
  15. package/src/interface/workflow_context.js +4 -2
  16. package/src/tracing/processors/local/index.js +10 -4
  17. package/src/tracing/processors/local/index.spec.js +52 -21
  18. package/src/tracing/processors/s3/index.js +3 -3
  19. package/src/tracing/processors/s3/index.spec.js +26 -1
  20. package/src/tracing/processors/s3/s3_client.js +11 -3
  21. package/src/tracing/processors/s3/s3_client.spec.js +27 -15
  22. package/src/tracing/tools/build_trace_tree.js +1 -1
  23. package/src/tracing/tools/build_trace_tree.spec.js +49 -11
  24. package/src/tracing/tools/utils.js +0 -28
  25. package/src/tracing/tools/utils.spec.js +2 -134
  26. package/src/tracing/trace_attribute.d.ts +38 -0
  27. package/src/tracing/trace_attribute.js +80 -0
  28. package/src/tracing/trace_engine.js +12 -2
  29. package/src/worker/index.js +1 -1
  30. package/src/worker/index.spec.js +1 -1
  31. package/src/worker/interceptors/activity.js +9 -2
  32. package/src/worker/interceptors/activity.spec.js +16 -3
  33. package/src/worker/interceptors.js +2 -2
  34. package/src/worker/sinks.js +6 -6
@@ -1,7 +1,10 @@
1
+ import { Signal } from '#consts';
1
2
  import { describe, it, expect, vi, beforeEach } from 'vitest';
2
3
  import { z } from 'zod';
3
4
 
4
5
  const inWorkflowContextMock = vi.hoisted( () => vi.fn( () => true ) );
6
+ const defineSignalMock = vi.hoisted( () => vi.fn( name => name ) );
7
+ const setHandlerMock = vi.hoisted( () => vi.fn() );
5
8
  const traceDestinationsStepMock = vi.fn().mockResolvedValue( { local: '/tmp/trace' } );
6
9
  const executeChildMock = vi.fn().mockResolvedValue( undefined );
7
10
  const continueAsNewMock = vi.fn().mockResolvedValue( undefined );
@@ -41,7 +44,16 @@ vi.mock( '@temporalio/workflow', () => ( {
41
44
  workflowInfo: workflowInfoMock,
42
45
  uuid4: () => '550e8400e29b41d4a716446655440000',
43
46
  ParentClosePolicy: { TERMINATE: 'TERMINATE', ABANDON: 'ABANDON' },
44
- continueAsNew: continueAsNewMock
47
+ ChildWorkflowFailure: class ChildWorkflowFailure extends Error {
48
+ constructor( message, cause ) {
49
+ super( message );
50
+ this.name = 'ChildWorkflowFailure';
51
+ this.cause = cause;
52
+ }
53
+ },
54
+ continueAsNew: continueAsNewMock,
55
+ defineSignal: ( ...args ) => defineSignalMock( ...args ),
56
+ setHandler: ( ...args ) => setHandlerMock( ...args )
45
57
  } ) );
46
58
 
47
59
  vi.mock( '#consts', async importOriginal => {
@@ -53,10 +65,17 @@ vi.mock( '#consts', async importOriginal => {
53
65
  };
54
66
  } );
55
67
 
68
+ const emptyAggregations = {
69
+ cost: { total: 0 },
70
+ tokens: { total: 0 },
71
+ httpRequests: { total: 0 }
72
+ };
73
+
56
74
  describe( 'workflow()', () => {
57
75
  beforeEach( () => {
58
76
  vi.clearAllMocks();
59
77
  inWorkflowContextMock.mockReturnValue( true );
78
+ defineSignalMock.mockImplementation( name => name );
60
79
  workflowInfoMock.mockReturnValue( { ...workflowInfoReturn } );
61
80
  workflowInfoReturn.memo = {};
62
81
  proxyActivitiesMock.mockImplementation( () => {
@@ -217,7 +236,7 @@ describe( 'workflow()', () => {
217
236
  } );
218
237
 
219
238
  describe( 'root workflow (in workflow context)', () => {
220
- it( 'calls getTraceDestinations, returns { output, trace } and assigns executionContext to memo', async () => {
239
+ it( 'calls getTraceDestinations, returns root trace data and assigns executionContext to memo', async () => {
221
240
  const { workflow } = await import( './workflow.js' );
222
241
 
223
242
  const wf = workflow( {
@@ -232,7 +251,9 @@ describe( 'workflow()', () => {
232
251
  expect( traceDestinationsStepMock ).toHaveBeenCalledTimes( 1 );
233
252
  expect( result ).toEqual( {
234
253
  output: { v: 42 },
235
- trace: { destinations: { local: '/tmp/trace' } }
254
+ trace: { destinations: { local: '/tmp/trace' } },
255
+ attributes: [],
256
+ aggregations: emptyAggregations
236
257
  } );
237
258
  const memo = workflowInfoMock().memo;
238
259
  expect( memo.executionContext ).toEqual( {
@@ -243,6 +264,68 @@ describe( 'workflow()', () => {
243
264
  } );
244
265
  } );
245
266
 
267
+ it( 'collects attribute signals and returns aggregated attributes', async () => {
268
+ const { workflow } = await import( './workflow.js' );
269
+ const { Attribute } = await import( '#trace_attribute' );
270
+ const handlers = { addAttribute: () => {} };
271
+ setHandlerMock.mockImplementation( ( signalName, handler ) => {
272
+ if ( signalName === Signal.ADD_ATTRIBUTE ) {
273
+ handlers.addAttribute = handler;
274
+ }
275
+ } );
276
+
277
+ const httpRequest = {
278
+ type: Attribute.HTTPRequestCount.TYPE,
279
+ url: 'https://api.example.test/items',
280
+ requestId: 'req-1'
281
+ };
282
+ const httpCost = {
283
+ type: Attribute.HTTPRequestCost.TYPE,
284
+ url: 'https://api.example.test/items',
285
+ requestId: 'req-1',
286
+ total: 2.5
287
+ };
288
+ const llmUsage = {
289
+ type: Attribute.LLMUsage.TYPE,
290
+ modelId: 'gpt-4o',
291
+ total: 0.25,
292
+ usage: [
293
+ { type: 'input', ppm: 5, amount: 20_000, total: 0.1 },
294
+ { type: 'output', ppm: 30, amount: 5_000, total: 0.15 }
295
+ ],
296
+ tokensUsed: 25_000
297
+ };
298
+
299
+ const wf = workflow( {
300
+ name: 'attr_wf',
301
+ description: 'Attributes',
302
+ inputSchema: z.object( {} ),
303
+ outputSchema: z.object( { ok: z.boolean() } ),
304
+ fn: async () => {
305
+ handlers.addAttribute( httpRequest );
306
+ handlers.addAttribute( httpCost );
307
+ handlers.addAttribute( llmUsage );
308
+ return { ok: true };
309
+ }
310
+ } );
311
+
312
+ const result = await wf( {} );
313
+ expect( result ).toEqual( {
314
+ output: { ok: true },
315
+ trace: { destinations: { local: '/tmp/trace' } },
316
+ attributes: [ httpRequest, httpCost, llmUsage ],
317
+ aggregations: {
318
+ cost: { total: 2.75 },
319
+ tokens: {
320
+ total: 25_000,
321
+ input: 20_000,
322
+ output: 5_000
323
+ },
324
+ httpRequests: { total: 1 }
325
+ }
326
+ } );
327
+ } );
328
+
246
329
  it( 'sets executionContext.disableTrace when options.disableTrace is true', async () => {
247
330
  const { workflow } = await import( './workflow.js' );
248
331
 
@@ -261,7 +344,7 @@ describe( 'workflow()', () => {
261
344
  } );
262
345
 
263
346
  describe( 'child workflow (memo.executionContext already set)', () => {
264
- it( 'does not call getTraceDestinations and returns plain output', async () => {
347
+ it( 'does not call getTraceDestinations and returns an internal output envelope', async () => {
265
348
  workflowInfoMock.mockReturnValue( {
266
349
  ...workflowInfoReturn,
267
350
  memo: { executionContext: { workflowId: 'parent-1', workflowName: 'parent_wf' } }
@@ -278,7 +361,7 @@ describe( 'workflow()', () => {
278
361
 
279
362
  const result = await wf( {} );
280
363
  expect( traceDestinationsStepMock ).not.toHaveBeenCalled();
281
- expect( result ).toEqual( { x: 'child' } );
364
+ expect( result ).toEqual( { output: { x: 'child' }, attributes: [] } );
282
365
  } );
283
366
  } );
284
367
 
@@ -381,6 +464,7 @@ describe( 'workflow()', () => {
381
464
  it( 'calls executeChild with correct args and TERMINATE when not detached', async () => {
382
465
  const { workflow } = await import( './workflow.js' );
383
466
  const { ParentClosePolicy } = await import( '@temporalio/workflow' );
467
+ executeChildMock.mockResolvedValueOnce( { output: {}, attributes: [] } );
384
468
 
385
469
  const wf = workflow( {
386
470
  name: 'parent_wf',
@@ -408,6 +492,7 @@ describe( 'workflow()', () => {
408
492
  it( 'uses ABANDON when extra.detached is true', async () => {
409
493
  const { workflow } = await import( './workflow.js' );
410
494
  const { ParentClosePolicy } = await import( '@temporalio/workflow' );
495
+ executeChildMock.mockResolvedValueOnce( { output: {}, attributes: [] } );
411
496
 
412
497
  const wf = workflow( {
413
498
  name: 'detach_wf',
@@ -428,6 +513,7 @@ describe( 'workflow()', () => {
428
513
 
429
514
  it( 'passes empty args when input is null/omitted', async () => {
430
515
  const { workflow } = await import( './workflow.js' );
516
+ executeChildMock.mockResolvedValueOnce( { output: {}, attributes: [] } );
431
517
 
432
518
  const wf = workflow( {
433
519
  name: 'no_input_wf',
@@ -445,11 +531,96 @@ describe( 'workflow()', () => {
445
531
  args: []
446
532
  } ) );
447
533
  } );
534
+
535
+ it( 'returns child output and merges child attributes into the root result', async () => {
536
+ const { workflow } = await import( './workflow.js' );
537
+ const { Attribute } = await import( '#trace_attribute' );
538
+ const childAttribute = {
539
+ type: Attribute.LLMUsage.TYPE,
540
+ modelId: 'gpt-4o',
541
+ total: 0.4,
542
+ tokensUsed: 20,
543
+ usage: [
544
+ { type: 'input', ppm: 10, amount: 20, total: 0.4 }
545
+ ]
546
+ };
547
+ executeChildMock.mockResolvedValueOnce( {
548
+ output: { child: 'ok' },
549
+ attributes: [ childAttribute ]
550
+ } );
551
+
552
+ const wf = workflow( {
553
+ name: 'merge_child_wf',
554
+ description: 'Merge child attributes',
555
+ inputSchema: z.object( {} ),
556
+ outputSchema: z.object( { child: z.string() } ),
557
+ async fn() {
558
+ return this.startWorkflow( 'child_wf', { id: 1 } );
559
+ }
560
+ } );
561
+
562
+ const result = await wf( {} );
563
+ expect( result ).toEqual( {
564
+ output: { child: 'ok' },
565
+ trace: { destinations: { local: '/tmp/trace' } },
566
+ attributes: [ childAttribute ],
567
+ aggregations: {
568
+ cost: { total: 0.4 },
569
+ tokens: {
570
+ total: 20,
571
+ input: 20
572
+ },
573
+ httpRequests: { total: 0 }
574
+ }
575
+ } );
576
+ } );
577
+
578
+ it( 'merges child error attributes before rethrowing to root metadata', async () => {
579
+ const { workflow } = await import( './workflow.js' );
580
+ const { ChildWorkflowFailure } = await import( '@temporalio/workflow' );
581
+ const { METADATA_ACCESS_SYMBOL } = await import( '#consts' );
582
+ const { Attribute } = await import( '#trace_attribute' );
583
+ const childAttribute = {
584
+ type: Attribute.HTTPRequestCost.TYPE,
585
+ url: 'https://api.example.test',
586
+ requestId: 'req-child',
587
+ total: 2
588
+ };
589
+ const childError = new ChildWorkflowFailure( 'child failed', {
590
+ message: 'Child workflow execution failed',
591
+ details: [ { attributes: [ childAttribute ] } ]
592
+ } );
593
+ executeChildMock.mockRejectedValueOnce( childError );
594
+
595
+ const wf = workflow( {
596
+ name: 'child_error_wf',
597
+ description: 'Child error attributes',
598
+ inputSchema: z.object( {} ),
599
+ outputSchema: z.object( {} ),
600
+ async fn() {
601
+ await this.startWorkflow( 'child_wf', { id: 1 } );
602
+ return {};
603
+ }
604
+ } );
605
+
606
+ await expect( wf( {} ) ).rejects.toThrow( 'child failed' );
607
+ expect( childError[METADATA_ACCESS_SYMBOL] ).toEqual( {
608
+ attributes: [ childAttribute ],
609
+ trace: { destinations: { local: '/tmp/trace' } },
610
+ aggregations: {
611
+ cost: { total: 2 },
612
+ tokens: { total: 0 },
613
+ httpRequests: { total: 0 }
614
+ }
615
+ } );
616
+ } );
448
617
  } );
449
618
 
450
619
  describe( 'error handling (root workflow)', () => {
451
- it( 'rethrows error from fn and rejects with same message', async () => {
620
+ it( 'rethrows error from fn with trace attributes and aggregation metadata', async () => {
452
621
  const { workflow } = await import( './workflow.js' );
622
+ const { METADATA_ACCESS_SYMBOL } = await import( '#consts' );
623
+ const error = new Error( 'workflow failed' );
453
624
 
454
625
  const wf = workflow( {
455
626
  name: 'err_wf',
@@ -457,11 +628,16 @@ describe( 'workflow()', () => {
457
628
  inputSchema: z.object( {} ),
458
629
  outputSchema: z.object( {} ),
459
630
  fn: async () => {
460
- throw new Error( 'workflow failed' );
631
+ throw error;
461
632
  }
462
633
  } );
463
634
 
464
635
  await expect( wf( {} ) ).rejects.toThrow( 'workflow failed' );
636
+ expect( error[METADATA_ACCESS_SYMBOL] ).toEqual( {
637
+ trace: { destinations: { local: '/tmp/trace' } },
638
+ attributes: [],
639
+ aggregations: emptyAggregations
640
+ } );
465
641
  } );
466
642
  } );
467
643
  } );
@@ -7,11 +7,12 @@ export class Context {
7
7
  * Builds a new context instance
8
8
  * @param {object} options - Arguments to build a new context instance
9
9
  * @param {string} workflowId
10
+ * @param {string} runId
10
11
  * @param {function} continueAsNew
11
12
  * @param {function} isContinueAsNewSuggested
12
13
  * @returns {object} context
13
14
  */
14
- static build( { workflowId, continueAsNew, isContinueAsNewSuggested } ) {
15
+ static build( { workflowId, runId, continueAsNew, isContinueAsNewSuggested } ) {
15
16
  return {
16
17
  /**
17
18
  * Control namespace: This object adds functions to interact with Temporal flow mechanisms
@@ -24,7 +25,8 @@ export class Context {
24
25
  * Info namespace: abstracts workflowInfo()
25
26
  */
26
27
  info: {
27
- workflowId
28
+ workflowId,
29
+ runId
28
30
  }
29
31
  };
30
32
  }
@@ -1,9 +1,11 @@
1
- import { appendFileSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
1
+ import { appendFileSync, mkdirSync, readdirSync, readFileSync, rmSync, createWriteStream } from 'node:fs';
2
2
  import { dirname, join } from 'node:path';
3
3
  import { fileURLToPath } from 'url';
4
4
  import buildTraceTree from '../../tools/build_trace_tree.js';
5
- import { safeFormatJSON } from '../../tools/utils.js';
6
5
  import { EOL } from 'node:os';
6
+ import { JsonStreamStringify } from 'json-stream-stringify';
7
+
8
+ import { pipeline } from 'stream/promises';
7
9
 
8
10
  const __dirname = dirname( fileURLToPath( import.meta.url ) );
9
11
 
@@ -109,7 +111,7 @@ export const init = () => {
109
111
  * @param {object} args.executionContext - Execution info: workflowId, workflowName, startTime
110
112
  * @returns {void}
111
113
  */
112
- export const exec = ( { entry, executionContext } ) => {
114
+ export const exec = async ( { entry, executionContext } ) => {
113
115
  const { workflowId, workflowName, startTime } = executionContext;
114
116
  const tempFilePath = createTempFilePath( executionContext );
115
117
  addEntry( entry, tempFilePath );
@@ -126,7 +128,11 @@ export const exec = ( { entry, executionContext } ) => {
126
128
  const path = join( dir, buildTraceFilename( { startTime, workflowId } ) );
127
129
 
128
130
  mkdirSync( dir, { recursive: true } );
129
- writeFileSync( path, safeFormatJSON( content ) + EOL, 'utf-8' );
131
+
132
+ await pipeline(
133
+ new JsonStreamStringify( content ),
134
+ createWriteStream( path )
135
+ );
130
136
  };
131
137
 
132
138
  /**
@@ -1,5 +1,4 @@
1
1
  import { describe, it, expect, vi, beforeEach } from 'vitest';
2
- import { EOL } from 'node:os';
3
2
 
4
3
  // In-memory fs mock store
5
4
  const store = { files: new Map() };
@@ -12,6 +11,7 @@ const appendFileSyncMock = vi.fn( ( path, data ) => {
12
11
  const readFileSyncMock = vi.fn( path => store.files.get( path ) ?? '' );
13
12
  const readdirSyncMock = vi.fn( () => [] );
14
13
  const rmSyncMock = vi.fn();
14
+ const createWriteStreamMock = vi.fn( path => ( { path } ) );
15
15
 
16
16
  vi.mock( 'node:fs', () => ( {
17
17
  mkdirSync: mkdirSyncMock,
@@ -19,9 +19,36 @@ vi.mock( 'node:fs', () => ( {
19
19
  appendFileSync: appendFileSyncMock,
20
20
  readFileSync: readFileSyncMock,
21
21
  readdirSync: readdirSyncMock,
22
- rmSync: rmSyncMock
22
+ rmSync: rmSyncMock,
23
+ createWriteStream: createWriteStreamMock
23
24
  } ) );
24
25
 
26
+ const pipelineMock = vi.fn( async ( source, destination ) => {
27
+ const chunks = [];
28
+ for await ( const chunk of source ) {
29
+ chunks.push( Buffer.isBuffer( chunk ) ? chunk : Buffer.from( chunk ) );
30
+ }
31
+ store.files.set( destination.path, Buffer.concat( chunks ).toString( 'utf8' ) );
32
+ } );
33
+ vi.mock( 'node:stream/promises', () => ( { pipeline: pipelineMock } ) );
34
+
35
+ vi.mock( 'json-stream-stringify', async () => {
36
+ const { Readable } = await import( 'node:stream' );
37
+ return {
38
+ JsonStreamStringify: class extends Readable {
39
+ constructor( body ) {
40
+ super();
41
+ this.body = body;
42
+ }
43
+
44
+ _read() {
45
+ this.push( JSON.stringify( this.body ) );
46
+ this.push( null );
47
+ }
48
+ }
49
+ };
50
+ } );
51
+
25
52
  const buildTraceTreeMock = vi.fn( entries => ( { count: entries.length } ) );
26
53
  vi.mock( '../../tools/build_trace_tree.js', () => ( { default: buildTraceTreeMock } ) );
27
54
 
@@ -58,17 +85,18 @@ describe( 'tracing/processors/local', () => {
58
85
  const workflowId = 'id1';
59
86
  const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
60
87
 
61
- exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
62
- exec( { ...ctx, entry: childTick( 'child-1', startTime + 1 ) } );
63
- exec( { ...ctx, entry: rootEnd( workflowId, startTime + 2 ) } );
88
+ await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
89
+ await exec( { ...ctx, entry: childTick( 'child-1', startTime + 1 ) } );
90
+ await exec( { ...ctx, entry: rootEnd( workflowId, startTime + 2 ) } );
64
91
 
65
92
  expect( buildTraceTreeMock ).toHaveBeenCalledTimes( 1 );
66
93
  expect( buildTraceTreeMock.mock.calls[0][0] ).toHaveLength( 3 );
67
94
 
68
- expect( writeFileSyncMock ).toHaveBeenCalledTimes( 1 );
69
- const [ writtenPath, content ] = writeFileSyncMock.mock.calls[0];
95
+ expect( createWriteStreamMock ).toHaveBeenCalledTimes( 1 );
96
+ expect( pipelineMock ).toHaveBeenCalledTimes( 1 );
97
+ const [ writtenPath ] = createWriteStreamMock.mock.calls[0];
70
98
  expect( writtenPath ).toMatch( /\/tmp\/project\/logs\/runs\/WF\// );
71
- expect( JSON.parse( content.trim() ).count ).toBe( 3 );
99
+ expect( JSON.parse( store.files.get( writtenPath ) ).count ).toBe( 3 );
72
100
  } );
73
101
 
74
102
  it( 'exec(): does not build or write on non-flush entries', async () => {
@@ -79,11 +107,13 @@ describe( 'tracing/processors/local', () => {
79
107
  const workflowId = 'id1';
80
108
  const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
81
109
 
82
- exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
83
- exec( { ...ctx, entry: childTick( 'child-1', startTime + 1 ) } );
110
+ await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
111
+ await exec( { ...ctx, entry: childTick( 'child-1', startTime + 1 ) } );
84
112
 
85
113
  expect( buildTraceTreeMock ).not.toHaveBeenCalled();
86
114
  expect( writeFileSyncMock ).not.toHaveBeenCalled();
115
+ expect( createWriteStreamMock ).not.toHaveBeenCalled();
116
+ expect( pipelineMock ).not.toHaveBeenCalled();
87
117
  } );
88
118
 
89
119
  it( 'exec(): flushes on error action before root end', async () => {
@@ -94,12 +124,13 @@ describe( 'tracing/processors/local', () => {
94
124
  const workflowId = 'id1';
95
125
  const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
96
126
 
97
- exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
98
- exec( { ...ctx, entry: { id: 'step-1', action: 'error', timestamp: startTime + 1 } } );
127
+ await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
128
+ await exec( { ...ctx, entry: { id: 'step-1', action: 'error', timestamp: startTime + 1 } } );
99
129
 
100
130
  expect( buildTraceTreeMock ).toHaveBeenCalledTimes( 1 );
101
131
  expect( buildTraceTreeMock.mock.calls[0][0] ).toHaveLength( 2 );
102
- expect( writeFileSyncMock ).toHaveBeenCalledTimes( 1 );
132
+ expect( createWriteStreamMock ).toHaveBeenCalledTimes( 1 );
133
+ expect( pipelineMock ).toHaveBeenCalledTimes( 1 );
103
134
  } );
104
135
 
105
136
  it( 'getDestination(): returns absolute path under callerDir logs', async () => {
@@ -128,11 +159,11 @@ describe( 'tracing/processors/local', () => {
128
159
  const workflowId = 'id1';
129
160
  const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
130
161
 
131
- exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
132
- exec( { ...ctx, entry: rootEnd( workflowId, startTime + 1 ) } );
162
+ await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
163
+ await exec( { ...ctx, entry: rootEnd( workflowId, startTime + 1 ) } );
133
164
 
134
- expect( writeFileSyncMock ).toHaveBeenCalledTimes( 1 );
135
- const [ writtenPath ] = writeFileSyncMock.mock.calls[0];
165
+ expect( createWriteStreamMock ).toHaveBeenCalledTimes( 1 );
166
+ const [ writtenPath ] = createWriteStreamMock.mock.calls[0];
136
167
 
137
168
  expect( writtenPath ).not.toContain( '/host/path/logs' );
138
169
  expect( writtenPath ).toMatch( /\/tmp\/project\/logs\/runs\/WF\// );
@@ -164,15 +195,15 @@ describe( 'tracing/processors/local', () => {
164
195
  const workflowName = 'test-workflow';
165
196
  const ctx = { executionContext: { workflowId, workflowName, startTime } };
166
197
 
167
- exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
168
- exec( { ...ctx, entry: rootEnd( workflowId, startTime + 1 ) } );
198
+ await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
199
+ await exec( { ...ctx, entry: rootEnd( workflowId, startTime + 1 ) } );
169
200
 
170
201
  const destination = getDestination( { startTime, workflowId, workflowName } );
171
202
 
172
- const [ writtenPath, payload ] = writeFileSyncMock.mock.calls[0];
203
+ const [ writtenPath ] = createWriteStreamMock.mock.calls[0];
173
204
  expect( writtenPath ).not.toContain( '/Users/ben/project' );
174
205
  expect( writtenPath ).toMatch( /\/tmp\/project\/logs\/runs\/test-workflow\// );
175
- expect( payload.endsWith( EOL ) ).toBe( true );
206
+ expect( JSON.parse( store.files.get( writtenPath ) ).count ).toBe( 2 );
176
207
 
177
208
  expect( destination ).toBe( '/Users/ben/project/logs/runs/test-workflow/2020-01-02-03-04-05-678Z_workflow-id-123.json' );
178
209
  } );
@@ -1,10 +1,9 @@
1
1
  import { upload } from './s3_client.js';
2
2
  import { getRedisClient } from './redis_client.js';
3
3
  import buildTraceTree from '../../tools/build_trace_tree.js';
4
- import { EOL } from 'node:os';
5
4
  import { loadEnv, getVars } from './configs.js';
6
5
  import { createChildLogger } from '#logger';
7
- import { safeFormatJSON } from '../../tools/utils.js';
6
+ import { JsonStreamStringify } from 'json-stream-stringify';
8
7
 
9
8
  const log = createChildLogger( 'S3 Processor' );
10
9
 
@@ -100,9 +99,10 @@ export const exec = async ( { entry, executionContext } ) => {
100
99
  log.warn( 'Incomplete trace file discarded', { workflowId, error: 'incomplete_trace_file' } );
101
100
  return;
102
101
  }
102
+
103
103
  await upload( {
104
104
  key: getS3Key( { workflowId, workflowName, startTime } ),
105
- content: safeFormatJSON( content ) + EOL
105
+ content: new JsonStreamStringify( content )
106
106
  } );
107
107
  await bustEntries( cacheKey );
108
108
  };
@@ -27,6 +27,31 @@ vi.mock( './s3_client.js', () => ( { upload: uploadMock } ) );
27
27
  const buildTraceTreeMock = vi.fn( entries => ( { count: entries.length } ) );
28
28
  vi.mock( '../../tools/build_trace_tree.js', () => ( { default: buildTraceTreeMock } ) );
29
29
 
30
+ vi.mock( 'json-stream-stringify', async () => {
31
+ const { Readable } = await import( 'node:stream' );
32
+ return {
33
+ JsonStreamStringify: class extends Readable {
34
+ constructor( body ) {
35
+ super();
36
+ this.body = body;
37
+ }
38
+
39
+ _read() {
40
+ this.push( JSON.stringify( this.body ) );
41
+ this.push( null );
42
+ }
43
+ }
44
+ };
45
+ } );
46
+
47
+ const streamToString = async stream => {
48
+ const chunks = [];
49
+ for await ( const chunk of stream ) {
50
+ chunks.push( Buffer.isBuffer( chunk ) ? chunk : Buffer.from( chunk ) );
51
+ }
52
+ return Buffer.concat( chunks ).toString( 'utf8' );
53
+ };
54
+
30
55
  describe( 'tracing/processors/s3', () => {
31
56
  beforeEach( () => {
32
57
  vi.useFakeTimers();
@@ -74,7 +99,7 @@ describe( 'tracing/processors/s3', () => {
74
99
  expect( uploadMock ).toHaveBeenCalledTimes( 1 );
75
100
  const { key, content } = uploadMock.mock.calls[0][0];
76
101
  expect( key ).toMatch( /^WF\/2020\/01\/02\// );
77
- expect( JSON.parse( content.trim() ).count ).toBe( 3 );
102
+ expect( JSON.parse( await streamToString( content ) ).count ).toBe( 3 );
78
103
  expect( delMock ).toHaveBeenCalledTimes( 1 );
79
104
  expect( delMock ).toHaveBeenCalledWith( 'traces/WF/id1' );
80
105
  } );
@@ -1,4 +1,5 @@
1
- import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
1
+ import { S3Client } from '@aws-sdk/client-s3';
2
+ import { Upload } from '@aws-sdk/lib-storage';
2
3
  import { getVars } from './configs.js';
3
4
 
4
5
  const state = { s3Client: null };
@@ -21,7 +22,14 @@ const getS3Client = () => {
21
22
  * Upload given file to S3
22
23
  * @param {object} args
23
24
  * @param {string} key - S3 file key
24
- * @param {string} content - File content
25
+ * @param {string|import('node:stream').Readable} content - File content
25
26
  */
26
27
  export const upload = ( { key, content } ) =>
27
- getS3Client().send( new PutObjectCommand( { Bucket: getVars().remoteS3Bucket, Key: key, Body: content } ) );
28
+ new Upload( {
29
+ client: getS3Client(),
30
+ params: {
31
+ Bucket: getVars().remoteS3Bucket,
32
+ Key: key,
33
+ Body: content
34
+ }
35
+ } ).done();
@@ -9,24 +9,27 @@ vi.mock( '#utils', () => ( {
9
9
  const getVarsMock = vi.fn();
10
10
  vi.mock( './configs', () => ( { getVars: () => getVarsMock() } ) );
11
11
 
12
- const sendMock = vi.fn();
13
12
  const ctorState = { args: null };
14
13
  class S3ClientMock {
15
14
  constructor( args ) {
16
15
  ctorState.args = args;
17
- } send = sendMock;
18
- }
19
- class PutObjectCommandMock {
20
- constructor( input ) {
21
- this.input = input;
22
16
  }
23
17
  }
24
-
25
18
  vi.mock( '@aws-sdk/client-s3', () => ( {
26
- S3Client: S3ClientMock,
27
- PutObjectCommand: PutObjectCommandMock
19
+ S3Client: S3ClientMock
28
20
  } ) );
29
21
 
22
+ const uploadDoneMock = vi.fn();
23
+ const uploadCtorState = { args: [] };
24
+ class UploadMock {
25
+ constructor( args ) {
26
+ uploadCtorState.args.push( args );
27
+ }
28
+
29
+ done = uploadDoneMock;
30
+ }
31
+ vi.mock( '@aws-sdk/lib-storage', () => ( { Upload: UploadMock } ) );
32
+
30
33
  async function loadModule() {
31
34
  vi.resetModules();
32
35
  return import( './s3_client.js' );
@@ -35,6 +38,9 @@ async function loadModule() {
35
38
  describe( 'tracing/processors/s3/s3_client', () => {
36
39
  beforeEach( () => {
37
40
  vi.clearAllMocks();
41
+ ctorState.args = null;
42
+ uploadCtorState.args = [];
43
+ uploadDoneMock.mockResolvedValue( undefined );
38
44
  getVarsMock.mockReturnValue( {
39
45
  awsRegion: 'us-east-1',
40
46
  awsAccessKeyId: 'id',
@@ -48,15 +54,21 @@ describe( 'tracing/processors/s3/s3_client', () => {
48
54
 
49
55
  await upload( { key: 'wf/key.json', content: '{"a":1}' } );
50
56
 
51
- expect( ctorState.args ).toEqual( { region: 'us-east-1', credentials: { secretAccessKey: 'sek', accessKeyId: 'id' } } );
52
- expect( sendMock ).toHaveBeenCalledTimes( 1 );
53
- const cmd = sendMock.mock.calls[0][0];
54
- expect( cmd ).toBeInstanceOf( PutObjectCommandMock );
55
- expect( cmd.input ).toEqual( { Bucket: 'bucket', Key: 'wf/key.json', Body: '{"a":1}' } );
57
+ expect( ctorState.args ).toEqual( {
58
+ region: 'us-east-1',
59
+ credentials: { secretAccessKey: 'sek', accessKeyId: 'id' }
60
+ } );
61
+ expect( uploadCtorState.args ).toHaveLength( 1 );
62
+ expect( uploadCtorState.args[0] ).toEqual( {
63
+ client: expect.any( S3ClientMock ),
64
+ params: { Bucket: 'bucket', Key: 'wf/key.json', Body: '{"a":1}' }
65
+ } );
66
+ expect( uploadDoneMock ).toHaveBeenCalledTimes( 1 );
56
67
 
57
68
  // subsequent upload uses cached client
58
69
  await upload( { key: 'wf/key2.json', content: '{}' } );
59
- expect( sendMock ).toHaveBeenCalledTimes( 2 );
70
+ expect( uploadCtorState.args ).toHaveLength( 2 );
71
+ expect( uploadDoneMock ).toHaveBeenCalledTimes( 2 );
60
72
  } );
61
73
  } );
62
74
 
@@ -62,7 +62,7 @@ export default entries => {
62
62
  if ( action === EventAction.START ) {
63
63
  Object.assign( node, { input: details, startedAt: timestamp, kind, name } );
64
64
  } else if ( action === EventAction.ADD_ATTR ) {
65
- node.attributes[details.name] = details.value;
65
+ node.attributes[details.type] = details;
66
66
  } else if ( action === EventAction.END ) {
67
67
  Object.assign( node, { output: details, endedAt: timestamp } );
68
68
  } else if ( action === EventAction.ERROR ) {