@outputai/core 0.4.1-dev.56c13a8.0 → 0.4.1-dev.6555a2c.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -4
- package/src/activity_integration/events.d.ts +6 -1
- package/src/activity_integration/events.js +2 -2
- package/src/activity_integration/events.spec.js +87 -0
- package/src/activity_integration/tracing.d.ts +5 -11
- package/src/activity_integration/tracing.js +4 -10
- package/src/consts.js +4 -0
- package/src/hooks/index.d.ts +40 -3
- package/src/hooks/index.js +6 -6
- package/src/interface/aggregations.js +24 -0
- package/src/interface/aggregations.spec.js +91 -0
- package/src/interface/workflow.d.ts +12 -1
- package/src/interface/workflow.js +44 -20
- package/src/interface/workflow.spec.js +183 -7
- package/src/interface/workflow_context.js +4 -2
- package/src/tracing/processors/local/index.js +10 -4
- package/src/tracing/processors/local/index.spec.js +52 -21
- package/src/tracing/processors/s3/index.js +3 -3
- package/src/tracing/processors/s3/index.spec.js +26 -1
- package/src/tracing/processors/s3/s3_client.js +11 -3
- package/src/tracing/processors/s3/s3_client.spec.js +27 -15
- package/src/tracing/tools/build_trace_tree.js +1 -1
- package/src/tracing/tools/build_trace_tree.spec.js +49 -11
- package/src/tracing/tools/utils.js +0 -28
- package/src/tracing/tools/utils.spec.js +2 -134
- package/src/tracing/trace_attribute.d.ts +38 -0
- package/src/tracing/trace_attribute.js +80 -0
- package/src/tracing/trace_engine.js +12 -2
- package/src/worker/index.js +1 -1
- package/src/worker/index.spec.js +1 -1
- package/src/worker/interceptors/activity.js +9 -2
- package/src/worker/interceptors/activity.spec.js +16 -3
- package/src/worker/interceptors.js +2 -2
- package/src/worker/sinks.js +6 -6
- package/src/tracing/tools/aggregate_trace_attributes.js +0 -118
- package/src/tracing/tools/aggregate_trace_attributes.spec.js +0 -231
- package/src/tracing/tools/index.js +0 -7
|
@@ -1,7 +1,10 @@
|
|
|
1
|
+
import { Signal } from '#consts';
|
|
1
2
|
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
2
3
|
import { z } from 'zod';
|
|
3
4
|
|
|
4
5
|
const inWorkflowContextMock = vi.hoisted( () => vi.fn( () => true ) );
|
|
6
|
+
const defineSignalMock = vi.hoisted( () => vi.fn( name => name ) );
|
|
7
|
+
const setHandlerMock = vi.hoisted( () => vi.fn() );
|
|
5
8
|
const traceDestinationsStepMock = vi.fn().mockResolvedValue( { local: '/tmp/trace' } );
|
|
6
9
|
const executeChildMock = vi.fn().mockResolvedValue( undefined );
|
|
7
10
|
const continueAsNewMock = vi.fn().mockResolvedValue( undefined );
|
|
@@ -41,7 +44,16 @@ vi.mock( '@temporalio/workflow', () => ( {
|
|
|
41
44
|
workflowInfo: workflowInfoMock,
|
|
42
45
|
uuid4: () => '550e8400e29b41d4a716446655440000',
|
|
43
46
|
ParentClosePolicy: { TERMINATE: 'TERMINATE', ABANDON: 'ABANDON' },
|
|
44
|
-
|
|
47
|
+
ChildWorkflowFailure: class ChildWorkflowFailure extends Error {
|
|
48
|
+
constructor( message, cause ) {
|
|
49
|
+
super( message );
|
|
50
|
+
this.name = 'ChildWorkflowFailure';
|
|
51
|
+
this.cause = cause;
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
continueAsNew: continueAsNewMock,
|
|
55
|
+
defineSignal: ( ...args ) => defineSignalMock( ...args ),
|
|
56
|
+
setHandler: ( ...args ) => setHandlerMock( ...args )
|
|
45
57
|
} ) );
|
|
46
58
|
|
|
47
59
|
vi.mock( '#consts', async importOriginal => {
|
|
@@ -53,10 +65,17 @@ vi.mock( '#consts', async importOriginal => {
|
|
|
53
65
|
};
|
|
54
66
|
} );
|
|
55
67
|
|
|
68
|
+
const emptyAggregations = {
|
|
69
|
+
cost: { total: 0 },
|
|
70
|
+
tokens: { total: 0 },
|
|
71
|
+
httpRequests: { total: 0 }
|
|
72
|
+
};
|
|
73
|
+
|
|
56
74
|
describe( 'workflow()', () => {
|
|
57
75
|
beforeEach( () => {
|
|
58
76
|
vi.clearAllMocks();
|
|
59
77
|
inWorkflowContextMock.mockReturnValue( true );
|
|
78
|
+
defineSignalMock.mockImplementation( name => name );
|
|
60
79
|
workflowInfoMock.mockReturnValue( { ...workflowInfoReturn } );
|
|
61
80
|
workflowInfoReturn.memo = {};
|
|
62
81
|
proxyActivitiesMock.mockImplementation( () => {
|
|
@@ -217,7 +236,7 @@ describe( 'workflow()', () => {
|
|
|
217
236
|
} );
|
|
218
237
|
|
|
219
238
|
describe( 'root workflow (in workflow context)', () => {
|
|
220
|
-
it( 'calls getTraceDestinations, returns
|
|
239
|
+
it( 'calls getTraceDestinations, returns root trace data and assigns executionContext to memo', async () => {
|
|
221
240
|
const { workflow } = await import( './workflow.js' );
|
|
222
241
|
|
|
223
242
|
const wf = workflow( {
|
|
@@ -232,7 +251,9 @@ describe( 'workflow()', () => {
|
|
|
232
251
|
expect( traceDestinationsStepMock ).toHaveBeenCalledTimes( 1 );
|
|
233
252
|
expect( result ).toEqual( {
|
|
234
253
|
output: { v: 42 },
|
|
235
|
-
trace: { destinations: { local: '/tmp/trace' } }
|
|
254
|
+
trace: { destinations: { local: '/tmp/trace' } },
|
|
255
|
+
attributes: [],
|
|
256
|
+
aggregations: emptyAggregations
|
|
236
257
|
} );
|
|
237
258
|
const memo = workflowInfoMock().memo;
|
|
238
259
|
expect( memo.executionContext ).toEqual( {
|
|
@@ -243,6 +264,68 @@ describe( 'workflow()', () => {
|
|
|
243
264
|
} );
|
|
244
265
|
} );
|
|
245
266
|
|
|
267
|
+
it( 'collects attribute signals and returns aggregated attributes', async () => {
|
|
268
|
+
const { workflow } = await import( './workflow.js' );
|
|
269
|
+
const { Attribute } = await import( '#trace_attribute' );
|
|
270
|
+
const handlers = { addAttribute: () => {} };
|
|
271
|
+
setHandlerMock.mockImplementation( ( signalName, handler ) => {
|
|
272
|
+
if ( signalName === Signal.ADD_ATTRIBUTE ) {
|
|
273
|
+
handlers.addAttribute = handler;
|
|
274
|
+
}
|
|
275
|
+
} );
|
|
276
|
+
|
|
277
|
+
const httpRequest = {
|
|
278
|
+
type: Attribute.HTTPRequestCount.TYPE,
|
|
279
|
+
url: 'https://api.example.test/items',
|
|
280
|
+
requestId: 'req-1'
|
|
281
|
+
};
|
|
282
|
+
const httpCost = {
|
|
283
|
+
type: Attribute.HTTPRequestCost.TYPE,
|
|
284
|
+
url: 'https://api.example.test/items',
|
|
285
|
+
requestId: 'req-1',
|
|
286
|
+
total: 2.5
|
|
287
|
+
};
|
|
288
|
+
const llmUsage = {
|
|
289
|
+
type: Attribute.LLMUsage.TYPE,
|
|
290
|
+
modelId: 'gpt-4o',
|
|
291
|
+
total: 0.25,
|
|
292
|
+
usage: [
|
|
293
|
+
{ type: 'input', ppm: 5, amount: 20_000, total: 0.1 },
|
|
294
|
+
{ type: 'output', ppm: 30, amount: 5_000, total: 0.15 }
|
|
295
|
+
],
|
|
296
|
+
tokensUsed: 25_000
|
|
297
|
+
};
|
|
298
|
+
|
|
299
|
+
const wf = workflow( {
|
|
300
|
+
name: 'attr_wf',
|
|
301
|
+
description: 'Attributes',
|
|
302
|
+
inputSchema: z.object( {} ),
|
|
303
|
+
outputSchema: z.object( { ok: z.boolean() } ),
|
|
304
|
+
fn: async () => {
|
|
305
|
+
handlers.addAttribute( httpRequest );
|
|
306
|
+
handlers.addAttribute( httpCost );
|
|
307
|
+
handlers.addAttribute( llmUsage );
|
|
308
|
+
return { ok: true };
|
|
309
|
+
}
|
|
310
|
+
} );
|
|
311
|
+
|
|
312
|
+
const result = await wf( {} );
|
|
313
|
+
expect( result ).toEqual( {
|
|
314
|
+
output: { ok: true },
|
|
315
|
+
trace: { destinations: { local: '/tmp/trace' } },
|
|
316
|
+
attributes: [ httpRequest, httpCost, llmUsage ],
|
|
317
|
+
aggregations: {
|
|
318
|
+
cost: { total: 2.75 },
|
|
319
|
+
tokens: {
|
|
320
|
+
total: 25_000,
|
|
321
|
+
input: 20_000,
|
|
322
|
+
output: 5_000
|
|
323
|
+
},
|
|
324
|
+
httpRequests: { total: 1 }
|
|
325
|
+
}
|
|
326
|
+
} );
|
|
327
|
+
} );
|
|
328
|
+
|
|
246
329
|
it( 'sets executionContext.disableTrace when options.disableTrace is true', async () => {
|
|
247
330
|
const { workflow } = await import( './workflow.js' );
|
|
248
331
|
|
|
@@ -261,7 +344,7 @@ describe( 'workflow()', () => {
|
|
|
261
344
|
} );
|
|
262
345
|
|
|
263
346
|
describe( 'child workflow (memo.executionContext already set)', () => {
|
|
264
|
-
it( 'does not call getTraceDestinations and returns
|
|
347
|
+
it( 'does not call getTraceDestinations and returns an internal output envelope', async () => {
|
|
265
348
|
workflowInfoMock.mockReturnValue( {
|
|
266
349
|
...workflowInfoReturn,
|
|
267
350
|
memo: { executionContext: { workflowId: 'parent-1', workflowName: 'parent_wf' } }
|
|
@@ -278,7 +361,7 @@ describe( 'workflow()', () => {
|
|
|
278
361
|
|
|
279
362
|
const result = await wf( {} );
|
|
280
363
|
expect( traceDestinationsStepMock ).not.toHaveBeenCalled();
|
|
281
|
-
expect( result ).toEqual( { x: 'child' } );
|
|
364
|
+
expect( result ).toEqual( { output: { x: 'child' }, attributes: [] } );
|
|
282
365
|
} );
|
|
283
366
|
} );
|
|
284
367
|
|
|
@@ -381,6 +464,7 @@ describe( 'workflow()', () => {
|
|
|
381
464
|
it( 'calls executeChild with correct args and TERMINATE when not detached', async () => {
|
|
382
465
|
const { workflow } = await import( './workflow.js' );
|
|
383
466
|
const { ParentClosePolicy } = await import( '@temporalio/workflow' );
|
|
467
|
+
executeChildMock.mockResolvedValueOnce( { output: {}, attributes: [] } );
|
|
384
468
|
|
|
385
469
|
const wf = workflow( {
|
|
386
470
|
name: 'parent_wf',
|
|
@@ -408,6 +492,7 @@ describe( 'workflow()', () => {
|
|
|
408
492
|
it( 'uses ABANDON when extra.detached is true', async () => {
|
|
409
493
|
const { workflow } = await import( './workflow.js' );
|
|
410
494
|
const { ParentClosePolicy } = await import( '@temporalio/workflow' );
|
|
495
|
+
executeChildMock.mockResolvedValueOnce( { output: {}, attributes: [] } );
|
|
411
496
|
|
|
412
497
|
const wf = workflow( {
|
|
413
498
|
name: 'detach_wf',
|
|
@@ -428,6 +513,7 @@ describe( 'workflow()', () => {
|
|
|
428
513
|
|
|
429
514
|
it( 'passes empty args when input is null/omitted', async () => {
|
|
430
515
|
const { workflow } = await import( './workflow.js' );
|
|
516
|
+
executeChildMock.mockResolvedValueOnce( { output: {}, attributes: [] } );
|
|
431
517
|
|
|
432
518
|
const wf = workflow( {
|
|
433
519
|
name: 'no_input_wf',
|
|
@@ -445,11 +531,96 @@ describe( 'workflow()', () => {
|
|
|
445
531
|
args: []
|
|
446
532
|
} ) );
|
|
447
533
|
} );
|
|
534
|
+
|
|
535
|
+
it( 'returns child output and merges child attributes into the root result', async () => {
|
|
536
|
+
const { workflow } = await import( './workflow.js' );
|
|
537
|
+
const { Attribute } = await import( '#trace_attribute' );
|
|
538
|
+
const childAttribute = {
|
|
539
|
+
type: Attribute.LLMUsage.TYPE,
|
|
540
|
+
modelId: 'gpt-4o',
|
|
541
|
+
total: 0.4,
|
|
542
|
+
tokensUsed: 20,
|
|
543
|
+
usage: [
|
|
544
|
+
{ type: 'input', ppm: 10, amount: 20, total: 0.4 }
|
|
545
|
+
]
|
|
546
|
+
};
|
|
547
|
+
executeChildMock.mockResolvedValueOnce( {
|
|
548
|
+
output: { child: 'ok' },
|
|
549
|
+
attributes: [ childAttribute ]
|
|
550
|
+
} );
|
|
551
|
+
|
|
552
|
+
const wf = workflow( {
|
|
553
|
+
name: 'merge_child_wf',
|
|
554
|
+
description: 'Merge child attributes',
|
|
555
|
+
inputSchema: z.object( {} ),
|
|
556
|
+
outputSchema: z.object( { child: z.string() } ),
|
|
557
|
+
async fn() {
|
|
558
|
+
return this.startWorkflow( 'child_wf', { id: 1 } );
|
|
559
|
+
}
|
|
560
|
+
} );
|
|
561
|
+
|
|
562
|
+
const result = await wf( {} );
|
|
563
|
+
expect( result ).toEqual( {
|
|
564
|
+
output: { child: 'ok' },
|
|
565
|
+
trace: { destinations: { local: '/tmp/trace' } },
|
|
566
|
+
attributes: [ childAttribute ],
|
|
567
|
+
aggregations: {
|
|
568
|
+
cost: { total: 0.4 },
|
|
569
|
+
tokens: {
|
|
570
|
+
total: 20,
|
|
571
|
+
input: 20
|
|
572
|
+
},
|
|
573
|
+
httpRequests: { total: 0 }
|
|
574
|
+
}
|
|
575
|
+
} );
|
|
576
|
+
} );
|
|
577
|
+
|
|
578
|
+
it( 'merges child error attributes before rethrowing to root metadata', async () => {
|
|
579
|
+
const { workflow } = await import( './workflow.js' );
|
|
580
|
+
const { ChildWorkflowFailure } = await import( '@temporalio/workflow' );
|
|
581
|
+
const { METADATA_ACCESS_SYMBOL } = await import( '#consts' );
|
|
582
|
+
const { Attribute } = await import( '#trace_attribute' );
|
|
583
|
+
const childAttribute = {
|
|
584
|
+
type: Attribute.HTTPRequestCost.TYPE,
|
|
585
|
+
url: 'https://api.example.test',
|
|
586
|
+
requestId: 'req-child',
|
|
587
|
+
total: 2
|
|
588
|
+
};
|
|
589
|
+
const childError = new ChildWorkflowFailure( 'child failed', {
|
|
590
|
+
message: 'Child workflow execution failed',
|
|
591
|
+
details: [ { attributes: [ childAttribute ] } ]
|
|
592
|
+
} );
|
|
593
|
+
executeChildMock.mockRejectedValueOnce( childError );
|
|
594
|
+
|
|
595
|
+
const wf = workflow( {
|
|
596
|
+
name: 'child_error_wf',
|
|
597
|
+
description: 'Child error attributes',
|
|
598
|
+
inputSchema: z.object( {} ),
|
|
599
|
+
outputSchema: z.object( {} ),
|
|
600
|
+
async fn() {
|
|
601
|
+
await this.startWorkflow( 'child_wf', { id: 1 } );
|
|
602
|
+
return {};
|
|
603
|
+
}
|
|
604
|
+
} );
|
|
605
|
+
|
|
606
|
+
await expect( wf( {} ) ).rejects.toThrow( 'child failed' );
|
|
607
|
+
expect( childError[METADATA_ACCESS_SYMBOL] ).toEqual( {
|
|
608
|
+
attributes: [ childAttribute ],
|
|
609
|
+
trace: { destinations: { local: '/tmp/trace' } },
|
|
610
|
+
aggregations: {
|
|
611
|
+
cost: { total: 2 },
|
|
612
|
+
tokens: { total: 0 },
|
|
613
|
+
httpRequests: { total: 0 }
|
|
614
|
+
}
|
|
615
|
+
} );
|
|
616
|
+
} );
|
|
448
617
|
} );
|
|
449
618
|
|
|
450
619
|
describe( 'error handling (root workflow)', () => {
|
|
451
|
-
it( 'rethrows error from fn
|
|
620
|
+
it( 'rethrows error from fn with trace attributes and aggregation metadata', async () => {
|
|
452
621
|
const { workflow } = await import( './workflow.js' );
|
|
622
|
+
const { METADATA_ACCESS_SYMBOL } = await import( '#consts' );
|
|
623
|
+
const error = new Error( 'workflow failed' );
|
|
453
624
|
|
|
454
625
|
const wf = workflow( {
|
|
455
626
|
name: 'err_wf',
|
|
@@ -457,11 +628,16 @@ describe( 'workflow()', () => {
|
|
|
457
628
|
inputSchema: z.object( {} ),
|
|
458
629
|
outputSchema: z.object( {} ),
|
|
459
630
|
fn: async () => {
|
|
460
|
-
throw
|
|
631
|
+
throw error;
|
|
461
632
|
}
|
|
462
633
|
} );
|
|
463
634
|
|
|
464
635
|
await expect( wf( {} ) ).rejects.toThrow( 'workflow failed' );
|
|
636
|
+
expect( error[METADATA_ACCESS_SYMBOL] ).toEqual( {
|
|
637
|
+
trace: { destinations: { local: '/tmp/trace' } },
|
|
638
|
+
attributes: [],
|
|
639
|
+
aggregations: emptyAggregations
|
|
640
|
+
} );
|
|
465
641
|
} );
|
|
466
642
|
} );
|
|
467
643
|
} );
|
|
@@ -7,11 +7,12 @@ export class Context {
|
|
|
7
7
|
* Builds a new context instance
|
|
8
8
|
* @param {object} options - Arguments to build a new context instance
|
|
9
9
|
* @param {string} workflowId
|
|
10
|
+
* @param {string} runId
|
|
10
11
|
* @param {function} continueAsNew
|
|
11
12
|
* @param {function} isContinueAsNewSuggested
|
|
12
13
|
* @returns {object} context
|
|
13
14
|
*/
|
|
14
|
-
static build( { workflowId, continueAsNew, isContinueAsNewSuggested } ) {
|
|
15
|
+
static build( { workflowId, runId, continueAsNew, isContinueAsNewSuggested } ) {
|
|
15
16
|
return {
|
|
16
17
|
/**
|
|
17
18
|
* Control namespace: This object adds functions to interact with Temporal flow mechanisms
|
|
@@ -24,7 +25,8 @@ export class Context {
|
|
|
24
25
|
* Info namespace: abstracts workflowInfo()
|
|
25
26
|
*/
|
|
26
27
|
info: {
|
|
27
|
-
workflowId
|
|
28
|
+
workflowId,
|
|
29
|
+
runId
|
|
28
30
|
}
|
|
29
31
|
};
|
|
30
32
|
}
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import { appendFileSync, mkdirSync, readdirSync, readFileSync, rmSync,
|
|
1
|
+
import { appendFileSync, mkdirSync, readdirSync, readFileSync, rmSync, createWriteStream } from 'node:fs';
|
|
2
2
|
import { dirname, join } from 'node:path';
|
|
3
3
|
import { fileURLToPath } from 'url';
|
|
4
4
|
import buildTraceTree from '../../tools/build_trace_tree.js';
|
|
5
|
-
import { safeFormatJSON } from '../../tools/utils.js';
|
|
6
5
|
import { EOL } from 'node:os';
|
|
6
|
+
import { JsonStreamStringify } from 'json-stream-stringify';
|
|
7
|
+
|
|
8
|
+
import { pipeline } from 'stream/promises';
|
|
7
9
|
|
|
8
10
|
const __dirname = dirname( fileURLToPath( import.meta.url ) );
|
|
9
11
|
|
|
@@ -109,7 +111,7 @@ export const init = () => {
|
|
|
109
111
|
* @param {object} args.executionContext - Execution info: workflowId, workflowName, startTime
|
|
110
112
|
* @returns {void}
|
|
111
113
|
*/
|
|
112
|
-
export const exec = ( { entry, executionContext } ) => {
|
|
114
|
+
export const exec = async ( { entry, executionContext } ) => {
|
|
113
115
|
const { workflowId, workflowName, startTime } = executionContext;
|
|
114
116
|
const tempFilePath = createTempFilePath( executionContext );
|
|
115
117
|
addEntry( entry, tempFilePath );
|
|
@@ -126,7 +128,11 @@ export const exec = ( { entry, executionContext } ) => {
|
|
|
126
128
|
const path = join( dir, buildTraceFilename( { startTime, workflowId } ) );
|
|
127
129
|
|
|
128
130
|
mkdirSync( dir, { recursive: true } );
|
|
129
|
-
|
|
131
|
+
|
|
132
|
+
await pipeline(
|
|
133
|
+
new JsonStreamStringify( content ),
|
|
134
|
+
createWriteStream( path )
|
|
135
|
+
);
|
|
130
136
|
};
|
|
131
137
|
|
|
132
138
|
/**
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
2
|
-
import { EOL } from 'node:os';
|
|
3
2
|
|
|
4
3
|
// In-memory fs mock store
|
|
5
4
|
const store = { files: new Map() };
|
|
@@ -12,6 +11,7 @@ const appendFileSyncMock = vi.fn( ( path, data ) => {
|
|
|
12
11
|
const readFileSyncMock = vi.fn( path => store.files.get( path ) ?? '' );
|
|
13
12
|
const readdirSyncMock = vi.fn( () => [] );
|
|
14
13
|
const rmSyncMock = vi.fn();
|
|
14
|
+
const createWriteStreamMock = vi.fn( path => ( { path } ) );
|
|
15
15
|
|
|
16
16
|
vi.mock( 'node:fs', () => ( {
|
|
17
17
|
mkdirSync: mkdirSyncMock,
|
|
@@ -19,9 +19,36 @@ vi.mock( 'node:fs', () => ( {
|
|
|
19
19
|
appendFileSync: appendFileSyncMock,
|
|
20
20
|
readFileSync: readFileSyncMock,
|
|
21
21
|
readdirSync: readdirSyncMock,
|
|
22
|
-
rmSync: rmSyncMock
|
|
22
|
+
rmSync: rmSyncMock,
|
|
23
|
+
createWriteStream: createWriteStreamMock
|
|
23
24
|
} ) );
|
|
24
25
|
|
|
26
|
+
const pipelineMock = vi.fn( async ( source, destination ) => {
|
|
27
|
+
const chunks = [];
|
|
28
|
+
for await ( const chunk of source ) {
|
|
29
|
+
chunks.push( Buffer.isBuffer( chunk ) ? chunk : Buffer.from( chunk ) );
|
|
30
|
+
}
|
|
31
|
+
store.files.set( destination.path, Buffer.concat( chunks ).toString( 'utf8' ) );
|
|
32
|
+
} );
|
|
33
|
+
vi.mock( 'node:stream/promises', () => ( { pipeline: pipelineMock } ) );
|
|
34
|
+
|
|
35
|
+
vi.mock( 'json-stream-stringify', async () => {
|
|
36
|
+
const { Readable } = await import( 'node:stream' );
|
|
37
|
+
return {
|
|
38
|
+
JsonStreamStringify: class extends Readable {
|
|
39
|
+
constructor( body ) {
|
|
40
|
+
super();
|
|
41
|
+
this.body = body;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
_read() {
|
|
45
|
+
this.push( JSON.stringify( this.body ) );
|
|
46
|
+
this.push( null );
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
} );
|
|
51
|
+
|
|
25
52
|
const buildTraceTreeMock = vi.fn( entries => ( { count: entries.length } ) );
|
|
26
53
|
vi.mock( '../../tools/build_trace_tree.js', () => ( { default: buildTraceTreeMock } ) );
|
|
27
54
|
|
|
@@ -58,17 +85,18 @@ describe( 'tracing/processors/local', () => {
|
|
|
58
85
|
const workflowId = 'id1';
|
|
59
86
|
const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
|
|
60
87
|
|
|
61
|
-
exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
62
|
-
exec( { ...ctx, entry: childTick( 'child-1', startTime + 1 ) } );
|
|
63
|
-
exec( { ...ctx, entry: rootEnd( workflowId, startTime + 2 ) } );
|
|
88
|
+
await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
89
|
+
await exec( { ...ctx, entry: childTick( 'child-1', startTime + 1 ) } );
|
|
90
|
+
await exec( { ...ctx, entry: rootEnd( workflowId, startTime + 2 ) } );
|
|
64
91
|
|
|
65
92
|
expect( buildTraceTreeMock ).toHaveBeenCalledTimes( 1 );
|
|
66
93
|
expect( buildTraceTreeMock.mock.calls[0][0] ).toHaveLength( 3 );
|
|
67
94
|
|
|
68
|
-
expect(
|
|
69
|
-
|
|
95
|
+
expect( createWriteStreamMock ).toHaveBeenCalledTimes( 1 );
|
|
96
|
+
expect( pipelineMock ).toHaveBeenCalledTimes( 1 );
|
|
97
|
+
const [ writtenPath ] = createWriteStreamMock.mock.calls[0];
|
|
70
98
|
expect( writtenPath ).toMatch( /\/tmp\/project\/logs\/runs\/WF\// );
|
|
71
|
-
expect( JSON.parse(
|
|
99
|
+
expect( JSON.parse( store.files.get( writtenPath ) ).count ).toBe( 3 );
|
|
72
100
|
} );
|
|
73
101
|
|
|
74
102
|
it( 'exec(): does not build or write on non-flush entries', async () => {
|
|
@@ -79,11 +107,13 @@ describe( 'tracing/processors/local', () => {
|
|
|
79
107
|
const workflowId = 'id1';
|
|
80
108
|
const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
|
|
81
109
|
|
|
82
|
-
exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
83
|
-
exec( { ...ctx, entry: childTick( 'child-1', startTime + 1 ) } );
|
|
110
|
+
await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
111
|
+
await exec( { ...ctx, entry: childTick( 'child-1', startTime + 1 ) } );
|
|
84
112
|
|
|
85
113
|
expect( buildTraceTreeMock ).not.toHaveBeenCalled();
|
|
86
114
|
expect( writeFileSyncMock ).not.toHaveBeenCalled();
|
|
115
|
+
expect( createWriteStreamMock ).not.toHaveBeenCalled();
|
|
116
|
+
expect( pipelineMock ).not.toHaveBeenCalled();
|
|
87
117
|
} );
|
|
88
118
|
|
|
89
119
|
it( 'exec(): flushes on error action before root end', async () => {
|
|
@@ -94,12 +124,13 @@ describe( 'tracing/processors/local', () => {
|
|
|
94
124
|
const workflowId = 'id1';
|
|
95
125
|
const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
|
|
96
126
|
|
|
97
|
-
exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
98
|
-
exec( { ...ctx, entry: { id: 'step-1', action: 'error', timestamp: startTime + 1 } } );
|
|
127
|
+
await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
128
|
+
await exec( { ...ctx, entry: { id: 'step-1', action: 'error', timestamp: startTime + 1 } } );
|
|
99
129
|
|
|
100
130
|
expect( buildTraceTreeMock ).toHaveBeenCalledTimes( 1 );
|
|
101
131
|
expect( buildTraceTreeMock.mock.calls[0][0] ).toHaveLength( 2 );
|
|
102
|
-
expect(
|
|
132
|
+
expect( createWriteStreamMock ).toHaveBeenCalledTimes( 1 );
|
|
133
|
+
expect( pipelineMock ).toHaveBeenCalledTimes( 1 );
|
|
103
134
|
} );
|
|
104
135
|
|
|
105
136
|
it( 'getDestination(): returns absolute path under callerDir logs', async () => {
|
|
@@ -128,11 +159,11 @@ describe( 'tracing/processors/local', () => {
|
|
|
128
159
|
const workflowId = 'id1';
|
|
129
160
|
const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
|
|
130
161
|
|
|
131
|
-
exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
132
|
-
exec( { ...ctx, entry: rootEnd( workflowId, startTime + 1 ) } );
|
|
162
|
+
await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
163
|
+
await exec( { ...ctx, entry: rootEnd( workflowId, startTime + 1 ) } );
|
|
133
164
|
|
|
134
|
-
expect(
|
|
135
|
-
const [ writtenPath ] =
|
|
165
|
+
expect( createWriteStreamMock ).toHaveBeenCalledTimes( 1 );
|
|
166
|
+
const [ writtenPath ] = createWriteStreamMock.mock.calls[0];
|
|
136
167
|
|
|
137
168
|
expect( writtenPath ).not.toContain( '/host/path/logs' );
|
|
138
169
|
expect( writtenPath ).toMatch( /\/tmp\/project\/logs\/runs\/WF\// );
|
|
@@ -164,15 +195,15 @@ describe( 'tracing/processors/local', () => {
|
|
|
164
195
|
const workflowName = 'test-workflow';
|
|
165
196
|
const ctx = { executionContext: { workflowId, workflowName, startTime } };
|
|
166
197
|
|
|
167
|
-
exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
168
|
-
exec( { ...ctx, entry: rootEnd( workflowId, startTime + 1 ) } );
|
|
198
|
+
await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
199
|
+
await exec( { ...ctx, entry: rootEnd( workflowId, startTime + 1 ) } );
|
|
169
200
|
|
|
170
201
|
const destination = getDestination( { startTime, workflowId, workflowName } );
|
|
171
202
|
|
|
172
|
-
const [ writtenPath
|
|
203
|
+
const [ writtenPath ] = createWriteStreamMock.mock.calls[0];
|
|
173
204
|
expect( writtenPath ).not.toContain( '/Users/ben/project' );
|
|
174
205
|
expect( writtenPath ).toMatch( /\/tmp\/project\/logs\/runs\/test-workflow\// );
|
|
175
|
-
expect(
|
|
206
|
+
expect( JSON.parse( store.files.get( writtenPath ) ).count ).toBe( 2 );
|
|
176
207
|
|
|
177
208
|
expect( destination ).toBe( '/Users/ben/project/logs/runs/test-workflow/2020-01-02-03-04-05-678Z_workflow-id-123.json' );
|
|
178
209
|
} );
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import { upload } from './s3_client.js';
|
|
2
2
|
import { getRedisClient } from './redis_client.js';
|
|
3
3
|
import buildTraceTree from '../../tools/build_trace_tree.js';
|
|
4
|
-
import { EOL } from 'node:os';
|
|
5
4
|
import { loadEnv, getVars } from './configs.js';
|
|
6
5
|
import { createChildLogger } from '#logger';
|
|
7
|
-
import {
|
|
6
|
+
import { JsonStreamStringify } from 'json-stream-stringify';
|
|
8
7
|
|
|
9
8
|
const log = createChildLogger( 'S3 Processor' );
|
|
10
9
|
|
|
@@ -100,9 +99,10 @@ export const exec = async ( { entry, executionContext } ) => {
|
|
|
100
99
|
log.warn( 'Incomplete trace file discarded', { workflowId, error: 'incomplete_trace_file' } );
|
|
101
100
|
return;
|
|
102
101
|
}
|
|
102
|
+
|
|
103
103
|
await upload( {
|
|
104
104
|
key: getS3Key( { workflowId, workflowName, startTime } ),
|
|
105
|
-
content:
|
|
105
|
+
content: new JsonStreamStringify( content )
|
|
106
106
|
} );
|
|
107
107
|
await bustEntries( cacheKey );
|
|
108
108
|
};
|
|
@@ -27,6 +27,31 @@ vi.mock( './s3_client.js', () => ( { upload: uploadMock } ) );
|
|
|
27
27
|
const buildTraceTreeMock = vi.fn( entries => ( { count: entries.length } ) );
|
|
28
28
|
vi.mock( '../../tools/build_trace_tree.js', () => ( { default: buildTraceTreeMock } ) );
|
|
29
29
|
|
|
30
|
+
vi.mock( 'json-stream-stringify', async () => {
|
|
31
|
+
const { Readable } = await import( 'node:stream' );
|
|
32
|
+
return {
|
|
33
|
+
JsonStreamStringify: class extends Readable {
|
|
34
|
+
constructor( body ) {
|
|
35
|
+
super();
|
|
36
|
+
this.body = body;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
_read() {
|
|
40
|
+
this.push( JSON.stringify( this.body ) );
|
|
41
|
+
this.push( null );
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
} );
|
|
46
|
+
|
|
47
|
+
const streamToString = async stream => {
|
|
48
|
+
const chunks = [];
|
|
49
|
+
for await ( const chunk of stream ) {
|
|
50
|
+
chunks.push( Buffer.isBuffer( chunk ) ? chunk : Buffer.from( chunk ) );
|
|
51
|
+
}
|
|
52
|
+
return Buffer.concat( chunks ).toString( 'utf8' );
|
|
53
|
+
};
|
|
54
|
+
|
|
30
55
|
describe( 'tracing/processors/s3', () => {
|
|
31
56
|
beforeEach( () => {
|
|
32
57
|
vi.useFakeTimers();
|
|
@@ -74,7 +99,7 @@ describe( 'tracing/processors/s3', () => {
|
|
|
74
99
|
expect( uploadMock ).toHaveBeenCalledTimes( 1 );
|
|
75
100
|
const { key, content } = uploadMock.mock.calls[0][0];
|
|
76
101
|
expect( key ).toMatch( /^WF\/2020\/01\/02\// );
|
|
77
|
-
expect( JSON.parse( content
|
|
102
|
+
expect( JSON.parse( await streamToString( content ) ).count ).toBe( 3 );
|
|
78
103
|
expect( delMock ).toHaveBeenCalledTimes( 1 );
|
|
79
104
|
expect( delMock ).toHaveBeenCalledWith( 'traces/WF/id1' );
|
|
80
105
|
} );
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { S3Client
|
|
1
|
+
import { S3Client } from '@aws-sdk/client-s3';
|
|
2
|
+
import { Upload } from '@aws-sdk/lib-storage';
|
|
2
3
|
import { getVars } from './configs.js';
|
|
3
4
|
|
|
4
5
|
const state = { s3Client: null };
|
|
@@ -21,7 +22,14 @@ const getS3Client = () => {
|
|
|
21
22
|
* Upload given file to S3
|
|
22
23
|
* @param {object} args
|
|
23
24
|
* @param {string} key - S3 file key
|
|
24
|
-
* @param {string} content - File content
|
|
25
|
+
* @param {string|import('node:stream').Readable} content - File content
|
|
25
26
|
*/
|
|
26
27
|
export const upload = ( { key, content } ) =>
|
|
27
|
-
|
|
28
|
+
new Upload( {
|
|
29
|
+
client: getS3Client(),
|
|
30
|
+
params: {
|
|
31
|
+
Bucket: getVars().remoteS3Bucket,
|
|
32
|
+
Key: key,
|
|
33
|
+
Body: content
|
|
34
|
+
}
|
|
35
|
+
} ).done();
|
|
@@ -9,24 +9,27 @@ vi.mock( '#utils', () => ( {
|
|
|
9
9
|
const getVarsMock = vi.fn();
|
|
10
10
|
vi.mock( './configs', () => ( { getVars: () => getVarsMock() } ) );
|
|
11
11
|
|
|
12
|
-
const sendMock = vi.fn();
|
|
13
12
|
const ctorState = { args: null };
|
|
14
13
|
class S3ClientMock {
|
|
15
14
|
constructor( args ) {
|
|
16
15
|
ctorState.args = args;
|
|
17
|
-
} send = sendMock;
|
|
18
|
-
}
|
|
19
|
-
class PutObjectCommandMock {
|
|
20
|
-
constructor( input ) {
|
|
21
|
-
this.input = input;
|
|
22
16
|
}
|
|
23
17
|
}
|
|
24
|
-
|
|
25
18
|
vi.mock( '@aws-sdk/client-s3', () => ( {
|
|
26
|
-
S3Client: S3ClientMock
|
|
27
|
-
PutObjectCommand: PutObjectCommandMock
|
|
19
|
+
S3Client: S3ClientMock
|
|
28
20
|
} ) );
|
|
29
21
|
|
|
22
|
+
const uploadDoneMock = vi.fn();
|
|
23
|
+
const uploadCtorState = { args: [] };
|
|
24
|
+
class UploadMock {
|
|
25
|
+
constructor( args ) {
|
|
26
|
+
uploadCtorState.args.push( args );
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
done = uploadDoneMock;
|
|
30
|
+
}
|
|
31
|
+
vi.mock( '@aws-sdk/lib-storage', () => ( { Upload: UploadMock } ) );
|
|
32
|
+
|
|
30
33
|
async function loadModule() {
|
|
31
34
|
vi.resetModules();
|
|
32
35
|
return import( './s3_client.js' );
|
|
@@ -35,6 +38,9 @@ async function loadModule() {
|
|
|
35
38
|
describe( 'tracing/processors/s3/s3_client', () => {
|
|
36
39
|
beforeEach( () => {
|
|
37
40
|
vi.clearAllMocks();
|
|
41
|
+
ctorState.args = null;
|
|
42
|
+
uploadCtorState.args = [];
|
|
43
|
+
uploadDoneMock.mockResolvedValue( undefined );
|
|
38
44
|
getVarsMock.mockReturnValue( {
|
|
39
45
|
awsRegion: 'us-east-1',
|
|
40
46
|
awsAccessKeyId: 'id',
|
|
@@ -48,15 +54,21 @@ describe( 'tracing/processors/s3/s3_client', () => {
|
|
|
48
54
|
|
|
49
55
|
await upload( { key: 'wf/key.json', content: '{"a":1}' } );
|
|
50
56
|
|
|
51
|
-
expect( ctorState.args ).toEqual( {
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
expect(
|
|
57
|
+
expect( ctorState.args ).toEqual( {
|
|
58
|
+
region: 'us-east-1',
|
|
59
|
+
credentials: { secretAccessKey: 'sek', accessKeyId: 'id' }
|
|
60
|
+
} );
|
|
61
|
+
expect( uploadCtorState.args ).toHaveLength( 1 );
|
|
62
|
+
expect( uploadCtorState.args[0] ).toEqual( {
|
|
63
|
+
client: expect.any( S3ClientMock ),
|
|
64
|
+
params: { Bucket: 'bucket', Key: 'wf/key.json', Body: '{"a":1}' }
|
|
65
|
+
} );
|
|
66
|
+
expect( uploadDoneMock ).toHaveBeenCalledTimes( 1 );
|
|
56
67
|
|
|
57
68
|
// subsequent upload uses cached client
|
|
58
69
|
await upload( { key: 'wf/key2.json', content: '{}' } );
|
|
59
|
-
expect(
|
|
70
|
+
expect( uploadCtorState.args ).toHaveLength( 2 );
|
|
71
|
+
expect( uploadDoneMock ).toHaveBeenCalledTimes( 2 );
|
|
60
72
|
} );
|
|
61
73
|
} );
|
|
62
74
|
|
|
@@ -62,7 +62,7 @@ export default entries => {
|
|
|
62
62
|
if ( action === EventAction.START ) {
|
|
63
63
|
Object.assign( node, { input: details, startedAt: timestamp, kind, name } );
|
|
64
64
|
} else if ( action === EventAction.ADD_ATTR ) {
|
|
65
|
-
node.attributes[details.
|
|
65
|
+
node.attributes[details.type] = details;
|
|
66
66
|
} else if ( action === EventAction.END ) {
|
|
67
67
|
Object.assign( node, { output: details, endedAt: timestamp } );
|
|
68
68
|
} else if ( action === EventAction.ERROR ) {
|