@outputai/core 0.6.0 → 0.6.1-next.383b24b.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/activity_integration/context.d.ts +5 -9
- package/src/activity_integration/context.js +5 -4
- package/src/activity_integration/context.spec.js +10 -15
- package/src/activity_integration/events.d.ts +2 -4
- package/src/activity_integration/events.js +8 -3
- package/src/activity_integration/events.spec.js +58 -29
- package/src/bus.js +18 -9
- package/src/bus.spec.js +30 -0
- package/src/hooks/index.d.ts +112 -58
- package/src/hooks/index.js +15 -12
- package/src/hooks/index.spec.js +60 -32
- package/src/interface/workflow.js +19 -35
- package/src/interface/workflow.spec.js +104 -15
- package/src/internal_activities/index.js +3 -3
- package/src/internal_activities/index.spec.js +31 -1
- package/src/internal_utils/temporal_context.js +12 -0
- package/src/internal_utils/temporal_context.spec.ts +83 -0
- package/src/internal_utils/trace_info.js +21 -0
- package/src/internal_utils/trace_info.spec.js +47 -0
- package/src/internal_utils/workflow_context.js +29 -0
- package/src/internal_utils/workflow_context.spec.js +46 -0
- package/src/tracing/internal_interface.js +4 -4
- package/src/tracing/processors/local/index.js +21 -26
- package/src/tracing/processors/local/index.spec.js +39 -45
- package/src/tracing/processors/s3/index.js +13 -23
- package/src/tracing/processors/s3/index.spec.js +33 -26
- package/src/tracing/trace_attribute.js +0 -1
- package/src/tracing/trace_engine.js +8 -12
- package/src/tracing/trace_engine.spec.js +31 -27
- package/src/worker/interceptors/activity.js +31 -29
- package/src/worker/interceptors/activity.spec.js +58 -26
- package/src/worker/interceptors/workflow.js +7 -2
- package/src/worker/interceptors/workflow.spec.js +42 -6
- package/src/worker/log_hooks.js +35 -46
- package/src/worker/log_hooks.spec.js +43 -46
- package/src/worker/sinks.js +24 -24
- package/src/interface/workflow_context.js +0 -33
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { describe, expect, it, vi, beforeEach } from 'vitest';
|
|
2
|
+
import { TraceInfo } from './trace_info.js';
|
|
3
|
+
|
|
4
|
+
const inWorkflowContextMock = vi.hoisted( () => vi.fn() );
|
|
5
|
+
const workflowInfoMock = vi.hoisted( () => vi.fn() );
|
|
6
|
+
|
|
7
|
+
vi.mock( '@temporalio/workflow', () => ( {
|
|
8
|
+
inWorkflowContext: inWorkflowContextMock,
|
|
9
|
+
workflowInfo: workflowInfoMock
|
|
10
|
+
} ) );
|
|
11
|
+
|
|
12
|
+
describe( 'TraceInfo', () => {
|
|
13
|
+
beforeEach( () => {
|
|
14
|
+
vi.clearAllMocks();
|
|
15
|
+
} );
|
|
16
|
+
|
|
17
|
+
it( 'builds trace info from Temporal workflow info in workflow context', () => {
|
|
18
|
+
inWorkflowContextMock.mockReturnValue( true );
|
|
19
|
+
workflowInfoMock.mockReturnValue( {
|
|
20
|
+
workflowId: 'workflow-id',
|
|
21
|
+
workflowType: 'workflow-type',
|
|
22
|
+
runId: 'run-id',
|
|
23
|
+
startTime: new Date( '2026-06-02T09:00:00.000Z' )
|
|
24
|
+
} );
|
|
25
|
+
|
|
26
|
+
expect( TraceInfo.build( { disableTrace: false } ) ).toEqual( {
|
|
27
|
+
workflowId: 'workflow-id',
|
|
28
|
+
workflowType: 'workflow-type',
|
|
29
|
+
runId: 'run-id',
|
|
30
|
+
startTime: Date.parse( '2026-06-02T09:00:00.000Z' ),
|
|
31
|
+
disableTrace: false
|
|
32
|
+
} );
|
|
33
|
+
} );
|
|
34
|
+
|
|
35
|
+
it( 'builds trace info without Temporal fields outside workflow context', () => {
|
|
36
|
+
inWorkflowContextMock.mockReturnValue( false );
|
|
37
|
+
|
|
38
|
+
expect( TraceInfo.build( { disableTrace: true } ) ).toEqual( {
|
|
39
|
+
workflowId: undefined,
|
|
40
|
+
workflowType: undefined,
|
|
41
|
+
runId: undefined,
|
|
42
|
+
startTime: undefined,
|
|
43
|
+
disableTrace: true
|
|
44
|
+
} );
|
|
45
|
+
expect( workflowInfoMock ).not.toHaveBeenCalled();
|
|
46
|
+
} );
|
|
47
|
+
} );
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { workflowInfo, continueAsNew, inWorkflowContext } from '@temporalio/workflow';
|
|
2
|
+
|
|
3
|
+
export class WorkflowContext {
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Builds a new context instance
|
|
7
|
+
* @returns {object} context
|
|
8
|
+
*/
|
|
9
|
+
static build() {
|
|
10
|
+
if ( !inWorkflowContext() ) {
|
|
11
|
+
return {
|
|
12
|
+
control: {
|
|
13
|
+
continueAsNew: async () => {},
|
|
14
|
+
isContinueAsNewSuggested: () => false
|
|
15
|
+
},
|
|
16
|
+
info: { workflowId: 'test-workflow', runId: 'test-run' }
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const { workflowId, runId } = workflowInfo();
|
|
21
|
+
return {
|
|
22
|
+
control: {
|
|
23
|
+
continueAsNew,
|
|
24
|
+
isContinueAsNewSuggested: () => workflowInfo().continueAsNewSuggested
|
|
25
|
+
},
|
|
26
|
+
info: { workflowId, runId }
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
};
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { describe, expect, it, vi, beforeEach } from 'vitest';
|
|
2
|
+
import { WorkflowContext } from './workflow_context.js';
|
|
3
|
+
|
|
4
|
+
const inWorkflowContextMock = vi.hoisted( () => vi.fn() );
|
|
5
|
+
const workflowInfoMock = vi.hoisted( () => vi.fn() );
|
|
6
|
+
const continueAsNewMock = vi.hoisted( () => vi.fn() );
|
|
7
|
+
|
|
8
|
+
vi.mock( '@temporalio/workflow', () => ( {
|
|
9
|
+
continueAsNew: continueAsNewMock,
|
|
10
|
+
inWorkflowContext: inWorkflowContextMock,
|
|
11
|
+
workflowInfo: workflowInfoMock
|
|
12
|
+
} ) );
|
|
13
|
+
|
|
14
|
+
describe( 'WorkflowContext', () => {
|
|
15
|
+
beforeEach( () => {
|
|
16
|
+
vi.clearAllMocks();
|
|
17
|
+
} );
|
|
18
|
+
|
|
19
|
+
it( 'builds a test context outside Temporal workflow context', async () => {
|
|
20
|
+
inWorkflowContextMock.mockReturnValue( false );
|
|
21
|
+
|
|
22
|
+
const context = WorkflowContext.build();
|
|
23
|
+
|
|
24
|
+
expect( context.info ).toEqual( { workflowId: 'test-workflow', runId: 'test-run' } );
|
|
25
|
+
expect( context.control.isContinueAsNewSuggested() ).toBe( false );
|
|
26
|
+
await expect( context.control.continueAsNew() ).resolves.toBeUndefined();
|
|
27
|
+
expect( workflowInfoMock ).not.toHaveBeenCalled();
|
|
28
|
+
expect( continueAsNewMock ).not.toHaveBeenCalled();
|
|
29
|
+
} );
|
|
30
|
+
|
|
31
|
+
it( 'builds a workflow context from Temporal workflow info', () => {
|
|
32
|
+
inWorkflowContextMock.mockReturnValue( true );
|
|
33
|
+
workflowInfoMock.mockReturnValue( {
|
|
34
|
+
workflowId: 'workflow-id',
|
|
35
|
+
runId: 'run-id',
|
|
36
|
+
continueAsNewSuggested: true
|
|
37
|
+
} );
|
|
38
|
+
|
|
39
|
+
const context = WorkflowContext.build();
|
|
40
|
+
|
|
41
|
+
expect( context.info ).toEqual( { workflowId: 'workflow-id', runId: 'run-id' } );
|
|
42
|
+
expect( context.control.continueAsNew ).toBe( continueAsNewMock );
|
|
43
|
+
expect( context.control.isContinueAsNewSuggested() ).toBe( true );
|
|
44
|
+
expect( workflowInfoMock ).toHaveBeenCalledTimes( 2 );
|
|
45
|
+
} );
|
|
46
|
+
} );
|
|
@@ -32,7 +32,7 @@ export { EventAction };
|
|
|
32
32
|
* @param {string} args.name - The human-friendly name of the Event: query, request, create.
|
|
33
33
|
* @param {any} args.details - Arbitrary data to add to this event, it will be used as the "input" field.
|
|
34
34
|
* @param {string} args.parentId - The parent Event, used to build a tree.
|
|
35
|
-
* @param {object} args.
|
|
35
|
+
* @param {object} args.traceInfo - The trace information object, propagated from the root-most workflow.
|
|
36
36
|
* @returns {void}
|
|
37
37
|
*/
|
|
38
38
|
export const addEventStart = options => addEventAction( EventAction.START, options );
|
|
@@ -45,7 +45,7 @@ export const addEventStart = options => addEventAction( EventAction.START, optio
|
|
|
45
45
|
* @param {object} args
|
|
46
46
|
* @param {string} args.id - The id of the event to conclude.
|
|
47
47
|
* @param {any} args.details - Arbitrary data to add to the event; it is used as the "output" field.
|
|
48
|
-
* @param {object} args.
|
|
48
|
+
* @param {object} args.traceInfo - The trace information object, propagated from the root-most workflow.
|
|
49
49
|
* @returns {void}
|
|
50
50
|
*/
|
|
51
51
|
export const addEventEnd = options => addEventAction( EventAction.END, options );
|
|
@@ -58,7 +58,7 @@ export const addEventEnd = options => addEventAction( EventAction.END, options )
|
|
|
58
58
|
* @param {object} args
|
|
59
59
|
* @param {string} args.id - The id of the event to conclude.
|
|
60
60
|
* @param {any} args.details - Arbitrary data to add to the event; it is used as the "error" field.
|
|
61
|
-
* @param {object} args.
|
|
61
|
+
* @param {object} args.traceInfo - The trace information object, propagated from the root-most workflow.
|
|
62
62
|
* @returns {void}
|
|
63
63
|
*/
|
|
64
64
|
export const addEventError = options => addEventAction( EventAction.ERROR, options );
|
|
@@ -71,7 +71,7 @@ export const addEventError = options => addEventAction( EventAction.ERROR, optio
|
|
|
71
71
|
* @param {object} args
|
|
72
72
|
* @param {string} args.id - The id of the event to attach the attribute to.
|
|
73
73
|
* @param {object} args.details - The attribute to add to this event, must be in `{ name: string, value: any }` format.
|
|
74
|
-
* @param {object} args.
|
|
74
|
+
* @param {object} args.traceInfo - The trace information object, propagated from the root-most workflow.
|
|
75
75
|
* @returns {void}
|
|
76
76
|
*/
|
|
77
77
|
export const addEventAttribute = options => addEventAction( EventAction.ADD_ATTR, options );
|
|
@@ -19,10 +19,10 @@ const tempTraceFilesDir = join( __dirname, 'temp', 'traces' );
|
|
|
19
19
|
/**
|
|
20
20
|
* Builds the temp file path to accumulate trace entries
|
|
21
21
|
*
|
|
22
|
-
* @param {object}
|
|
22
|
+
* @param {object} traceInfo - Trace information object
|
|
23
23
|
* @returns {string}
|
|
24
24
|
*/
|
|
25
|
-
const createTempFilePath = ( {
|
|
25
|
+
const createTempFilePath = ( { startTime, runId } ) => join( tempTraceFilesDir, `${startTime}_${runId}.trace` );
|
|
26
26
|
|
|
27
27
|
/**
|
|
28
28
|
* Adds a trace entry to the accumulation file.
|
|
@@ -49,18 +49,18 @@ const cleanupOldTempFiles = ( threshold = Date.now() - tempFilesTTL ) =>
|
|
|
49
49
|
|
|
50
50
|
/**
|
|
51
51
|
* Resolves the deep folder structure that stores a workflow trace.
|
|
52
|
-
* @param {string}
|
|
52
|
+
* @param {string} workflowType
|
|
53
53
|
* @returns {string}
|
|
54
54
|
*/
|
|
55
|
-
const resolveTraceFolder =
|
|
55
|
+
const resolveTraceFolder = workflowType => join( 'runs', workflowType );
|
|
56
56
|
|
|
57
57
|
/**
|
|
58
58
|
* Resolves the local file system path for ALL file I/O operations (read/write)
|
|
59
59
|
* Uses the project root path
|
|
60
|
-
* @param {string}
|
|
60
|
+
* @param {string} workflowType
|
|
61
61
|
* @returns {string} The local filesystem path for file operations
|
|
62
62
|
*/
|
|
63
|
-
const resolveIOPath =
|
|
63
|
+
const resolveIOPath = workflowType => join( callerDir, 'logs', resolveTraceFolder( workflowType ) );
|
|
64
64
|
|
|
65
65
|
/**
|
|
66
66
|
* Resolves the file path to be reported as the trace destination.
|
|
@@ -71,19 +71,17 @@ const resolveIOPath = workflowName => join( callerDir, 'logs', resolveTraceFolde
|
|
|
71
71
|
*
|
|
72
72
|
* If the env variable is not present, it falls back to the same value used to write files locally.
|
|
73
73
|
*
|
|
74
|
-
* @param {string}
|
|
74
|
+
* @param {string} workflowType - The name of the workflow
|
|
75
75
|
* @returns {string} The path to report, reflecting the actual filesystem
|
|
76
76
|
*/
|
|
77
|
-
const resolveReportPath =
|
|
78
|
-
join( process.env.OUTPUT_TRACE_HOST_PATH, resolveTraceFolder(
|
|
79
|
-
resolveIOPath(
|
|
77
|
+
const resolveReportPath = workflowType => process.env.OUTPUT_TRACE_HOST_PATH ?
|
|
78
|
+
join( process.env.OUTPUT_TRACE_HOST_PATH, resolveTraceFolder( workflowType ) ) :
|
|
79
|
+
resolveIOPath( workflowType );
|
|
80
80
|
|
|
81
81
|
/**
|
|
82
82
|
* Builds the actual trace filename
|
|
83
83
|
*
|
|
84
|
-
* @param {object}
|
|
85
|
-
* @param {number} options.startTime
|
|
86
|
-
* @param {string} options.workflowId
|
|
84
|
+
* @param {object} traceInfo - The trace information object
|
|
87
85
|
* @returns {string}
|
|
88
86
|
*/
|
|
89
87
|
const buildTraceFilename = ( { startTime, workflowId } ) => {
|
|
@@ -108,15 +106,15 @@ export const init = () => {
|
|
|
108
106
|
*
|
|
109
107
|
* @param {object} args
|
|
110
108
|
* @param {object} args.entry - The trace entry to append.
|
|
111
|
-
* @param {object} args.
|
|
109
|
+
* @param {object} args.traceInfo - Trace information object
|
|
112
110
|
* @returns {void}
|
|
113
111
|
*/
|
|
114
|
-
export const exec = async ( { entry,
|
|
115
|
-
const {
|
|
116
|
-
const tempFilePath = createTempFilePath(
|
|
112
|
+
export const exec = async ( { entry, traceInfo } ) => {
|
|
113
|
+
const { runId, workflowType } = traceInfo;
|
|
114
|
+
const tempFilePath = createTempFilePath( traceInfo );
|
|
117
115
|
addEntry( entry, tempFilePath );
|
|
118
116
|
|
|
119
|
-
const isRootWorkflowEnd = entry.id ===
|
|
117
|
+
const isRootWorkflowEnd = entry.id === runId && entry.action !== 'start';
|
|
120
118
|
const isError = entry.action === 'error';
|
|
121
119
|
|
|
122
120
|
if ( !isRootWorkflowEnd && !isError ) {
|
|
@@ -124,8 +122,8 @@ export const exec = async ( { entry, executionContext } ) => {
|
|
|
124
122
|
}
|
|
125
123
|
|
|
126
124
|
const content = buildTraceTree( getEntries( tempFilePath ) );
|
|
127
|
-
const dir = resolveIOPath(
|
|
128
|
-
const path = join( dir, buildTraceFilename(
|
|
125
|
+
const dir = resolveIOPath( workflowType );
|
|
126
|
+
const path = join( dir, buildTraceFilename( traceInfo ) );
|
|
129
127
|
|
|
130
128
|
mkdirSync( dir, { recursive: true } );
|
|
131
129
|
|
|
@@ -140,11 +138,8 @@ export const exec = async ( { entry, executionContext } ) => {
|
|
|
140
138
|
*
|
|
141
139
|
* This uses the optional OUTPUT_TRACE_HOST_PATH to return values relative to the host OS, not the container, if applicable.
|
|
142
140
|
*
|
|
143
|
-
* @param {object}
|
|
144
|
-
* @param {string} executionContext.startTime - The start time of the workflow
|
|
145
|
-
* @param {string} executionContext.workflowId - The id of the workflow execution
|
|
146
|
-
* @param {string} executionContext.workflowName - The name of the workflow
|
|
141
|
+
* @param {object} info
|
|
147
142
|
* @returns {string} The absolute path where the trace will be saved
|
|
148
143
|
*/
|
|
149
|
-
export const getDestination =
|
|
150
|
-
join( resolveReportPath(
|
|
144
|
+
export const getDestination = traceInfo =>
|
|
145
|
+
join( resolveReportPath( traceInfo.workflowType ), buildTraceFilename( traceInfo ) );
|
|
@@ -52,12 +52,20 @@ vi.mock( 'json-stream-stringify', async () => {
|
|
|
52
52
|
const buildTraceTreeMock = vi.fn( entries => ( { count: entries.length } ) );
|
|
53
53
|
vi.mock( '../../tools/build_trace_tree.js', () => ( { default: buildTraceTreeMock } ) );
|
|
54
54
|
|
|
55
|
-
/** Flush happens when the root id matches
|
|
56
|
-
const rootStart = (
|
|
57
|
-
const rootEnd = (
|
|
55
|
+
/** Flush happens when the root id matches runId and action is not 'start', or when action is 'error'. */
|
|
56
|
+
const rootStart = ( runId, ts ) => ( { id: runId, action: 'start', timestamp: ts } );
|
|
57
|
+
const rootEnd = ( runId, ts ) => ( { id: runId, action: 'end', timestamp: ts } );
|
|
58
58
|
const childTick = ( id, ts ) => ( { id, action: 'tick', timestamp: ts } );
|
|
59
59
|
|
|
60
60
|
describe( 'tracing/processors/local', () => {
|
|
61
|
+
const startTime = Date.parse( '2020-01-02T03:04:05.678Z' );
|
|
62
|
+
const traceInfo = {
|
|
63
|
+
workflowId: 'id1',
|
|
64
|
+
runId: 'run-1',
|
|
65
|
+
workflowType: 'WF',
|
|
66
|
+
startTime
|
|
67
|
+
};
|
|
68
|
+
|
|
61
69
|
beforeEach( () => {
|
|
62
70
|
vi.clearAllMocks();
|
|
63
71
|
store.files.clear();
|
|
@@ -81,13 +89,9 @@ describe( 'tracing/processors/local', () => {
|
|
|
81
89
|
const { exec, init } = await import( './index.js' );
|
|
82
90
|
init();
|
|
83
91
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
89
|
-
await exec( { ...ctx, entry: childTick( 'child-1', startTime + 1 ) } );
|
|
90
|
-
await exec( { ...ctx, entry: rootEnd( workflowId, startTime + 2 ) } );
|
|
92
|
+
await exec( { traceInfo, entry: rootStart( traceInfo.runId, startTime ) } );
|
|
93
|
+
await exec( { traceInfo, entry: childTick( 'child-1', startTime + 1 ) } );
|
|
94
|
+
await exec( { traceInfo, entry: rootEnd( traceInfo.runId, startTime + 2 ) } );
|
|
91
95
|
|
|
92
96
|
expect( buildTraceTreeMock ).toHaveBeenCalledTimes( 1 );
|
|
93
97
|
expect( buildTraceTreeMock.mock.calls[0][0] ).toHaveLength( 3 );
|
|
@@ -103,12 +107,8 @@ describe( 'tracing/processors/local', () => {
|
|
|
103
107
|
const { exec, init } = await import( './index.js' );
|
|
104
108
|
init();
|
|
105
109
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
|
|
109
|
-
|
|
110
|
-
await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
111
|
-
await exec( { ...ctx, entry: childTick( 'child-1', startTime + 1 ) } );
|
|
110
|
+
await exec( { traceInfo, entry: rootStart( traceInfo.runId, startTime ) } );
|
|
111
|
+
await exec( { traceInfo, entry: childTick( 'child-1', startTime + 1 ) } );
|
|
112
112
|
|
|
113
113
|
expect( buildTraceTreeMock ).not.toHaveBeenCalled();
|
|
114
114
|
expect( writeFileSyncMock ).not.toHaveBeenCalled();
|
|
@@ -120,12 +120,8 @@ describe( 'tracing/processors/local', () => {
|
|
|
120
120
|
const { exec, init } = await import( './index.js' );
|
|
121
121
|
init();
|
|
122
122
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
|
|
126
|
-
|
|
127
|
-
await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
128
|
-
await exec( { ...ctx, entry: { id: 'step-1', action: 'error', timestamp: startTime + 1 } } );
|
|
123
|
+
await exec( { traceInfo, entry: rootStart( traceInfo.runId, startTime ) } );
|
|
124
|
+
await exec( { traceInfo, entry: { id: 'step-1', action: 'error', timestamp: startTime + 1 } } );
|
|
129
125
|
|
|
130
126
|
expect( buildTraceTreeMock ).toHaveBeenCalledTimes( 1 );
|
|
131
127
|
expect( buildTraceTreeMock.mock.calls[0][0] ).toHaveLength( 2 );
|
|
@@ -136,11 +132,11 @@ describe( 'tracing/processors/local', () => {
|
|
|
136
132
|
it( 'getDestination(): returns absolute path under callerDir logs', async () => {
|
|
137
133
|
const { getDestination } = await import( './index.js' );
|
|
138
134
|
|
|
139
|
-
const
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
135
|
+
const destination = getDestination( {
|
|
136
|
+
...traceInfo,
|
|
137
|
+
workflowId: 'workflow-id-123',
|
|
138
|
+
workflowType: 'test-workflow'
|
|
139
|
+
} );
|
|
144
140
|
|
|
145
141
|
expect( destination ).toMatch( /^\/|^[A-Z]:\\/i );
|
|
146
142
|
expect( destination ).toBe(
|
|
@@ -155,12 +151,8 @@ describe( 'tracing/processors/local', () => {
|
|
|
155
151
|
|
|
156
152
|
init();
|
|
157
153
|
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
|
|
161
|
-
|
|
162
|
-
await exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
163
|
-
await exec( { ...ctx, entry: rootEnd( workflowId, startTime + 1 ) } );
|
|
154
|
+
await exec( { traceInfo, entry: rootStart( traceInfo.runId, startTime ) } );
|
|
155
|
+
await exec( { traceInfo, entry: rootEnd( traceInfo.runId, startTime + 1 ) } );
|
|
164
156
|
|
|
165
157
|
expect( createWriteStreamMock ).toHaveBeenCalledTimes( 1 );
|
|
166
158
|
const [ writtenPath ] = createWriteStreamMock.mock.calls[0];
|
|
@@ -174,11 +166,11 @@ describe( 'tracing/processors/local', () => {
|
|
|
174
166
|
|
|
175
167
|
process.env.OUTPUT_TRACE_HOST_PATH = '/host/path/logs';
|
|
176
168
|
|
|
177
|
-
const
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
169
|
+
const destination = getDestination( {
|
|
170
|
+
...traceInfo,
|
|
171
|
+
workflowId: 'workflow-id-123',
|
|
172
|
+
workflowType: 'test-workflow'
|
|
173
|
+
} );
|
|
182
174
|
|
|
183
175
|
expect( destination ).toBe( '/host/path/logs/runs/test-workflow/2020-01-02-03-04-05-678Z_workflow-id-123.json' );
|
|
184
176
|
} );
|
|
@@ -190,15 +182,17 @@ describe( 'tracing/processors/local', () => {
|
|
|
190
182
|
|
|
191
183
|
init();
|
|
192
184
|
|
|
193
|
-
const
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
185
|
+
const testTraceInfo = {
|
|
186
|
+
...traceInfo,
|
|
187
|
+
workflowId: 'workflow-id-123',
|
|
188
|
+
runId: 'run-123',
|
|
189
|
+
workflowType: 'test-workflow'
|
|
190
|
+
};
|
|
197
191
|
|
|
198
|
-
await exec( {
|
|
199
|
-
await exec( {
|
|
192
|
+
await exec( { traceInfo: testTraceInfo, entry: rootStart( testTraceInfo.runId, startTime ) } );
|
|
193
|
+
await exec( { traceInfo: testTraceInfo, entry: rootEnd( testTraceInfo.runId, startTime + 1 ) } );
|
|
200
194
|
|
|
201
|
-
const destination = getDestination(
|
|
195
|
+
const destination = getDestination( testTraceInfo );
|
|
202
196
|
|
|
203
197
|
const [ writtenPath ] = createWriteStreamMock.mock.calls[0];
|
|
204
198
|
expect( writtenPath ).not.toContain( '/Users/ben/project' );
|
|
@@ -7,7 +7,7 @@ import { JsonStreamStringify } from 'json-stream-stringify';
|
|
|
7
7
|
|
|
8
8
|
const log = createChildLogger( 'S3 Processor' );
|
|
9
9
|
|
|
10
|
-
const createRedisKey =
|
|
10
|
+
const createRedisKey = runId => `traces/${runId}`;
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
13
|
* Add new entry to list of entries
|
|
@@ -44,17 +44,14 @@ const bustEntries = async key => {
|
|
|
44
44
|
|
|
45
45
|
/**
|
|
46
46
|
* Return the S3 key for the trace file
|
|
47
|
-
* @param {object}
|
|
48
|
-
* @param {number} args.startTime
|
|
49
|
-
* @param {string} args.workflowId
|
|
50
|
-
* @param {string} args.workflowName
|
|
47
|
+
* @param {object} traceInfo
|
|
51
48
|
* @returns
|
|
52
49
|
*/
|
|
53
|
-
const getS3Key = ( { startTime, workflowId,
|
|
50
|
+
const getS3Key = ( { startTime, workflowId, workflowType } ) => {
|
|
54
51
|
const isoDate = new Date( startTime ).toISOString();
|
|
55
52
|
const [ year, month, day ] = isoDate.split( /\D/, 3 );
|
|
56
53
|
const timeStamp = isoDate.replace( /[:T.]/g, '-' );
|
|
57
|
-
return `${
|
|
54
|
+
return `${workflowType}/${year}/${month}/${day}/${timeStamp}_${workflowId}.json`;
|
|
58
55
|
};
|
|
59
56
|
|
|
60
57
|
/**
|
|
@@ -70,19 +67,19 @@ export const init = async () => {
|
|
|
70
67
|
*
|
|
71
68
|
* Appends each trace entry to Redis.
|
|
72
69
|
*
|
|
73
|
-
* When the root workflow
|
|
70
|
+
* When the root workflow finishes or errors, builds the trace tree and uploads it to S3.
|
|
74
71
|
*
|
|
75
72
|
* @param {object} args
|
|
76
73
|
* @param {object} args.entry - The trace entry to append
|
|
77
|
-
* @param {object} args.
|
|
74
|
+
* @param {object} args.traceInfo - Trace information object
|
|
78
75
|
*/
|
|
79
|
-
export const exec = async ( { entry,
|
|
80
|
-
const {
|
|
81
|
-
const cacheKey = createRedisKey(
|
|
76
|
+
export const exec = async ( { entry, traceInfo } ) => {
|
|
77
|
+
const { workflowId, runId } = traceInfo;
|
|
78
|
+
const cacheKey = createRedisKey( runId );
|
|
82
79
|
|
|
83
80
|
await addEntry( entry, cacheKey );
|
|
84
81
|
|
|
85
|
-
const isRootWorkflowEnd = entry.id ===
|
|
82
|
+
const isRootWorkflowEnd = entry.id === runId && entry.action !== 'start';
|
|
86
83
|
if ( !isRootWorkflowEnd ) {
|
|
87
84
|
return;
|
|
88
85
|
}
|
|
@@ -100,20 +97,13 @@ export const exec = async ( { entry, executionContext } ) => {
|
|
|
100
97
|
return;
|
|
101
98
|
}
|
|
102
99
|
|
|
103
|
-
await upload( {
|
|
104
|
-
key: getS3Key( { workflowId, workflowName, startTime } ),
|
|
105
|
-
content: new JsonStreamStringify( content )
|
|
106
|
-
} );
|
|
100
|
+
await upload( { key: getS3Key( traceInfo ), content: new JsonStreamStringify( content ) } );
|
|
107
101
|
await bustEntries( cacheKey );
|
|
108
102
|
};
|
|
109
103
|
|
|
110
104
|
/**
|
|
111
105
|
* Returns where the trace is saved
|
|
112
|
-
* @param {object}
|
|
113
|
-
* @param {string} executionContext.startTime - The start time of the workflow
|
|
114
|
-
* @param {string} executionContext.workflowId - The id of the workflow execution
|
|
115
|
-
* @param {string} executionContext.workflowName - The name of the workflow
|
|
106
|
+
* @param {object} traceInfo - Trace information object
|
|
116
107
|
* @returns {string} The S3 url of the trace file
|
|
117
108
|
*/
|
|
118
|
-
export const getDestination =
|
|
119
|
-
`https://${getVars().remoteS3Bucket}.s3.amazonaws.com/${getS3Key( { workflowId, workflowName, startTime } )}`;
|
|
109
|
+
export const getDestination = traceInfo => `https://${getVars().remoteS3Bucket}.s3.amazonaws.com/${getS3Key( traceInfo )}`;
|
|
@@ -53,6 +53,14 @@ const streamToString = async stream => {
|
|
|
53
53
|
};
|
|
54
54
|
|
|
55
55
|
describe( 'tracing/processors/s3', () => {
|
|
56
|
+
const startTime = Date.parse( '2020-01-02T03:04:05.678Z' );
|
|
57
|
+
const traceInfo = {
|
|
58
|
+
workflowId: 'id1',
|
|
59
|
+
runId: 'run-1',
|
|
60
|
+
workflowType: 'WF',
|
|
61
|
+
startTime
|
|
62
|
+
};
|
|
63
|
+
|
|
56
64
|
beforeEach( () => {
|
|
57
65
|
vi.useFakeTimers();
|
|
58
66
|
vi.clearAllMocks();
|
|
@@ -72,24 +80,30 @@ describe( 'tracing/processors/s3', () => {
|
|
|
72
80
|
|
|
73
81
|
it( 'exec(): accumulates via redis, uploads only on root workflow end', async () => {
|
|
74
82
|
const { exec } = await import( './index.js' );
|
|
75
|
-
const startTime = Date.parse( '2020-01-02T03:04:05.678Z' );
|
|
76
|
-
const ctx = { executionContext: { workflowId: 'id1', workflowName: 'WF', startTime } };
|
|
77
83
|
|
|
78
84
|
redisMulti.exec.mockResolvedValue( [] );
|
|
79
85
|
|
|
80
|
-
const workflowStart = { id: '
|
|
81
|
-
const activityStart = {
|
|
82
|
-
|
|
86
|
+
const workflowStart = { id: 'run-1', name: 'WF', kind: 'workflow', action: 'start', details: {}, timestamp: startTime };
|
|
87
|
+
const activityStart = {
|
|
88
|
+
id: 'act-1',
|
|
89
|
+
name: 'DoSomething',
|
|
90
|
+
kind: 'step',
|
|
91
|
+
parentId: 'run-1',
|
|
92
|
+
action: 'start',
|
|
93
|
+
details: {},
|
|
94
|
+
timestamp: startTime + 1
|
|
95
|
+
};
|
|
96
|
+
const workflowEnd = { id: 'run-1', action: 'end', details: { ok: true }, timestamp: startTime + 2 };
|
|
83
97
|
zRangeMock.mockResolvedValue( [
|
|
84
98
|
JSON.stringify( workflowStart ),
|
|
85
99
|
JSON.stringify( activityStart ),
|
|
86
100
|
JSON.stringify( workflowEnd )
|
|
87
101
|
] );
|
|
88
102
|
|
|
89
|
-
await exec( {
|
|
90
|
-
await exec( {
|
|
91
|
-
// Root end: id matches
|
|
92
|
-
const endPromise = exec( {
|
|
103
|
+
await exec( { traceInfo, entry: workflowStart } );
|
|
104
|
+
await exec( { traceInfo, entry: activityStart } );
|
|
105
|
+
// Root end: id matches runId and not start — triggers the 10s delay before upload
|
|
106
|
+
const endPromise = exec( { traceInfo, entry: workflowEnd } );
|
|
93
107
|
await vi.advanceTimersByTimeAsync( 10_000 );
|
|
94
108
|
await endPromise;
|
|
95
109
|
|
|
@@ -101,14 +115,13 @@ describe( 'tracing/processors/s3', () => {
|
|
|
101
115
|
expect( key ).toMatch( /^WF\/2020\/01\/02\// );
|
|
102
116
|
expect( JSON.parse( await streamToString( content ) ).count ).toBe( 3 );
|
|
103
117
|
expect( delMock ).toHaveBeenCalledTimes( 1 );
|
|
104
|
-
expect( delMock ).toHaveBeenCalledWith( 'traces/
|
|
118
|
+
expect( delMock ).toHaveBeenCalledWith( 'traces/run-1' );
|
|
105
119
|
} );
|
|
106
120
|
|
|
107
121
|
it( 'getDestination(): returns S3 URL using bucket and key from getVars', async () => {
|
|
108
122
|
getVarsMock.mockReturnValue( { remoteS3Bucket: 'my-bucket', redisIncompleteWorkflowsTTL: 3600, traceUploadDelayMs: 10_000 } );
|
|
109
123
|
const { getDestination } = await import( './index.js' );
|
|
110
|
-
const
|
|
111
|
-
const url = getDestination( { workflowId: 'id1', workflowName: 'WF', startTime } );
|
|
124
|
+
const url = getDestination( traceInfo );
|
|
112
125
|
expect( getVarsMock ).toHaveBeenCalled();
|
|
113
126
|
expect( url ).toBe(
|
|
114
127
|
'https://my-bucket.s3.amazonaws.com/WF/2020/01/02/2020-01-02-03-04-05-678Z_id1.json'
|
|
@@ -117,36 +130,32 @@ describe( 'tracing/processors/s3', () => {
|
|
|
117
130
|
|
|
118
131
|
it( 'exec(): sets expiry on the redis key for each entry', async () => {
|
|
119
132
|
const { exec } = await import( './index.js' );
|
|
120
|
-
const startTime = Date.parse( '2020-01-02T03:04:05.678Z' );
|
|
121
|
-
const ctx = { executionContext: { workflowId: 'id1', workflowName: 'WF', startTime } };
|
|
122
133
|
|
|
123
134
|
redisMulti.exec.mockResolvedValue( [] );
|
|
124
135
|
const workflowStart = {
|
|
125
|
-
kind: 'workflow', id: '
|
|
136
|
+
kind: 'workflow', id: 'run-1', name: 'WF', parentId: undefined, action: 'start', details: {}, timestamp: startTime
|
|
126
137
|
};
|
|
127
138
|
zRangeMock.mockResolvedValue( [ JSON.stringify( workflowStart ) ] );
|
|
128
139
|
|
|
129
|
-
await exec( {
|
|
140
|
+
await exec( { traceInfo, entry: workflowStart } );
|
|
130
141
|
|
|
131
142
|
expect( redisMulti.expire ).toHaveBeenCalledTimes( 1 );
|
|
132
|
-
expect( redisMulti.expire ).toHaveBeenCalledWith( 'traces/
|
|
143
|
+
expect( redisMulti.expire ).toHaveBeenCalledWith( 'traces/run-1', 3600 );
|
|
133
144
|
} );
|
|
134
145
|
|
|
135
146
|
it( 'exec(): does not treat a non-root end (e.g. step without parentId) as root workflow end — regression for wrong root detection', async () => {
|
|
136
147
|
const { exec } = await import( './index.js' );
|
|
137
|
-
const startTime = Date.parse( '2020-01-02T03:04:05.678Z' );
|
|
138
|
-
const ctx = { executionContext: { workflowId: 'id1', workflowName: 'WF', startTime } };
|
|
139
148
|
|
|
140
149
|
redisMulti.exec.mockResolvedValue( [] );
|
|
141
|
-
const workflowStart = { id: '
|
|
150
|
+
const workflowStart = { id: 'run-1', name: 'WF', kind: 'workflow', action: 'start', details: {}, timestamp: startTime };
|
|
142
151
|
const stepEndNoParent = { id: 'step-1', action: 'end', details: { done: true }, timestamp: startTime + 1 };
|
|
143
152
|
zRangeMock.mockResolvedValue( [
|
|
144
153
|
JSON.stringify( workflowStart ),
|
|
145
154
|
JSON.stringify( stepEndNoParent )
|
|
146
155
|
] );
|
|
147
156
|
|
|
148
|
-
await exec( {
|
|
149
|
-
await exec( {
|
|
157
|
+
await exec( { traceInfo, entry: workflowStart } );
|
|
158
|
+
await exec( { traceInfo, entry: stepEndNoParent } );
|
|
150
159
|
|
|
151
160
|
expect( redisMulti.zAdd ).toHaveBeenCalledTimes( 2 );
|
|
152
161
|
expect( buildTraceTreeMock ).not.toHaveBeenCalled();
|
|
@@ -156,17 +165,15 @@ describe( 'tracing/processors/s3', () => {
|
|
|
156
165
|
|
|
157
166
|
it( 'exec(): when buildTraceTree returns null (incomplete tree), does not upload or bust cache', async () => {
|
|
158
167
|
const { exec } = await import( './index.js' );
|
|
159
|
-
const startTime = Date.parse( '2020-01-02T03:04:05.678Z' );
|
|
160
|
-
const ctx = { executionContext: { workflowId: 'id1', workflowName: 'WF', startTime } };
|
|
161
168
|
|
|
162
169
|
redisMulti.exec.mockResolvedValue( [] );
|
|
163
170
|
const workflowEnd = {
|
|
164
|
-
kind: 'workflow', id: '
|
|
171
|
+
kind: 'workflow', id: 'run-1', name: 'WF', parentId: undefined, action: 'end', details: {}, timestamp: startTime
|
|
165
172
|
};
|
|
166
173
|
zRangeMock.mockResolvedValue( [ JSON.stringify( workflowEnd ) ] );
|
|
167
174
|
buildTraceTreeMock.mockReturnValueOnce( null );
|
|
168
175
|
|
|
169
|
-
const endPromise = exec( {
|
|
176
|
+
const endPromise = exec( { traceInfo, entry: workflowEnd } );
|
|
170
177
|
await vi.advanceTimersByTimeAsync( 10_000 );
|
|
171
178
|
await endPromise;
|
|
172
179
|
|