@outputai/core 0.2.1-next.af8a069.0 → 0.2.1-next.eadab44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/tracing/processors/local/index.js +40 -8
- package/src/tracing/processors/local/index.spec.js +66 -36
- package/src/tracing/processors/s3/index.js +2 -1
- package/src/tracing/tools/utils.js +28 -0
- package/src/tracing/tools/utils.spec.js +134 -2
- package/src/worker/configs.js +9 -1
- package/src/worker/index.js +8 -2
- package/src/worker/index.spec.js +7 -1
- package/src/worker/proxy.js +32 -0
- package/src/worker/proxy.spec.js +71 -0
package/package.json
CHANGED
|
@@ -2,6 +2,7 @@ import { appendFileSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFile
|
|
|
2
2
|
import { dirname, join } from 'node:path';
|
|
3
3
|
import { fileURLToPath } from 'url';
|
|
4
4
|
import buildTraceTree from '../../tools/build_trace_tree.js';
|
|
5
|
+
import { safeFormatJSON } from '../../tools/utils.js';
|
|
5
6
|
import { EOL } from 'node:os';
|
|
6
7
|
|
|
7
8
|
const __dirname = dirname( fileURLToPath( import.meta.url ) );
|
|
@@ -13,12 +14,32 @@ const callerDir = process.argv[2];
|
|
|
13
14
|
|
|
14
15
|
const tempTraceFilesDir = join( __dirname, 'temp', 'traces' );
|
|
15
16
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
}
|
|
17
|
+
/**
|
|
18
|
+
* Builds the temp file path to accumulate trace entries
|
|
19
|
+
*
|
|
20
|
+
* @param {object} executionContext - The execution context around a given trace entry
|
|
21
|
+
* @returns {string}
|
|
22
|
+
*/
|
|
23
|
+
const createTempFilePath = ( { workflowId, startTime } ) => join( tempTraceFilesDir, `${startTime}_${workflowId}.trace` );
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Adds an trace entry to the accumulation file
|
|
27
|
+
* @param {object} entry - The trace entry
|
|
28
|
+
* @param {string} path - Accumulation file path
|
|
29
|
+
*/
|
|
30
|
+
const addEntry = ( entry, path ) => appendFileSync( path, JSON.stringify( entry ) + EOL, 'utf-8' );
|
|
21
31
|
|
|
32
|
+
/**
|
|
33
|
+
* Reads the accumulation file and returns all the entries, each serialized to JSON
|
|
34
|
+
* @param {string} path - Accumulation file path
|
|
35
|
+
* @returns {object[]} Trace entries
|
|
36
|
+
*/
|
|
37
|
+
const getEntries = path => readFileSync( path, 'utf-8' ).split( EOL ).slice( 0, -1 ).map( v => JSON.parse( v ) );
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Deletes old accumulation files
|
|
41
|
+
* @param {number} [threshold] Timestamp in ms epoch. All files below this date are considered old
|
|
42
|
+
*/
|
|
22
43
|
const cleanupOldTempFiles = ( threshold = Date.now() - tempFilesTTL ) =>
|
|
23
44
|
readdirSync( tempTraceFilesDir )
|
|
24
45
|
.filter( f => +f.split( '_' )[0] < threshold )
|
|
@@ -79,7 +100,8 @@ export const init = () => {
|
|
|
79
100
|
/**
|
|
80
101
|
* Execute this processor:
|
|
81
102
|
*
|
|
82
|
-
*
|
|
103
|
+
* Append each trace entry to a temp file; when the root workflow ends (non-start phase on the
|
|
104
|
+
* workflow id) or any entry is an error phase, build the trace tree and write the JSON file once.
|
|
83
105
|
*
|
|
84
106
|
* @param {object} args
|
|
85
107
|
* @param {object} entry - Trace event phase
|
|
@@ -88,12 +110,22 @@ export const init = () => {
|
|
|
88
110
|
*/
|
|
89
111
|
export const exec = ( { entry, executionContext } ) => {
|
|
90
112
|
const { workflowId, workflowName, startTime } = executionContext;
|
|
91
|
-
const
|
|
113
|
+
const tempFilePath = createTempFilePath( executionContext );
|
|
114
|
+
addEntry( entry, tempFilePath );
|
|
115
|
+
|
|
116
|
+
const isRootWorkflowEnd = entry.id === workflowId && entry.phase !== 'start';
|
|
117
|
+
const isError = entry.phase === 'error';
|
|
118
|
+
|
|
119
|
+
if ( !isRootWorkflowEnd && !isError ) {
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const content = buildTraceTree( getEntries( tempFilePath ) );
|
|
92
124
|
const dir = resolveIOPath( workflowName );
|
|
93
125
|
const path = join( dir, buildTraceFilename( { startTime, workflowId } ) );
|
|
94
126
|
|
|
95
127
|
mkdirSync( dir, { recursive: true } );
|
|
96
|
-
writeFileSync( path,
|
|
128
|
+
writeFileSync( path, safeFormatJSON( content ) + EOL, 'utf-8' );
|
|
97
129
|
};
|
|
98
130
|
|
|
99
131
|
/**
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
2
|
+
import { EOL } from 'node:os';
|
|
2
3
|
|
|
3
4
|
// In-memory fs mock store
|
|
4
5
|
const store = { files: new Map() };
|
|
@@ -24,12 +25,17 @@ vi.mock( 'node:fs', () => ( {
|
|
|
24
25
|
const buildTraceTreeMock = vi.fn( entries => ( { count: entries.length } ) );
|
|
25
26
|
vi.mock( '../../tools/build_trace_tree.js', () => ( { default: buildTraceTreeMock } ) );
|
|
26
27
|
|
|
28
|
+
/** Flush happens when root id matches workflowId and phase is not start, or when phase is error. */
|
|
29
|
+
const rootStart = ( workflowId, ts ) => ( { id: workflowId, phase: 'start', timestamp: ts } );
|
|
30
|
+
const rootEnd = ( workflowId, ts ) => ( { id: workflowId, phase: 'end', timestamp: ts } );
|
|
31
|
+
const childTick = ( id, ts ) => ( { id, phase: 'tick', timestamp: ts } );
|
|
32
|
+
|
|
27
33
|
describe( 'tracing/processors/local', () => {
|
|
28
34
|
beforeEach( () => {
|
|
29
35
|
vi.clearAllMocks();
|
|
30
36
|
store.files.clear();
|
|
31
37
|
process.argv[2] = '/tmp/project';
|
|
32
|
-
delete process.env.OUTPUT_TRACE_HOST_PATH;
|
|
38
|
+
delete process.env.OUTPUT_TRACE_HOST_PATH;
|
|
33
39
|
} );
|
|
34
40
|
|
|
35
41
|
it( 'init(): creates temp dir and cleans up old files', async () => {
|
|
@@ -40,34 +46,63 @@ describe( 'tracing/processors/local', () => {
|
|
|
40
46
|
|
|
41
47
|
init();
|
|
42
48
|
|
|
43
|
-
// Should create temp dir relative to module location using __dirname
|
|
44
49
|
expect( mkdirSyncMock ).toHaveBeenCalledWith( expect.stringMatching( /temp\/traces$/ ), { recursive: true } );
|
|
45
50
|
expect( rmSyncMock ).toHaveBeenCalledTimes( 1 );
|
|
46
51
|
} );
|
|
47
52
|
|
|
48
|
-
it( 'exec():
|
|
53
|
+
it( 'exec(): appends each entry and writes aggregated tree once on root workflow end', async () => {
|
|
49
54
|
const { exec, init } = await import( './index.js' );
|
|
50
55
|
init();
|
|
51
56
|
|
|
52
57
|
const startTime = Date.parse( '2020-01-02T03:04:05.678Z' );
|
|
53
|
-
const
|
|
58
|
+
const workflowId = 'id1';
|
|
59
|
+
const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
|
|
54
60
|
|
|
55
|
-
exec( { ...ctx, entry:
|
|
56
|
-
exec( { ...ctx, entry:
|
|
57
|
-
exec( { ...ctx, entry:
|
|
61
|
+
exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
62
|
+
exec( { ...ctx, entry: childTick( 'child-1', startTime + 1 ) } );
|
|
63
|
+
exec( { ...ctx, entry: rootEnd( workflowId, startTime + 2 ) } );
|
|
58
64
|
|
|
59
|
-
|
|
60
|
-
expect( buildTraceTreeMock ).
|
|
61
|
-
expect( buildTraceTreeMock.mock.calls.at( -1 )[0].length ).toBe( 3 );
|
|
65
|
+
expect( buildTraceTreeMock ).toHaveBeenCalledTimes( 1 );
|
|
66
|
+
expect( buildTraceTreeMock.mock.calls[0][0] ).toHaveLength( 3 );
|
|
62
67
|
|
|
63
|
-
expect( writeFileSyncMock ).toHaveBeenCalledTimes(
|
|
64
|
-
const [ writtenPath, content ] = writeFileSyncMock.mock.calls
|
|
65
|
-
|
|
66
|
-
expect( writtenPath ).toMatch( /\/runs\/WF\// );
|
|
68
|
+
expect( writeFileSyncMock ).toHaveBeenCalledTimes( 1 );
|
|
69
|
+
const [ writtenPath, content ] = writeFileSyncMock.mock.calls[0];
|
|
70
|
+
expect( writtenPath ).toMatch( /\/tmp\/project\/logs\/runs\/WF\// );
|
|
67
71
|
expect( JSON.parse( content.trim() ).count ).toBe( 3 );
|
|
68
72
|
} );
|
|
69
73
|
|
|
70
|
-
it( '
|
|
74
|
+
it( 'exec(): does not build or write on non-flush entries', async () => {
|
|
75
|
+
const { exec, init } = await import( './index.js' );
|
|
76
|
+
init();
|
|
77
|
+
|
|
78
|
+
const startTime = Date.parse( '2020-01-02T03:04:05.678Z' );
|
|
79
|
+
const workflowId = 'id1';
|
|
80
|
+
const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
|
|
81
|
+
|
|
82
|
+
exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
83
|
+
exec( { ...ctx, entry: childTick( 'child-1', startTime + 1 ) } );
|
|
84
|
+
|
|
85
|
+
expect( buildTraceTreeMock ).not.toHaveBeenCalled();
|
|
86
|
+
expect( writeFileSyncMock ).not.toHaveBeenCalled();
|
|
87
|
+
} );
|
|
88
|
+
|
|
89
|
+
it( 'exec(): flushes on error phase before root end', async () => {
|
|
90
|
+
const { exec, init } = await import( './index.js' );
|
|
91
|
+
init();
|
|
92
|
+
|
|
93
|
+
const startTime = Date.parse( '2020-01-02T03:04:05.678Z' );
|
|
94
|
+
const workflowId = 'id1';
|
|
95
|
+
const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
|
|
96
|
+
|
|
97
|
+
exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
98
|
+
exec( { ...ctx, entry: { id: 'step-1', phase: 'error', timestamp: startTime + 1 } } );
|
|
99
|
+
|
|
100
|
+
expect( buildTraceTreeMock ).toHaveBeenCalledTimes( 1 );
|
|
101
|
+
expect( buildTraceTreeMock.mock.calls[0][0] ).toHaveLength( 2 );
|
|
102
|
+
expect( writeFileSyncMock ).toHaveBeenCalledTimes( 1 );
|
|
103
|
+
} );
|
|
104
|
+
|
|
105
|
+
it( 'getDestination(): returns absolute path under callerDir logs', async () => {
|
|
71
106
|
const { getDestination } = await import( './index.js' );
|
|
72
107
|
|
|
73
108
|
const startTime = Date.parse( '2020-01-02T03:04:05.678Z' );
|
|
@@ -76,36 +111,36 @@ describe( 'tracing/processors/local', () => {
|
|
|
76
111
|
|
|
77
112
|
const destination = getDestination( { startTime, workflowId, workflowName } );
|
|
78
113
|
|
|
79
|
-
|
|
80
|
-
expect( destination ).
|
|
81
|
-
|
|
114
|
+
expect( destination ).toMatch( /^\/|^[A-Z]:\\/i );
|
|
115
|
+
expect( destination ).toBe(
|
|
116
|
+
'/tmp/project/logs/runs/test-workflow/2020-01-02-03-04-05-678Z_workflow-id-123.json'
|
|
117
|
+
);
|
|
82
118
|
} );
|
|
83
119
|
|
|
84
|
-
it( 'exec(): writes
|
|
120
|
+
it( 'exec(): writes under process.argv[2] logs even when OUTPUT_TRACE_HOST_PATH is set', async () => {
|
|
85
121
|
const { exec, init } = await import( './index.js' );
|
|
86
122
|
|
|
87
|
-
// Set OUTPUT_TRACE_HOST_PATH to simulate Docker environment
|
|
88
123
|
process.env.OUTPUT_TRACE_HOST_PATH = '/host/path/logs';
|
|
89
124
|
|
|
90
125
|
init();
|
|
91
126
|
|
|
92
127
|
const startTime = Date.parse( '2020-01-02T03:04:05.678Z' );
|
|
93
|
-
const
|
|
128
|
+
const workflowId = 'id1';
|
|
129
|
+
const ctx = { executionContext: { workflowId, workflowName: 'WF', startTime } };
|
|
94
130
|
|
|
95
|
-
exec( { ...ctx, entry:
|
|
131
|
+
exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
132
|
+
exec( { ...ctx, entry: rootEnd( workflowId, startTime + 1 ) } );
|
|
96
133
|
|
|
97
134
|
expect( writeFileSyncMock ).toHaveBeenCalledTimes( 1 );
|
|
98
|
-
const [ writtenPath ] = writeFileSyncMock.mock.calls
|
|
135
|
+
const [ writtenPath ] = writeFileSyncMock.mock.calls[0];
|
|
99
136
|
|
|
100
|
-
// Should write to process.cwd()/logs, NOT to OUTPUT_TRACE_HOST_PATH
|
|
101
137
|
expect( writtenPath ).not.toContain( '/host/path/logs' );
|
|
102
|
-
expect( writtenPath ).toMatch(
|
|
138
|
+
expect( writtenPath ).toMatch( /\/tmp\/project\/logs\/runs\/WF\// );
|
|
103
139
|
} );
|
|
104
140
|
|
|
105
141
|
it( 'getDestination(): returns OUTPUT_TRACE_HOST_PATH when set', async () => {
|
|
106
142
|
const { getDestination } = await import( './index.js' );
|
|
107
143
|
|
|
108
|
-
// Set OUTPUT_TRACE_HOST_PATH to simulate Docker environment
|
|
109
144
|
process.env.OUTPUT_TRACE_HOST_PATH = '/host/path/logs';
|
|
110
145
|
|
|
111
146
|
const startTime = Date.parse( '2020-01-02T03:04:05.678Z' );
|
|
@@ -114,14 +149,12 @@ describe( 'tracing/processors/local', () => {
|
|
|
114
149
|
|
|
115
150
|
const destination = getDestination( { startTime, workflowId, workflowName } );
|
|
116
151
|
|
|
117
|
-
// Should return OUTPUT_TRACE_HOST_PATH-based path for reporting
|
|
118
152
|
expect( destination ).toBe( '/host/path/logs/runs/test-workflow/2020-01-02-03-04-05-678Z_workflow-id-123.json' );
|
|
119
153
|
} );
|
|
120
154
|
|
|
121
155
|
it( 'separation of write and report paths works correctly', async () => {
|
|
122
156
|
const { exec, getDestination, init } = await import( './index.js' );
|
|
123
157
|
|
|
124
|
-
// Set OUTPUT_TRACE_HOST_PATH to simulate Docker environment
|
|
125
158
|
process.env.OUTPUT_TRACE_HOST_PATH = '/Users/ben/project/logs';
|
|
126
159
|
|
|
127
160
|
init();
|
|
@@ -131,19 +164,16 @@ describe( 'tracing/processors/local', () => {
|
|
|
131
164
|
const workflowName = 'test-workflow';
|
|
132
165
|
const ctx = { executionContext: { workflowId, workflowName, startTime } };
|
|
133
166
|
|
|
134
|
-
|
|
135
|
-
exec( { ...ctx, entry:
|
|
167
|
+
exec( { ...ctx, entry: rootStart( workflowId, startTime ) } );
|
|
168
|
+
exec( { ...ctx, entry: rootEnd( workflowId, startTime + 1 ) } );
|
|
136
169
|
|
|
137
|
-
// Get destination for reporting
|
|
138
170
|
const destination = getDestination( { startTime, workflowId, workflowName } );
|
|
139
171
|
|
|
140
|
-
|
|
141
|
-
const [ writtenPath ] = writeFileSyncMock.mock.calls.at( -1 );
|
|
172
|
+
const [ writtenPath, payload ] = writeFileSyncMock.mock.calls[0];
|
|
142
173
|
expect( writtenPath ).not.toContain( '/Users/ben/project' );
|
|
143
|
-
expect( writtenPath ).toMatch(
|
|
174
|
+
expect( writtenPath ).toMatch( /\/tmp\/project\/logs\/runs\/test-workflow\// );
|
|
175
|
+
expect( payload.endsWith( EOL ) ).toBe( true );
|
|
144
176
|
|
|
145
|
-
// Verify report path uses OUTPUT_TRACE_HOST_PATH
|
|
146
177
|
expect( destination ).toBe( '/Users/ben/project/logs/runs/test-workflow/2020-01-02-03-04-05-678Z_workflow-id-123.json' );
|
|
147
178
|
} );
|
|
148
179
|
} );
|
|
149
|
-
|
|
@@ -4,6 +4,7 @@ import buildTraceTree from '../../tools/build_trace_tree.js';
|
|
|
4
4
|
import { EOL } from 'node:os';
|
|
5
5
|
import { loadEnv, getVars } from './configs.js';
|
|
6
6
|
import { createChildLogger } from '#logger';
|
|
7
|
+
import { safeFormatJSON } from '../../tools/utils.js';
|
|
7
8
|
|
|
8
9
|
const log = createChildLogger( 'S3 Processor' );
|
|
9
10
|
|
|
@@ -97,7 +98,7 @@ export const exec = async ( { entry, executionContext } ) => {
|
|
|
97
98
|
}
|
|
98
99
|
await upload( {
|
|
99
100
|
key: getS3Key( { workflowId, workflowName, startTime } ),
|
|
100
|
-
content:
|
|
101
|
+
content: safeFormatJSON( content ) + EOL
|
|
101
102
|
} );
|
|
102
103
|
await bustEntries( cacheKey );
|
|
103
104
|
};
|
|
@@ -19,3 +19,31 @@ export const serializeError = error =>
|
|
|
19
19
|
message: error.message,
|
|
20
20
|
stack: error.stack
|
|
21
21
|
};
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Tries to stringify an object to an indented JSON string.
|
|
25
|
+
* If its byte size is bigger than threshold returns a plain JSON string without formatting.
|
|
26
|
+
*
|
|
27
|
+
* @param {object|array} content
|
|
28
|
+
* @param {*} [threshold] - The max allowed size to try to stringify with formatting (in bytes). Default is 50mb
|
|
29
|
+
* @returns {string} String representation of the object
|
|
30
|
+
*/
|
|
31
|
+
export const safeFormatJSON = ( content, threshold = 50 * 1024 * 1024 /* 50mb */ ) => {
|
|
32
|
+
const plainString = JSON.stringify( content );
|
|
33
|
+
const plainStringSize = Buffer.byteLength( plainString, 'utf8' );
|
|
34
|
+
|
|
35
|
+
if ( plainStringSize > threshold ) {
|
|
36
|
+
return plainString;
|
|
37
|
+
}
|
|
38
|
+
try {
|
|
39
|
+
return JSON.stringify( content, undefined, 2 );
|
|
40
|
+
} catch ( error ) {
|
|
41
|
+
// Only handles this specific error because other common parsing errors like:
|
|
42
|
+
// "TypeError: cyclic object value" and "RangeError: Maximum call stack size exceeded"
|
|
43
|
+
// would have been thrown on the first parsing.
|
|
44
|
+
if ( error instanceof RangeError && error.message === 'Invalid string length' ) {
|
|
45
|
+
return plainString;
|
|
46
|
+
}
|
|
47
|
+
throw error;
|
|
48
|
+
}
|
|
49
|
+
};
|
|
@@ -1,5 +1,15 @@
|
|
|
1
|
-
import { describe, it, expect } from 'vitest';
|
|
2
|
-
import { serializeError } from './utils.js';
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
import { safeFormatJSON, serializeError } from './utils.js';
|
|
3
|
+
|
|
4
|
+
const isPrettyStringifyCall = args => args.length >= 3 && args[2] === 2;
|
|
5
|
+
|
|
6
|
+
/** @param {number} targetBytes UTF-8 size of compact JSON.stringify( { a: "<xs>" } ) */
|
|
7
|
+
const objectWithCompactByteLength = targetBytes => {
|
|
8
|
+
const sample = { a: '' };
|
|
9
|
+
const overhead = Buffer.byteLength( JSON.stringify( sample ), 'utf8' );
|
|
10
|
+
const repeat = Math.max( 0, targetBytes - overhead );
|
|
11
|
+
return { a: 'x'.repeat( repeat ) };
|
|
12
|
+
};
|
|
3
13
|
|
|
4
14
|
describe( 'tracing/utils', () => {
|
|
5
15
|
it( 'serializeError unwraps causes and keeps message/stack', () => {
|
|
@@ -11,4 +21,126 @@ describe( 'tracing/utils', () => {
|
|
|
11
21
|
expect( out.message ).toBe( 'inner' );
|
|
12
22
|
expect( typeof out.stack ).toBe( 'string' );
|
|
13
23
|
} );
|
|
24
|
+
|
|
25
|
+
describe( 'safeFormatJSON', () => {
|
|
26
|
+
it( 'formats small objects with indentation when under threshold', () => {
|
|
27
|
+
const content = { a: 1, b: [ 2, 3 ] };
|
|
28
|
+
const out = safeFormatJSON( content, 10_000 );
|
|
29
|
+
|
|
30
|
+
expect( out ).toContain( '\n' );
|
|
31
|
+
expect( out ).toMatch( /^\{\n/ );
|
|
32
|
+
expect( JSON.parse( out ) ).toEqual( content );
|
|
33
|
+
} );
|
|
34
|
+
|
|
35
|
+
it( 'formats small arrays with indentation when under threshold', () => {
|
|
36
|
+
const content = [ 1, { nested: true } ];
|
|
37
|
+
const out = safeFormatJSON( content, 10_000 );
|
|
38
|
+
|
|
39
|
+
expect( out ).toContain( '\n' );
|
|
40
|
+
expect( out.trimStart() ).toMatch( /^\[/ );
|
|
41
|
+
expect( JSON.parse( out ) ).toEqual( content );
|
|
42
|
+
} );
|
|
43
|
+
|
|
44
|
+
it( 'returns compact JSON when compact UTF-8 size is strictly greater than threshold', () => {
|
|
45
|
+
const content = objectWithCompactByteLength( 40 );
|
|
46
|
+
const compact = JSON.stringify( content );
|
|
47
|
+
expect( Buffer.byteLength( compact, 'utf8' ) ).toBe( 40 );
|
|
48
|
+
|
|
49
|
+
const out = safeFormatJSON( content, 39 );
|
|
50
|
+
expect( out ).toBe( compact );
|
|
51
|
+
expect( out ).not.toContain( '\n ' );
|
|
52
|
+
expect( JSON.parse( out ) ).toEqual( content );
|
|
53
|
+
} );
|
|
54
|
+
|
|
55
|
+
it( 'uses pretty JSON when compact UTF-8 size equals threshold', () => {
|
|
56
|
+
const content = objectWithCompactByteLength( 40 );
|
|
57
|
+
const compact = JSON.stringify( content );
|
|
58
|
+
expect( Buffer.byteLength( compact, 'utf8' ) ).toBe( 40 );
|
|
59
|
+
|
|
60
|
+
const out = safeFormatJSON( content, 40 );
|
|
61
|
+
expect( out ).not.toBe( compact );
|
|
62
|
+
expect( out ).toContain( '\n' );
|
|
63
|
+
expect( JSON.parse( out ) ).toEqual( content );
|
|
64
|
+
} );
|
|
65
|
+
|
|
66
|
+
it( 'uses UTF-8 byte length for threshold, not JavaScript string length', () => {
|
|
67
|
+
const content = { label: 'éclair' };
|
|
68
|
+
const compact = JSON.stringify( content );
|
|
69
|
+
expect( compact.length ).toBeLessThan( Buffer.byteLength( compact, 'utf8' ) );
|
|
70
|
+
|
|
71
|
+
const bytes = Buffer.byteLength( compact, 'utf8' );
|
|
72
|
+
const outCompact = safeFormatJSON( content, bytes - 1 );
|
|
73
|
+
expect( outCompact ).toBe( compact );
|
|
74
|
+
|
|
75
|
+
const outPretty = safeFormatJSON( content, bytes + 100 );
|
|
76
|
+
expect( outPretty ).toContain( '\n' );
|
|
77
|
+
expect( JSON.parse( outPretty ) ).toEqual( content );
|
|
78
|
+
} );
|
|
79
|
+
|
|
80
|
+
it( 'round-trips empty object and primitives for both branches', () => {
|
|
81
|
+
const tiny = {};
|
|
82
|
+
const pretty = safeFormatJSON( tiny, 100 );
|
|
83
|
+
expect( JSON.parse( pretty ) ).toEqual( tiny );
|
|
84
|
+
|
|
85
|
+
const forcedCompact = safeFormatJSON( tiny, 0 );
|
|
86
|
+
expect( JSON.parse( forcedCompact ) ).toEqual( tiny );
|
|
87
|
+
} );
|
|
88
|
+
|
|
89
|
+
it( 'returns compact JSON when pretty stringify throws Invalid string length', () => {
|
|
90
|
+
const content = { a: 1 };
|
|
91
|
+
const compact = JSON.stringify( content );
|
|
92
|
+
const origStringify = JSON.stringify.bind( JSON );
|
|
93
|
+
|
|
94
|
+
const spy = vi.spyOn( JSON, 'stringify' ).mockImplementation( ( ...args ) => {
|
|
95
|
+
if ( isPrettyStringifyCall( args ) ) {
|
|
96
|
+
throw new RangeError( 'Invalid string length' );
|
|
97
|
+
}
|
|
98
|
+
return origStringify( ...args );
|
|
99
|
+
} );
|
|
100
|
+
|
|
101
|
+
try {
|
|
102
|
+
const out = safeFormatJSON( content, 10_000 );
|
|
103
|
+
expect( out ).toBe( compact );
|
|
104
|
+
expect( JSON.parse( out ) ).toEqual( content );
|
|
105
|
+
} finally {
|
|
106
|
+
spy.mockRestore();
|
|
107
|
+
}
|
|
108
|
+
} );
|
|
109
|
+
|
|
110
|
+
it( 'rethrows RangeError when message is not Invalid string length', () => {
|
|
111
|
+
const content = { a: 1 };
|
|
112
|
+
const origStringify = JSON.stringify.bind( JSON );
|
|
113
|
+
|
|
114
|
+
const spy = vi.spyOn( JSON, 'stringify' ).mockImplementation( ( ...args ) => {
|
|
115
|
+
if ( isPrettyStringifyCall( args ) ) {
|
|
116
|
+
throw new RangeError( 'not the string length error' );
|
|
117
|
+
}
|
|
118
|
+
return origStringify( ...args );
|
|
119
|
+
} );
|
|
120
|
+
|
|
121
|
+
try {
|
|
122
|
+
expect( () => safeFormatJSON( content, 10_000 ) ).toThrow( RangeError );
|
|
123
|
+
} finally {
|
|
124
|
+
spy.mockRestore();
|
|
125
|
+
}
|
|
126
|
+
} );
|
|
127
|
+
|
|
128
|
+
it( 'rethrows non-RangeError from pretty stringify', () => {
|
|
129
|
+
const content = { a: 1 };
|
|
130
|
+
const origStringify = JSON.stringify.bind( JSON );
|
|
131
|
+
|
|
132
|
+
const spy = vi.spyOn( JSON, 'stringify' ).mockImplementation( ( ...args ) => {
|
|
133
|
+
if ( isPrettyStringifyCall( args ) ) {
|
|
134
|
+
throw new TypeError( 'cyclic structure' );
|
|
135
|
+
}
|
|
136
|
+
return origStringify( ...args );
|
|
137
|
+
} );
|
|
138
|
+
|
|
139
|
+
try {
|
|
140
|
+
expect( () => safeFormatJSON( content, 10_000 ) ).toThrow( TypeError );
|
|
141
|
+
} finally {
|
|
142
|
+
spy.mockRestore();
|
|
143
|
+
}
|
|
144
|
+
} );
|
|
145
|
+
} );
|
|
14
146
|
} );
|
package/src/worker/configs.js
CHANGED
|
@@ -26,7 +26,14 @@ const envVarSchema = z.object( {
|
|
|
26
26
|
// Whether to send activity heartbeats (enabled by default)
|
|
27
27
|
OUTPUT_ACTIVITY_HEARTBEAT_ENABLED: z.transform( v => v === undefined ? true : isStringboolTrue( v ) ),
|
|
28
28
|
// Time to allow for hooks to flush before shutdown
|
|
29
|
-
OUTPUT_PROCESS_FAILURE_SHUTDOWN_DELAY: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 3000 ) )
|
|
29
|
+
OUTPUT_PROCESS_FAILURE_SHUTDOWN_DELAY: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 3000 ) ),
|
|
30
|
+
// HTTP CONNECT proxy for Temporal gRPC connections (e.g. "proxy-host:8080").
|
|
31
|
+
// Must be a bare host:port — no scheme (Temporal's native HTTP CONNECT
|
|
32
|
+
// option is not a URL).
|
|
33
|
+
TEMPORAL_GRPC_PROXY: z.string().optional().refine(
|
|
34
|
+
v => !v || !v.includes( '://' ),
|
|
35
|
+
'TEMPORAL_GRPC_PROXY must be host:port without a scheme (e.g. "proxy:8080", not "http://proxy:8080")'
|
|
36
|
+
)
|
|
30
37
|
} );
|
|
31
38
|
|
|
32
39
|
const { data: envVars, error } = envVarSchema.safeParse( process.env );
|
|
@@ -47,3 +54,4 @@ export const catalogId = envVars.OUTPUT_CATALOG_ID;
|
|
|
47
54
|
export const activityHeartbeatIntervalMs = envVars.OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS;
|
|
48
55
|
export const activityHeartbeatEnabled = envVars.OUTPUT_ACTIVITY_HEARTBEAT_ENABLED;
|
|
49
56
|
export const processFailureShutdownDelay = envVars.OUTPUT_PROCESS_FAILURE_SHUTDOWN_DELAY;
|
|
57
|
+
export const grpcProxy = envVars.TEMPORAL_GRPC_PROXY;
|
package/src/worker/index.js
CHANGED
|
@@ -9,6 +9,7 @@ import { initInterceptors } from './interceptors.js';
|
|
|
9
9
|
import { createChildLogger } from '#logger';
|
|
10
10
|
import { registerShutdown } from './shutdown.js';
|
|
11
11
|
import { startCatalog } from './start_catalog.js';
|
|
12
|
+
import { bootstrapFetchProxy } from './proxy.js';
|
|
12
13
|
import { messageBus } from '#bus';
|
|
13
14
|
import './log_hooks.js';
|
|
14
15
|
import { BusEventType } from '#consts';
|
|
@@ -24,6 +25,7 @@ const callerDir = process.argv[2];
|
|
|
24
25
|
apiKey,
|
|
25
26
|
namespace,
|
|
26
27
|
taskQueue,
|
|
28
|
+
grpcProxy,
|
|
27
29
|
maxConcurrentWorkflowTaskExecutions,
|
|
28
30
|
maxConcurrentActivityTaskExecutions,
|
|
29
31
|
maxCachedWorkflows,
|
|
@@ -41,6 +43,7 @@ const callerDir = process.argv[2];
|
|
|
41
43
|
const activities = await loadActivities( callerDir, workflows );
|
|
42
44
|
|
|
43
45
|
messageBus.emit( BusEventType.WORKER_BEFORE_START );
|
|
46
|
+
bootstrapFetchProxy();
|
|
44
47
|
|
|
45
48
|
log.info( 'Creating worker entry point...' );
|
|
46
49
|
const workflowsPath = createWorkflowsEntryPoint( workflows );
|
|
@@ -52,8 +55,11 @@ const callerDir = process.argv[2];
|
|
|
52
55
|
const catalog = createCatalog( { workflows, activities } );
|
|
53
56
|
|
|
54
57
|
log.info( 'Connecting Temporal...' );
|
|
55
|
-
|
|
56
|
-
|
|
58
|
+
const proxy = grpcProxy ? { type: 'http-connect', targetHost: grpcProxy } : undefined;
|
|
59
|
+
if ( proxy ) {
|
|
60
|
+
log.info( 'Using gRPC proxy', { targetHost: grpcProxy } );
|
|
61
|
+
}
|
|
62
|
+
const connection = await NativeConnection.connect( { address, tls: Boolean( apiKey ), apiKey, proxy } );
|
|
57
63
|
|
|
58
64
|
log.info( 'Creating worker...' );
|
|
59
65
|
const worker = await Worker.create( {
|
package/src/worker/index.spec.js
CHANGED
|
@@ -16,6 +16,7 @@ const configValues = {
|
|
|
16
16
|
namespace: 'default',
|
|
17
17
|
taskQueue: 'test-queue',
|
|
18
18
|
catalogId: 'test-catalog',
|
|
19
|
+
grpcProxy: undefined,
|
|
19
20
|
maxConcurrentWorkflowTaskExecutions: 200,
|
|
20
21
|
maxConcurrentActivityTaskExecutions: 40,
|
|
21
22
|
maxCachedWorkflows: 1000,
|
|
@@ -52,6 +53,9 @@ vi.mock( './interceptors.js', () => ( { initInterceptors: initInterceptorsMock }
|
|
|
52
53
|
const startCatalogMock = vi.fn().mockResolvedValue( undefined );
|
|
53
54
|
vi.mock( './start_catalog.js', () => ( { startCatalog: startCatalogMock } ) );
|
|
54
55
|
|
|
56
|
+
const bootstrapFetchProxyMock = vi.fn();
|
|
57
|
+
vi.mock( './proxy.js', () => ( { bootstrapFetchProxy: bootstrapFetchProxyMock } ) );
|
|
58
|
+
|
|
55
59
|
const registerShutdownMock = vi.fn();
|
|
56
60
|
vi.mock( './shutdown.js', () => ( { registerShutdown: registerShutdownMock } ) );
|
|
57
61
|
|
|
@@ -101,10 +105,12 @@ describe( 'worker/index', () => {
|
|
|
101
105
|
expect( createWorkflowsEntryPointMock ).toHaveBeenCalledWith( [] );
|
|
102
106
|
expect( initTracing ).toHaveBeenCalled();
|
|
103
107
|
expect( createCatalogMock ).toHaveBeenCalledWith( { workflows: [], activities: {} } );
|
|
108
|
+
expect( bootstrapFetchProxyMock ).toHaveBeenCalled();
|
|
104
109
|
expect( NativeConnection.connect ).toHaveBeenCalledWith( {
|
|
105
110
|
address: configValues.address,
|
|
106
111
|
tls: false,
|
|
107
|
-
apiKey: undefined
|
|
112
|
+
apiKey: undefined,
|
|
113
|
+
proxy: undefined
|
|
108
114
|
} );
|
|
109
115
|
expect( Worker.create ).toHaveBeenCalledWith( expect.objectContaining( {
|
|
110
116
|
namespace: configValues.namespace,
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { EnvHttpProxyAgent, setGlobalDispatcher } from 'undici';
|
|
2
|
+
import { createChildLogger } from '#logger';
|
|
3
|
+
|
|
4
|
+
const log = createChildLogger( 'Proxy' );
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Routes all `fetch()` calls (including those inside Temporal activities)
|
|
8
|
+
* through an HTTP/HTTPS proxy when standard proxy env vars are set
|
|
9
|
+
* (`HTTPS_PROXY`, `https_proxy`, `HTTP_PROXY`, `http_proxy`). No-op when
|
|
10
|
+
* none are set. Invalid URLs are logged and skipped so the worker keeps
|
|
11
|
+
* running.
|
|
12
|
+
*
|
|
13
|
+
* Call once at worker startup, before any network activity.
|
|
14
|
+
*/
|
|
15
|
+
export const bootstrapFetchProxy = () => {
|
|
16
|
+
const proxyUrl = process.env.HTTPS_PROXY || process.env.https_proxy ||
|
|
17
|
+
process.env.HTTP_PROXY || process.env.http_proxy;
|
|
18
|
+
|
|
19
|
+
if ( !proxyUrl ) {
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
try {
|
|
24
|
+
new URL( proxyUrl );
|
|
25
|
+
} catch {
|
|
26
|
+
log.warn( 'Ignoring invalid proxy URL', { proxyUrl } );
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
log.info( 'Routing fetch() through HTTP proxy', { proxyUrl } );
|
|
31
|
+
setGlobalDispatcher( new EnvHttpProxyAgent() );
|
|
32
|
+
};
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
|
|
3
|
+
const mockSetGlobalDispatcher = vi.fn();
|
|
4
|
+
const MockEnvHttpProxyAgent = vi.fn();
|
|
5
|
+
|
|
6
|
+
vi.mock( 'undici', () => ( {
|
|
7
|
+
EnvHttpProxyAgent: MockEnvHttpProxyAgent,
|
|
8
|
+
setGlobalDispatcher: mockSetGlobalDispatcher
|
|
9
|
+
} ) );
|
|
10
|
+
|
|
11
|
+
vi.mock( '#logger', () => ( {
|
|
12
|
+
createChildLogger: () => ( { info: vi.fn(), warn: vi.fn(), error: vi.fn() } )
|
|
13
|
+
} ) );
|
|
14
|
+
|
|
15
|
+
describe( 'worker/proxy', () => {
|
|
16
|
+
const originalEnv = { ...process.env };
|
|
17
|
+
|
|
18
|
+
beforeEach( () => {
|
|
19
|
+
vi.clearAllMocks();
|
|
20
|
+
delete process.env.HTTPS_PROXY;
|
|
21
|
+
delete process.env.https_proxy;
|
|
22
|
+
delete process.env.HTTP_PROXY;
|
|
23
|
+
delete process.env.http_proxy;
|
|
24
|
+
} );
|
|
25
|
+
|
|
26
|
+
afterEach( () => {
|
|
27
|
+
process.env = { ...originalEnv };
|
|
28
|
+
} );
|
|
29
|
+
|
|
30
|
+
it( 'does nothing when no proxy env vars are set', async () => {
|
|
31
|
+
const { bootstrapFetchProxy } = await import( './proxy.js' );
|
|
32
|
+
bootstrapFetchProxy();
|
|
33
|
+
|
|
34
|
+
expect( mockSetGlobalDispatcher ).not.toHaveBeenCalled();
|
|
35
|
+
} );
|
|
36
|
+
|
|
37
|
+
it( 'sets global dispatcher when HTTPS_PROXY is set', async () => {
|
|
38
|
+
process.env.HTTPS_PROXY = 'http://proxy:8080';
|
|
39
|
+
const { bootstrapFetchProxy } = await import( './proxy.js' );
|
|
40
|
+
bootstrapFetchProxy();
|
|
41
|
+
|
|
42
|
+
expect( MockEnvHttpProxyAgent ).toHaveBeenCalled();
|
|
43
|
+
expect( mockSetGlobalDispatcher ).toHaveBeenCalledTimes( 1 );
|
|
44
|
+
} );
|
|
45
|
+
|
|
46
|
+
it( 'sets global dispatcher when HTTP_PROXY is set', async () => {
|
|
47
|
+
process.env.HTTP_PROXY = 'http://proxy:8080';
|
|
48
|
+
const { bootstrapFetchProxy } = await import( './proxy.js' );
|
|
49
|
+
bootstrapFetchProxy();
|
|
50
|
+
|
|
51
|
+
expect( MockEnvHttpProxyAgent ).toHaveBeenCalled();
|
|
52
|
+
expect( mockSetGlobalDispatcher ).toHaveBeenCalledTimes( 1 );
|
|
53
|
+
} );
|
|
54
|
+
|
|
55
|
+
it( 'prefers HTTPS_PROXY over HTTP_PROXY for detection', async () => {
|
|
56
|
+
process.env.HTTPS_PROXY = 'http://secure-proxy:8080';
|
|
57
|
+
process.env.HTTP_PROXY = 'http://plain-proxy:8080';
|
|
58
|
+
const { bootstrapFetchProxy } = await import( './proxy.js' );
|
|
59
|
+
bootstrapFetchProxy();
|
|
60
|
+
|
|
61
|
+
expect( mockSetGlobalDispatcher ).toHaveBeenCalledTimes( 1 );
|
|
62
|
+
} );
|
|
63
|
+
|
|
64
|
+
it( 'does not set global dispatcher when proxy URL is malformed', async () => {
|
|
65
|
+
process.env.HTTPS_PROXY = 'not a url';
|
|
66
|
+
const { bootstrapFetchProxy } = await import( './proxy.js' );
|
|
67
|
+
bootstrapFetchProxy();
|
|
68
|
+
|
|
69
|
+
expect( mockSetGlobalDispatcher ).not.toHaveBeenCalled();
|
|
70
|
+
} );
|
|
71
|
+
} );
|