@output.ai/core 0.0.15 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/worker.sh +1 -1
- package/package.json +17 -10
- package/src/configs.js +1 -6
- package/src/configs.spec.js +2 -50
- package/src/consts.js +6 -8
- package/src/index.d.ts +169 -7
- package/src/index.js +18 -1
- package/src/interface/evaluator.js +146 -0
- package/src/interface/step.js +4 -9
- package/src/interface/{schema_utils.js → validations/runtime.js} +0 -14
- package/src/interface/validations/runtime.spec.js +29 -0
- package/src/interface/validations/schema_utils.js +8 -0
- package/src/interface/validations/static.js +13 -1
- package/src/interface/validations/static.spec.js +29 -1
- package/src/interface/webhook.js +16 -4
- package/src/interface/workflow.js +32 -54
- package/src/internal_activities/index.js +16 -12
- package/src/tracing/index.d.ts +47 -0
- package/src/tracing/index.js +154 -0
- package/src/tracing/index.private.spec.js +84 -0
- package/src/tracing/index.public.spec.js +86 -0
- package/src/tracing/tracer_tree.js +83 -0
- package/src/tracing/tracer_tree.spec.js +115 -0
- package/src/tracing/utils.js +21 -0
- package/src/tracing/utils.spec.js +14 -0
- package/src/worker/catalog_workflow/catalog.js +19 -10
- package/src/worker/index.js +1 -5
- package/src/worker/interceptors/activity.js +28 -10
- package/src/worker/interceptors/workflow.js +19 -1
- package/src/worker/loader.js +6 -6
- package/src/worker/loader.spec.js +6 -9
- package/src/worker/sinks.js +56 -10
- package/src/worker/webpack_loaders/workflow_rewriter/collect_target_imports.js +35 -4
- package/src/worker/webpack_loaders/workflow_rewriter/collect_target_imports.spec.js +12 -4
- package/src/worker/webpack_loaders/workflow_rewriter/index.mjs +5 -4
- package/src/worker/webpack_loaders/workflow_rewriter/rewrite_fn_bodies.js +13 -4
- package/src/worker/webpack_loaders/workflow_rewriter/rewrite_fn_bodies.spec.js +16 -2
- package/src/worker/webpack_loaders/workflow_rewriter/tools.js +46 -13
- package/src/worker/webpack_loaders/workflow_rewriter/tools.spec.js +20 -2
- package/src/worker/tracer/index.js +0 -75
- package/src/worker/tracer/index.test.js +0 -103
- package/src/worker/tracer/tracer_tree.js +0 -84
- package/src/worker/tracer/tracer_tree.test.js +0 -115
- /package/src/{worker/async_storage.js → async_storage.js} +0 -0
- /package/src/interface/{schema_utils.spec.js → validations/schema_utils.spec.js} +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as z from 'zod';
|
|
2
|
-
import { isZodSchema } from '
|
|
2
|
+
import { isZodSchema } from './schema_utils.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Error is thrown when the definition of a step/workflow has problems
|
|
@@ -25,6 +25,8 @@ const stepAndWorkflowSchema = z.object( {
|
|
|
25
25
|
fn: z.function()
|
|
26
26
|
} );
|
|
27
27
|
|
|
28
|
+
const evaluatorSchema = stepAndWorkflowSchema.omit( { outputSchema: true } );
|
|
29
|
+
|
|
28
30
|
const webhookSchema = z.object( {
|
|
29
31
|
url: z.url( { protocol: /^https?$/ } ),
|
|
30
32
|
payload: z.any().optional()
|
|
@@ -47,6 +49,16 @@ export function validateStep( args ) {
|
|
|
47
49
|
validateAgainstSchema( stepAndWorkflowSchema, args );
|
|
48
50
|
};
|
|
49
51
|
|
|
52
|
+
/**
|
|
53
|
+
* Validate evaluator payload
|
|
54
|
+
*
|
|
55
|
+
* @param {object} args - The evaluator arguments
|
|
56
|
+
* @throws {StaticValidationError} Throws if args are invalid
|
|
57
|
+
*/
|
|
58
|
+
export function validateEvaluator( args ) {
|
|
59
|
+
validateAgainstSchema( evaluatorSchema, args );
|
|
60
|
+
};
|
|
61
|
+
|
|
50
62
|
/**
|
|
51
63
|
* Validate workflow payload
|
|
52
64
|
*
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { describe, it, expect } from 'vitest';
|
|
2
2
|
import { z } from 'zod';
|
|
3
|
-
import { validateStep, validateWorkflow, validateCreateWebhook, StaticValidationError } from './static.js';
|
|
3
|
+
import { validateStep, validateWorkflow, validateCreateWebhook, validateEvaluator, StaticValidationError } from './static.js';
|
|
4
4
|
|
|
5
5
|
const validArgs = Object.freeze( {
|
|
6
6
|
name: 'valid_name',
|
|
@@ -73,6 +73,34 @@ describe( 'interface/validator', () => {
|
|
|
73
73
|
} );
|
|
74
74
|
} );
|
|
75
75
|
|
|
76
|
+
describe( 'validateEvaluator', () => {
|
|
77
|
+
const base = Object.freeze( {
|
|
78
|
+
name: 'valid_name',
|
|
79
|
+
description: 'desc',
|
|
80
|
+
inputSchema: z.object( {} ),
|
|
81
|
+
fn: () => {}
|
|
82
|
+
} );
|
|
83
|
+
|
|
84
|
+
it( 'passes for valid args (no outputSchema)', () => {
|
|
85
|
+
expect( () => validateEvaluator( { ...base } ) ).not.toThrow();
|
|
86
|
+
} );
|
|
87
|
+
|
|
88
|
+
it( 'rejects invalid name pattern', () => {
|
|
89
|
+
const error = new StaticValidationError( '✖ Invalid string: must match pattern /^[a-z_][a-z0-9_]*$/i\n → at name' );
|
|
90
|
+
expect( () => validateEvaluator( { ...base, name: '-bad' } ) ).toThrow( error );
|
|
91
|
+
} );
|
|
92
|
+
|
|
93
|
+
it( 'rejects non-Zod inputSchema', () => {
|
|
94
|
+
const error = new StaticValidationError( '✖ Schema must be a Zod schema\n → at inputSchema' );
|
|
95
|
+
expect( () => validateEvaluator( { ...base, inputSchema: 'not-a-zod-schema' } ) ).toThrow( error );
|
|
96
|
+
} );
|
|
97
|
+
|
|
98
|
+
it( 'rejects missing fn', () => {
|
|
99
|
+
const error = new StaticValidationError( '✖ Invalid input: expected function, received undefined\n → at fn' );
|
|
100
|
+
expect( () => validateEvaluator( { ...base, fn: undefined } ) ).toThrow( error );
|
|
101
|
+
} );
|
|
102
|
+
} );
|
|
103
|
+
|
|
76
104
|
describe( 'validate webhook', () => {
|
|
77
105
|
it( 'passes with valid http url', () => {
|
|
78
106
|
expect( () => validateCreateWebhook( { url: 'http://example.com' } ) ).not.toThrow();
|
package/src/interface/webhook.js
CHANGED
|
@@ -1,18 +1,30 @@
|
|
|
1
1
|
// THIS RUNS IN THE TEMPORAL'S SANDBOX ENVIRONMENT
|
|
2
|
-
import { defineSignal, setHandler, proxyActivities, workflowInfo } from '@temporalio/workflow';
|
|
3
|
-
import {
|
|
2
|
+
import { defineSignal, setHandler, proxyActivities, workflowInfo, proxySinks } from '@temporalio/workflow';
|
|
3
|
+
import { ACTIVITY_SEND_WEBHOOK } from '#consts';
|
|
4
|
+
import { FatalError } from '#errors';
|
|
4
5
|
import { validateCreateWebhook } from './validations/static.js';
|
|
5
6
|
|
|
6
7
|
export async function createWebhook( { url, payload } ) {
|
|
7
8
|
validateCreateWebhook( { url, payload } );
|
|
8
|
-
const workflowId = workflowInfo();
|
|
9
|
+
const { workflowId } = workflowInfo();
|
|
9
10
|
|
|
10
|
-
await proxyActivities(
|
|
11
|
+
await proxyActivities( {
|
|
12
|
+
startToCloseTimeout: '3m',
|
|
13
|
+
retry: {
|
|
14
|
+
initialInterval: '15s',
|
|
15
|
+
maximumAttempts: 5,
|
|
16
|
+
nonRetryableErrorTypes: [ FatalError.name ]
|
|
17
|
+
}
|
|
18
|
+
} )[ACTIVITY_SEND_WEBHOOK]( { url, workflowId, payload } );
|
|
11
19
|
|
|
20
|
+
const sinks = await proxySinks();
|
|
12
21
|
const resumeSignal = defineSignal( 'resume' );
|
|
13
22
|
|
|
23
|
+
const traceId = `${workflowId}-${url}-${Date.now()}`;
|
|
24
|
+
sinks.trace.addEventStart( { id: traceId, name: 'resume', kind: 'webhook' } );
|
|
14
25
|
return new Promise( resolve =>
|
|
15
26
|
setHandler( resumeSignal, responsePayload => {
|
|
27
|
+
sinks.trace.addEventEnd( { id: traceId, details: responsePayload } );
|
|
16
28
|
resolve( responsePayload );
|
|
17
29
|
} )
|
|
18
30
|
);
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
// THIS RUNS IN THE TEMPORAL'S SANDBOX ENVIRONMENT
|
|
2
|
-
import { proxyActivities, inWorkflowContext, executeChild, workflowInfo
|
|
2
|
+
import { proxyActivities, inWorkflowContext, executeChild, workflowInfo } from '@temporalio/workflow';
|
|
3
3
|
import { getInvocationDir } from './utils.js';
|
|
4
4
|
import { setMetadata } from './metadata.js';
|
|
5
5
|
import { FatalError, ValidationError } from '#errors';
|
|
6
6
|
import { validateWorkflow } from './validations/static.js';
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
7
|
+
import { validateWithSchema } from './validations/runtime.js';
|
|
8
|
+
import { ACTIVITY_READ_TRACE_FILE } from '#consts';
|
|
9
9
|
|
|
10
10
|
const temporalActivityConfigs = {
|
|
11
11
|
startToCloseTimeout: '20 minute',
|
|
@@ -19,71 +19,49 @@ const temporalActivityConfigs = {
|
|
|
19
19
|
};
|
|
20
20
|
|
|
21
21
|
export function workflow( { name, description, inputSchema, outputSchema, fn } ) {
|
|
22
|
-
validateWorkflow( {
|
|
23
|
-
name,
|
|
24
|
-
description,
|
|
25
|
-
inputSchema,
|
|
26
|
-
outputSchema,
|
|
27
|
-
fn
|
|
28
|
-
} );
|
|
22
|
+
validateWorkflow( { name, description, inputSchema, outputSchema, fn } );
|
|
29
23
|
const workflowPath = getInvocationDir();
|
|
30
24
|
|
|
31
25
|
const steps = proxyActivities( temporalActivityConfigs );
|
|
32
|
-
const sinks = proxySinks();
|
|
33
26
|
|
|
34
27
|
const wrapper = async input => {
|
|
35
|
-
|
|
36
|
-
if ( inWorkflowContext() ) {
|
|
37
|
-
sinks.log.trace( { event: TraceEvent.WORKFLOW_START, input } );
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
validateWithSchema( inputSchema, input, `Workflow ${name} input` );
|
|
41
|
-
|
|
42
|
-
// this returns a plain function, for example, in unit tests
|
|
43
|
-
if ( !inWorkflowContext() ) {
|
|
44
|
-
const output = await fn( input );
|
|
45
|
-
validateWithSchema( outputSchema, output, `Workflow ${name} output` );
|
|
46
|
-
return output;
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
const { memo, workflowId } = workflowInfo();
|
|
50
|
-
|
|
51
|
-
Object.assign( workflowInfo().memo, { workflowPath } );
|
|
52
|
-
|
|
53
|
-
// binds the methods called in the code that Webpack loader will add, they will exposed via "this"
|
|
54
|
-
const output = await fn.call( {
|
|
55
|
-
invokeStep: async ( stepName, input ) => steps[`${workflowPath}#${stepName}`]( input ),
|
|
28
|
+
validateWithSchema( inputSchema, input, `Workflow ${name} input` );
|
|
56
29
|
|
|
57
|
-
|
|
30
|
+
// this returns a plain function, for example, in unit tests
|
|
31
|
+
if ( !inWorkflowContext() ) {
|
|
32
|
+
const output = await fn( input );
|
|
33
|
+
validateWithSchema( outputSchema, output, `Workflow ${name} output` );
|
|
34
|
+
return output;
|
|
35
|
+
}
|
|
58
36
|
|
|
59
|
-
|
|
60
|
-
// Then it sets the memory for the child execution passing along who's the original workflow is and its type
|
|
61
|
-
const workflowMemory = memo.rootWorkflowId ?
|
|
62
|
-
{ parentWorkflowId: workflowId, rootWorkflowType: memo.rootWorkflowType, rootWorkflowId: memo.rootWorkflowId } :
|
|
63
|
-
{ parentWorkflowId: workflowId, rootWorkflowId: workflowId, rootWorkflowType: name };
|
|
37
|
+
const { workflowId, memo } = workflowInfo();
|
|
64
38
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
39
|
+
// keep trace id and helm from memo if present (child workflow)
|
|
40
|
+
const traceId = memo.traceId ?? workflowId;
|
|
41
|
+
const traceHelm = memo.traceHelm ?? name;
|
|
68
42
|
|
|
69
|
-
|
|
43
|
+
Object.assign( memo, { traceId, traceHelm } );
|
|
70
44
|
|
|
71
|
-
|
|
45
|
+
// binds the methods called in the code that Webpack loader will add, they will exposed via "this"
|
|
46
|
+
const output = await fn.call( {
|
|
47
|
+
invokeStep: async ( stepName, input ) => steps[`${workflowPath}#${stepName}`]( input ),
|
|
48
|
+
invokeEvaluator: async ( evaluatorName, input ) => steps[`${workflowPath}#${evaluatorName}`]( input ),
|
|
72
49
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
const trace = await steps[READ_TRACE_FILE]( { workflowType: name, workflowId } );
|
|
76
|
-
return { output, trace };
|
|
50
|
+
startWorkflow: async ( childName, input ) => {
|
|
51
|
+
return executeChild( childName, { args: input ? [ input ] : [], memo: { traceId, traceHelm, parentId: workflowId } } );
|
|
77
52
|
}
|
|
53
|
+
}, input );
|
|
78
54
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
55
|
+
validateWithSchema( outputSchema, output, `Workflow ${name} output` );
|
|
56
|
+
|
|
57
|
+
// Adds trace file content to the output if it is the root workflow
|
|
58
|
+
// @TODO this will be replaced in favor of a persistent storage elsewhere
|
|
59
|
+
if ( !memo.parentId ) {
|
|
60
|
+
const trace = await steps[ACTIVITY_READ_TRACE_FILE]( { traceId, traceHelm } );
|
|
61
|
+
return { output, trace };
|
|
86
62
|
}
|
|
63
|
+
|
|
64
|
+
return output;
|
|
87
65
|
};
|
|
88
66
|
|
|
89
67
|
setMetadata( wrapper, { name, description, inputSchema, outputSchema } );
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { FatalError } from '#errors';
|
|
2
2
|
import { existsSync, readFileSync, readdirSync } from 'node:fs';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
|
+
import { setMetadata } from '../interface/metadata.js';
|
|
5
|
+
import { ComponentType } from '#consts';
|
|
4
6
|
|
|
5
7
|
const callerDir = process.argv[2];
|
|
6
8
|
|
|
@@ -13,7 +15,7 @@ const callerDir = process.argv[2];
|
|
|
13
15
|
* @param {any} options.payload - The payload to send url
|
|
14
16
|
* @throws {FatalError}
|
|
15
17
|
*/
|
|
16
|
-
export const
|
|
18
|
+
export const sendWebhook = async ( { url, workflowId, payload } ) => {
|
|
17
19
|
const request = fetch( url, {
|
|
18
20
|
method: 'POST',
|
|
19
21
|
headers: {
|
|
@@ -31,37 +33,39 @@ export const sendWebhookPost = async ( { url, workflowId, payload } ) => {
|
|
|
31
33
|
}
|
|
32
34
|
} )();
|
|
33
35
|
|
|
34
|
-
console.log( '[Core.
|
|
36
|
+
console.log( '[Core.SendWebhook]', res.status, res.statusText );
|
|
35
37
|
|
|
36
38
|
if ( !res.ok ) {
|
|
37
39
|
throw new FatalError( `Webhook fail: ${res.status}` );
|
|
38
40
|
}
|
|
39
41
|
};
|
|
40
42
|
|
|
43
|
+
setMetadata( sendWebhook, { type: ComponentType.INTERNAL_STEP } );
|
|
44
|
+
|
|
41
45
|
/**
|
|
42
46
|
* Read the trace file of a given execution and returns the content
|
|
43
47
|
*
|
|
44
48
|
* @param {object} options
|
|
45
|
-
* @param {string} options.
|
|
46
|
-
* @param {string} options.
|
|
49
|
+
* @param {string} options.traceId - The is of the trace
|
|
50
|
+
* @param {string} options.traceHelm - The helm of the trace file
|
|
47
51
|
* @returns {string[]} Each line of the trace file
|
|
48
52
|
*/
|
|
49
|
-
export const readTraceFile = async ( {
|
|
50
|
-
const dir = join( callerDir, 'logs', 'runs',
|
|
53
|
+
export const readTraceFile = async ( { traceId, traceHelm } ) => {
|
|
54
|
+
const dir = join( callerDir, 'logs', 'runs', traceHelm );
|
|
51
55
|
|
|
52
56
|
if ( !existsSync( dir ) ) {
|
|
53
57
|
console.log( '[Core.ReadTraceFile]', 'Trace folder not found', dir );
|
|
54
58
|
return [];
|
|
55
59
|
}
|
|
56
60
|
|
|
57
|
-
const
|
|
58
|
-
const matchingFile = readdirSync( dir ).find( f => f.endsWith( suffix ) );
|
|
61
|
+
const fileName = readdirSync( dir ).find( f => f.endsWith( `-${traceId}.raw` ) );
|
|
59
62
|
|
|
60
|
-
if ( !
|
|
61
|
-
console.log( '[Core.ReadTraceFile]', 'Trace file not found',
|
|
63
|
+
if ( !fileName ) {
|
|
64
|
+
console.log( '[Core.ReadTraceFile]', 'Trace file not found', { traceId, traceHelm } );
|
|
62
65
|
return [];
|
|
63
66
|
}
|
|
64
67
|
|
|
65
|
-
|
|
66
|
-
return existsSync( file ) ? readFileSync( file, 'utf-8' ).split( '\n' ) : [];
|
|
68
|
+
return readFileSync( join( dir, fileName ), 'utf-8' ).split( '\n' );
|
|
67
69
|
};
|
|
70
|
+
|
|
71
|
+
setMetadata( readTraceFile, { type: ComponentType.INTERNAL_STEP, skipTrace: true } );
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/*
|
|
2
|
+
╭───────────╮
|
|
3
|
+
│ T R A C E │╮
|
|
4
|
+
╰───────────╯│
|
|
5
|
+
╰───────────╯
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The public namespace for tracing
|
|
10
|
+
*
|
|
11
|
+
* @namespace
|
|
12
|
+
*/
|
|
13
|
+
export declare const Tracing: {
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Adds the start phase of a new event at the default trace for the current workflow.
|
|
17
|
+
*
|
|
18
|
+
* @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
19
|
+
* @param {string} kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
|
|
20
|
+
* @param {string} name - The human friendly name of the Event: eg: query, request, create.
|
|
21
|
+
* @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
|
|
22
|
+
* @returns {void}
|
|
23
|
+
*/
|
|
24
|
+
addEventStart( args: { id: string; kind: string; name: string; details: any } ): void; // eslint-disable-line @typescript-eslint/no-explicit-any
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Adds the end phase at an event at the default trace for the current workflow.
|
|
28
|
+
*
|
|
29
|
+
* It needs to use the same id of the start phase.
|
|
30
|
+
*
|
|
31
|
+
* @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
32
|
+
* @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
|
|
33
|
+
* @returns {void}
|
|
34
|
+
*/
|
|
35
|
+
addEventEnd( args: { id: string; details: any } ): void; // eslint-disable-line @typescript-eslint/no-explicit-any
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Adds the error phase at an event as error at the default trace for the current workflow.
|
|
39
|
+
*
|
|
40
|
+
* It needs to use the same id of the start phase.
|
|
41
|
+
*
|
|
42
|
+
* @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
43
|
+
* @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
|
|
44
|
+
* @returns {void}
|
|
45
|
+
*/
|
|
46
|
+
addEventError( args: { id: string; details: any } ): void; // eslint-disable-line @typescript-eslint/no-explicit-any
|
|
47
|
+
};
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import { Storage } from '#async_storage';
|
|
2
|
+
import { mkdirSync, existsSync, readdirSync, appendFileSync } from 'node:fs';
|
|
3
|
+
import { join } from 'path';
|
|
4
|
+
import { EOL } from 'os';
|
|
5
|
+
import { buildLogTree } from './tracer_tree.js';
|
|
6
|
+
import { serializeError } from './utils.js';
|
|
7
|
+
|
|
8
|
+
const callerDir = process.argv[2];
|
|
9
|
+
// It is is isolated here instead of #configs to allow this module to be exported without all other configs requirements
|
|
10
|
+
const tracingEnabled = [ '1', 'true', 'on' ].includes( process.env.TRACING_ENABLED );
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Trace nomenclature
|
|
14
|
+
*
|
|
15
|
+
* Trace - The collection of Events;
|
|
16
|
+
* Event - Any entry in the Trace file, must have the two phases START and END or ERROR;
|
|
17
|
+
* Phase - An specific part of an Event, either START or the conclusive END or ERROR;
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Adds the Trace Event Phase to the trace file
|
|
22
|
+
*
|
|
23
|
+
* @param {string} phase - The phase
|
|
24
|
+
* @param {object} options - All the trace fields
|
|
25
|
+
* @returns {void}
|
|
26
|
+
*/
|
|
27
|
+
function addEventPhase( phase, { kind, details, id, name, parentId, traceId, traceHelm } ) {
|
|
28
|
+
if ( !tracingEnabled || name === 'catalog' ) {
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const parsedDetails = details instanceof Error ? serializeError( details ) : details;
|
|
33
|
+
const timestamp = Date.now();
|
|
34
|
+
const entry = { phase, kind, details: parsedDetails, id, name, timestamp, parentId };
|
|
35
|
+
const outputDir = join( callerDir, 'logs', 'runs', traceHelm );
|
|
36
|
+
|
|
37
|
+
if ( !existsSync( outputDir ) ) {
|
|
38
|
+
mkdirSync( outputDir, { recursive: true } );
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const suffix = `-${traceId}.raw`;
|
|
42
|
+
const logFile = readdirSync( outputDir ).find( f => f.endsWith( suffix ) ) ?? `${new Date( timestamp ).toISOString()}-${suffix}`;
|
|
43
|
+
const logPath = join( outputDir, logFile );
|
|
44
|
+
|
|
45
|
+
appendFileSync( logPath, JSON.stringify( entry ) + EOL, 'utf-8' );
|
|
46
|
+
buildLogTree( logPath );
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Internal use only
|
|
51
|
+
*
|
|
52
|
+
* Adds the start phase of a new event at the default trace for the current workflow.
|
|
53
|
+
*
|
|
54
|
+
* @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
55
|
+
* @param {string} kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
|
|
56
|
+
* @param {string} name - The human friendly name of the Event: eg: query, request, create.
|
|
57
|
+
* @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
|
|
58
|
+
* @param {string} parentId - The parent Event, used to build a three.
|
|
59
|
+
* @param {string} traceId - The traceId, which identifies from which trace this Event belongs.
|
|
60
|
+
* @param {string} traceHelm - The trace helm, which is a taxonomical naming of the Trace.
|
|
61
|
+
* @returns {void}
|
|
62
|
+
*/
|
|
63
|
+
export const addEventStart = options => addEventPhase( 'start', options );
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Internal use only
|
|
67
|
+
*
|
|
68
|
+
* Adds the end phase at an event at the default trace for the current workflow.
|
|
69
|
+
*
|
|
70
|
+
* It needs to use the same id of the start phase.
|
|
71
|
+
*
|
|
72
|
+
* @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
73
|
+
* @param {string} kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
|
|
74
|
+
* @param {string} name - The human friendly name of the Event: eg: query, request, create.
|
|
75
|
+
* @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
|
|
76
|
+
* @param {string} parentId - The parent Event, used to build a three.
|
|
77
|
+
* @param {string} traceId - The traceId, which identifies from which trace this Event belongs.
|
|
78
|
+
* @param {string} traceHelm - The trace helm, which is a taxonomical naming of the Trace.
|
|
79
|
+
* @returns {void}
|
|
80
|
+
*/
|
|
81
|
+
export const addEventEnd = options => addEventPhase( 'end', options );
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Internal use only
|
|
85
|
+
*
|
|
86
|
+
* Adds the error phase at an event as error at the default trace for the current workflow.
|
|
87
|
+
*
|
|
88
|
+
* It needs to use the same id of the start phase.
|
|
89
|
+
*
|
|
90
|
+
* @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
91
|
+
* @param {string} kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
|
|
92
|
+
* @param {string} name - The human friendly name of the Event: eg: query, request, create.
|
|
93
|
+
* @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
|
|
94
|
+
* @param {string} parentId - The parent Event, used to build a three.
|
|
95
|
+
* @param {string} traceId - The traceId, which identifies from which trace this Event belongs.
|
|
96
|
+
* @param {string} traceHelm - The trace helm, which is a taxonomical naming of the Trace.
|
|
97
|
+
* @returns {void}
|
|
98
|
+
*/
|
|
99
|
+
export const addEventError = options => addEventPhase( 'error', options );
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Adds an Event Phase reading traceId, traceHelm and parentId from the context.
|
|
103
|
+
*
|
|
104
|
+
* @param {object} options - The common trace configurations
|
|
105
|
+
*/
|
|
106
|
+
function addEventPhaseWithContext( phase, options ) {
|
|
107
|
+
const storeContent = Storage.load();
|
|
108
|
+
if ( !storeContent ) { // means this was called from a unit test
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
const { parentId, traceId, traceHelm } = storeContent;
|
|
112
|
+
addEventPhase( phase, { ...options, parentId, traceId, traceHelm } );
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* The public namespace for tracing
|
|
117
|
+
*
|
|
118
|
+
* @namespace
|
|
119
|
+
*/
|
|
120
|
+
export const Tracing = {
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Adds the start phase of a new event at the default trace for the current workflow.
|
|
124
|
+
*
|
|
125
|
+
* @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
126
|
+
* @param {string} kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
|
|
127
|
+
* @param {string} name - The human friendly name of the Event: eg: query, request, create.
|
|
128
|
+
* @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
|
|
129
|
+
* @returns {void}
|
|
130
|
+
*/
|
|
131
|
+
addEventStart: ( { id, kind, name, details } ) => addEventPhaseWithContext( 'start', { kind, name, details, id } ),
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Adds the end phase at an event at the default trace for the current workflow.
|
|
135
|
+
*
|
|
136
|
+
* It needs to use the same id of the start phase.
|
|
137
|
+
*
|
|
138
|
+
* @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
139
|
+
* @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
|
|
140
|
+
* @returns {void}
|
|
141
|
+
*/
|
|
142
|
+
addEventEnd: ( { id, details } ) => addEventPhaseWithContext( 'end', { id, details } ),
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Adds the error phase at an event as error at the default trace for the current workflow.
|
|
146
|
+
*
|
|
147
|
+
* It needs to use the same id of the start phase.
|
|
148
|
+
*
|
|
149
|
+
* @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
150
|
+
* @param {any} details - All details attached to this Event Phase. Eg: DB queried records, HTTP response body.
|
|
151
|
+
* @returns {void}
|
|
152
|
+
*/
|
|
153
|
+
addEventError: ( { id, details } ) => addEventPhaseWithContext( 'error', { id, details } )
|
|
154
|
+
};
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
3
|
+
import { tmpdir, EOL } from 'node:os';
|
|
4
|
+
import { join } from 'path';
|
|
5
|
+
|
|
6
|
+
const createTempDir = () => mkdtempSync( join( tmpdir(), 'output-sdk-trace-' ) );
|
|
7
|
+
|
|
8
|
+
// Use env var to enable tracing
|
|
9
|
+
|
|
10
|
+
vi.mock( '#async_storage', () => ( { Storage: { load: () => ( { parentId: undefined, traceId: 'trace-1', traceHelm: 'tests' } ) } } ) );
|
|
11
|
+
vi.mock( 'path', async importActual => {
|
|
12
|
+
const actual = await importActual();
|
|
13
|
+
return { ...actual, join: ( first, ...rest ) => actual.join( first ?? process.cwd(), ...rest ) };
|
|
14
|
+
} );
|
|
15
|
+
vi.mock( './tracer_tree.js', () => ( { buildLogTree: vi.fn() } ) );
|
|
16
|
+
|
|
17
|
+
describe( 'tracing private exports', () => {
|
|
18
|
+
beforeEach( () => {
|
|
19
|
+
vi.resetModules();
|
|
20
|
+
vi.clearAllMocks();
|
|
21
|
+
vi.useFakeTimers();
|
|
22
|
+
vi.setSystemTime( new Date( '2020-01-01T00:00:00.000Z' ) );
|
|
23
|
+
process.env.TRACING_ENABLED = '1';
|
|
24
|
+
} );
|
|
25
|
+
|
|
26
|
+
afterEach( () => {
|
|
27
|
+
vi.useRealTimers();
|
|
28
|
+
delete process.env.TRACING_ENABLED;
|
|
29
|
+
} );
|
|
30
|
+
|
|
31
|
+
it( 'addEventStart (private) writes start', async () => {
|
|
32
|
+
const originalArgv2 = process.argv[2];
|
|
33
|
+
const tmp = createTempDir();
|
|
34
|
+
process.argv[2] = tmp;
|
|
35
|
+
|
|
36
|
+
const { addEventStart } = await import( './index.js?v=private1' );
|
|
37
|
+
addEventStart( { id: 'a', kind: 'evaluator', name: 'start', details: { foo: 1 }, traceId: 'trace-1', traceHelm: 'tests' } );
|
|
38
|
+
|
|
39
|
+
const { buildLogTree } = await import( './tracer_tree.js' );
|
|
40
|
+
const logPath = buildLogTree.mock.calls[0][0];
|
|
41
|
+
const raw = readFileSync( logPath, 'utf-8' );
|
|
42
|
+
const entry = JSON.parse( raw.split( EOL )[0] );
|
|
43
|
+
expect( entry.phase ).toBe( 'start' );
|
|
44
|
+
|
|
45
|
+
rmSync( tmp, { recursive: true, force: true } );
|
|
46
|
+
process.argv[2] = originalArgv2;
|
|
47
|
+
} );
|
|
48
|
+
|
|
49
|
+
it( 'addEventEnd (private) writes end', async () => {
|
|
50
|
+
const originalArgv2 = process.argv[2];
|
|
51
|
+
const tmp = createTempDir();
|
|
52
|
+
process.argv[2] = tmp;
|
|
53
|
+
|
|
54
|
+
const { addEventEnd } = await import( './index.js?v=private2' );
|
|
55
|
+
addEventEnd( { id: 'a', details: { ok: true }, traceId: 'trace-1', traceHelm: 'tests' } );
|
|
56
|
+
|
|
57
|
+
const { buildLogTree } = await import( './tracer_tree.js' );
|
|
58
|
+
const logPath = buildLogTree.mock.calls[0][0];
|
|
59
|
+
const raw = readFileSync( logPath, 'utf-8' );
|
|
60
|
+
const entry = JSON.parse( raw.split( EOL )[0] );
|
|
61
|
+
expect( entry.phase ).toBe( 'end' );
|
|
62
|
+
|
|
63
|
+
rmSync( tmp, { recursive: true, force: true } );
|
|
64
|
+
process.argv[2] = originalArgv2;
|
|
65
|
+
} );
|
|
66
|
+
|
|
67
|
+
it( 'addEventError (private) writes error', async () => {
|
|
68
|
+
const originalArgv2 = process.argv[2];
|
|
69
|
+
const tmp = createTempDir();
|
|
70
|
+
process.argv[2] = tmp;
|
|
71
|
+
|
|
72
|
+
const { addEventError } = await import( './index.js?v=private3' );
|
|
73
|
+
addEventError( { id: 'a', details: new Error( 'oops' ), traceId: 'trace-1', traceHelm: 'tests' } );
|
|
74
|
+
|
|
75
|
+
const { buildLogTree } = await import( './tracer_tree.js' );
|
|
76
|
+
const logPath = buildLogTree.mock.calls[0][0];
|
|
77
|
+
const raw = readFileSync( logPath, 'utf-8' );
|
|
78
|
+
const entry = JSON.parse( raw.split( EOL )[0] );
|
|
79
|
+
expect( entry.phase ).toBe( 'error' );
|
|
80
|
+
|
|
81
|
+
rmSync( tmp, { recursive: true, force: true } );
|
|
82
|
+
process.argv[2] = originalArgv2;
|
|
83
|
+
} );
|
|
84
|
+
} );
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
3
|
+
import { tmpdir, EOL } from 'node:os';
|
|
4
|
+
import { join } from 'path';
|
|
5
|
+
|
|
6
|
+
const createTempDir = () => mkdtempSync( join( tmpdir(), 'output-sdk-trace-' ) );
|
|
7
|
+
|
|
8
|
+
// Async storage mock to drive parent ids
|
|
9
|
+
|
|
10
|
+
const mockStorageData = { parentId: undefined, traceId: 'trace-1', traceHelm: 'tests' };
|
|
11
|
+
vi.mock( '#async_storage', () => ( { Storage: { load: () => mockStorageData } } ) );
|
|
12
|
+
|
|
13
|
+
vi.mock( './tracer_tree.js', () => ( { buildLogTree: vi.fn() } ) );
|
|
14
|
+
|
|
15
|
+
describe( 'Tracing (public namespace)', () => {
|
|
16
|
+
beforeEach( () => {
|
|
17
|
+
vi.resetModules();
|
|
18
|
+
vi.clearAllMocks();
|
|
19
|
+
vi.useFakeTimers();
|
|
20
|
+
vi.setSystemTime( new Date( '2020-01-01T00:00:00.000Z' ) );
|
|
21
|
+
process.env.TRACING_ENABLED = '1';
|
|
22
|
+
} );
|
|
23
|
+
|
|
24
|
+
afterEach( () => {
|
|
25
|
+
vi.useRealTimers();
|
|
26
|
+
delete process.env.TRACING_ENABLED;
|
|
27
|
+
} );
|
|
28
|
+
|
|
29
|
+
it( 'addEventStart writes a start entry', async () => {
|
|
30
|
+
const originalArgv2 = process.argv[2];
|
|
31
|
+
const tmp = createTempDir();
|
|
32
|
+
process.argv[2] = tmp;
|
|
33
|
+
|
|
34
|
+
const { Tracing } = await import( './index.js' );
|
|
35
|
+
Tracing.addEventStart( { id: '1', kind: 'evaluator', name: 'start', details: { a: 1 } } );
|
|
36
|
+
|
|
37
|
+
const { buildLogTree } = await import( './tracer_tree.js' );
|
|
38
|
+
const logPath = buildLogTree.mock.calls[0][0];
|
|
39
|
+
const raw = readFileSync( logPath, 'utf-8' );
|
|
40
|
+
const firstLine = raw.split( EOL )[0];
|
|
41
|
+
const entry = JSON.parse( firstLine );
|
|
42
|
+
expect( entry ).toMatchObject( { phase: 'start', kind: 'evaluator', name: 'start', id: '1', details: { a: 1 } } );
|
|
43
|
+
|
|
44
|
+
rmSync( tmp, { recursive: true, force: true } );
|
|
45
|
+
process.argv[2] = originalArgv2;
|
|
46
|
+
} );
|
|
47
|
+
|
|
48
|
+
it( 'addEventEnd writes an end entry', async () => {
|
|
49
|
+
const originalArgv2 = process.argv[2];
|
|
50
|
+
const tmp = createTempDir();
|
|
51
|
+
process.argv[2] = tmp;
|
|
52
|
+
|
|
53
|
+
const { Tracing } = await import( './index.js' );
|
|
54
|
+
Tracing.addEventEnd( { id: '1', details: { ok: true } } );
|
|
55
|
+
|
|
56
|
+
const { buildLogTree } = await import( './tracer_tree.js' );
|
|
57
|
+
const logPath = buildLogTree.mock.calls[0][0];
|
|
58
|
+
const raw = readFileSync( logPath, 'utf-8' );
|
|
59
|
+
const firstLine = raw.split( EOL )[0];
|
|
60
|
+
const entry = JSON.parse( firstLine );
|
|
61
|
+
expect( entry ).toMatchObject( { phase: 'end', id: '1', details: { ok: true } } );
|
|
62
|
+
|
|
63
|
+
rmSync( tmp, { recursive: true, force: true } );
|
|
64
|
+
process.argv[2] = originalArgv2;
|
|
65
|
+
} );
|
|
66
|
+
|
|
67
|
+
it( 'addEventError writes an error entry', async () => {
|
|
68
|
+
const originalArgv2 = process.argv[2];
|
|
69
|
+
const tmp = createTempDir();
|
|
70
|
+
process.argv[2] = tmp;
|
|
71
|
+
|
|
72
|
+
const { Tracing } = await import( './index.js' );
|
|
73
|
+
const error = new Error( 'boom' );
|
|
74
|
+
Tracing.addEventError( { id: '1', details: error } );
|
|
75
|
+
|
|
76
|
+
const { buildLogTree } = await import( './tracer_tree.js' );
|
|
77
|
+
const logPath = buildLogTree.mock.calls[0][0];
|
|
78
|
+
const raw = readFileSync( logPath, 'utf-8' );
|
|
79
|
+
const firstLine = raw.split( EOL )[0];
|
|
80
|
+
const entry = JSON.parse( firstLine );
|
|
81
|
+
expect( entry.phase ).toBe( 'error' );
|
|
82
|
+
|
|
83
|
+
rmSync( tmp, { recursive: true, force: true } );
|
|
84
|
+
process.argv[2] = originalArgv2;
|
|
85
|
+
} );
|
|
86
|
+
} );
|