@output.ai/core 0.0.16 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -22
- package/package.json +19 -7
- package/src/consts.js +2 -11
- package/src/interface/evaluator.js +8 -4
- package/src/interface/step.js +1 -1
- package/src/interface/webhook.js +16 -4
- package/src/interface/workflow.js +28 -48
- package/src/internal_activities/index.js +3 -37
- package/src/tracing/index.d.ts +47 -0
- package/src/tracing/index.js +45 -0
- package/src/tracing/internal_interface.js +66 -0
- package/src/tracing/processors/local/index.js +50 -0
- package/src/tracing/processors/local/index.spec.js +67 -0
- package/src/tracing/processors/s3/index.js +51 -0
- package/src/tracing/processors/s3/index.spec.js +64 -0
- package/src/tracing/processors/s3/redis_client.js +19 -0
- package/src/tracing/processors/s3/redis_client.spec.js +50 -0
- package/src/tracing/processors/s3/s3_client.js +33 -0
- package/src/tracing/processors/s3/s3_client.spec.js +67 -0
- package/src/tracing/tools/build_trace_tree.js +76 -0
- package/src/tracing/tools/build_trace_tree.spec.js +99 -0
- package/src/tracing/tools/utils.js +28 -0
- package/src/tracing/tools/utils.spec.js +14 -0
- package/src/tracing/trace_engine.js +63 -0
- package/src/tracing/trace_engine.spec.js +91 -0
- package/src/utils.js +8 -0
- package/src/worker/catalog_workflow/index.js +2 -1
- package/src/worker/catalog_workflow/index.spec.js +6 -10
- package/src/worker/configs.js +24 -0
- package/src/worker/index.js +7 -8
- package/src/worker/interceptors/activity.js +15 -17
- package/src/worker/interceptors/workflow.js +18 -1
- package/src/worker/loader.js +40 -31
- package/src/worker/loader.spec.js +22 -29
- package/src/worker/loader_tools.js +63 -0
- package/src/worker/loader_tools.spec.js +85 -0
- package/src/worker/sinks.js +60 -10
- package/src/configs.js +0 -36
- package/src/configs.spec.js +0 -379
- package/src/worker/internal_utils.js +0 -60
- package/src/worker/internal_utils.spec.js +0 -134
- package/src/worker/tracer/index.js +0 -75
- package/src/worker/tracer/index.test.js +0 -102
- package/src/worker/tracer/tracer_tree.js +0 -85
- package/src/worker/tracer/tracer_tree.test.js +0 -115
- /package/src/{worker/async_storage.js → async_storage.js} +0 -0
package/README.md
CHANGED
|
@@ -23,26 +23,18 @@ Think that workflows is the orchestrator and steps are executors. So the workflo
|
|
|
23
23
|
### workflow.js
|
|
24
24
|
|
|
25
25
|
```js
|
|
26
|
-
import { workflow } from '@output.ai/workflow';
|
|
26
|
+
import { workflow, z } from '@output.ai/workflow';
|
|
27
27
|
import { guessByName } from './steps.js';
|
|
28
28
|
|
|
29
29
|
export default workflow( {
|
|
30
30
|
name: 'guessMyProfession',
|
|
31
31
|
description: 'Guess a person profession by its name',
|
|
32
|
-
inputSchema: {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
},
|
|
39
|
-
outputSchema: {
|
|
40
|
-
type: 'object',
|
|
41
|
-
required: [ 'profession' ],
|
|
42
|
-
properties: {
|
|
43
|
-
profession: { type: 'string'}
|
|
44
|
-
}
|
|
45
|
-
},
|
|
32
|
+
inputSchema: z.object( {
|
|
33
|
+
name: z.string()
|
|
34
|
+
} ),
|
|
35
|
+
outputSchema: z.object( {
|
|
36
|
+
profession: z.string()
|
|
37
|
+
} ),
|
|
46
38
|
fn: async input => {
|
|
47
39
|
const profession = await guessByName( input.name );
|
|
48
40
|
return { profession };
|
|
@@ -57,12 +49,8 @@ import { api } from './api.js'
|
|
|
57
49
|
|
|
58
50
|
export const guessByName = step( {
|
|
59
51
|
name: 'guessByName',
|
|
60
|
-
inputSchema:
|
|
61
|
-
|
|
62
|
-
},
|
|
63
|
-
outputSchema: {
|
|
64
|
-
type: 'string'
|
|
65
|
-
},
|
|
52
|
+
inputSchema: z.string(),
|
|
53
|
+
outputSchema: z.string(),
|
|
66
54
|
fn: async name => {
|
|
67
55
|
const res = await api.consumer( name );
|
|
68
56
|
return res.body;
|
|
@@ -138,4 +126,10 @@ Necessary env variables to run the worker locally:
|
|
|
138
126
|
- `TEMPORAL_API_KEY`: The API key to access remote temporal. If using local temporal, leave it blank;
|
|
139
127
|
- `CATALOG_ID`: The name of the local catalog, always set this. Use your email;
|
|
140
128
|
- `API_AUTH_KEY`: The API key to access the Framework API. Local can be blank, remote use the proper API Key;
|
|
141
|
-
- `
|
|
129
|
+
- `TRACE_LOCAL_ON`: A "stringbool" value indicating if traces should be saved locally, needs REDIS_URL;
|
|
130
|
+
- `TRACE_REMOTE_ON`: A "stringbool" value indicating if traces should be saved remotely, needs REDIS_URL and AWS_* secrets;
|
|
131
|
+
- `REDIS_URL`: The redis address to connect. Only necessary when any type of trace is enabled;
|
|
132
|
+
- `TRACE_REMOTE_S3_BUCKET`: The AWS S3 bucket to send the traces. Only necessary when remote trace is enabled;
|
|
133
|
+
- `AWS_REGION`: AWS region to connect to send the traces, must match the bucket region. Only necessary when remote trace is enabled;
|
|
134
|
+
- `AWS_ACCESS_KEY_ID`: AWS key id. Only necessary when remote trace is enabled;
|
|
135
|
+
- `AWS_SECRET_ACCESS_KEY`: AWS secrete. Only necessary when remote trace is enabled;
|
package/package.json
CHANGED
|
@@ -1,10 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@output.ai/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"description": "The core module of the output framework",
|
|
5
5
|
"type": "module",
|
|
6
|
-
"
|
|
7
|
-
|
|
6
|
+
"exports": {
|
|
7
|
+
".": {
|
|
8
|
+
"types": "./src/index.d.ts",
|
|
9
|
+
"import": "./src/index.js"
|
|
10
|
+
},
|
|
11
|
+
"./tracing": {
|
|
12
|
+
"types": "./src/tracing/index.d.ts",
|
|
13
|
+
"import": "./src/tracing/index.js"
|
|
14
|
+
}
|
|
15
|
+
},
|
|
8
16
|
"files": [
|
|
9
17
|
"./src",
|
|
10
18
|
"./bin"
|
|
@@ -17,19 +25,23 @@
|
|
|
17
25
|
"worker": "node ./src/worker/index.js"
|
|
18
26
|
},
|
|
19
27
|
"dependencies": {
|
|
28
|
+
"@aws-sdk/client-s3": "3.913.0",
|
|
20
29
|
"@babel/generator": "7.28.3",
|
|
21
30
|
"@babel/parser": "7.28.4",
|
|
22
31
|
"@babel/traverse": "7.28.4",
|
|
23
32
|
"@babel/types": "7.28.4",
|
|
24
|
-
"@temporalio/worker": "1.13.
|
|
25
|
-
"@temporalio/workflow": "1.13.
|
|
26
|
-
"
|
|
33
|
+
"@temporalio/worker": "1.13.1",
|
|
34
|
+
"@temporalio/workflow": "1.13.1",
|
|
35
|
+
"redis": "5.8.3",
|
|
36
|
+
"zod": "4.1.12"
|
|
27
37
|
},
|
|
28
38
|
"license": "UNLICENSED",
|
|
29
39
|
"imports": {
|
|
30
40
|
"#consts": "./src/consts.js",
|
|
31
|
-
"#configs": "./src/configs.js",
|
|
32
41
|
"#errors": "./src/errors.js",
|
|
42
|
+
"#utils": "./src/utils.js",
|
|
43
|
+
"#tracing": "./src/tracing/internal_interface.js",
|
|
44
|
+
"#async_storage": "./src/async_storage.js",
|
|
33
45
|
"#internal_activities": "./src/internal_activities/index.js"
|
|
34
46
|
}
|
|
35
47
|
}
|
package/src/consts.js
CHANGED
|
@@ -1,16 +1,7 @@
|
|
|
1
|
-
export const
|
|
2
|
-
export const READ_TRACE_FILE = '__internal#readTraceFile';
|
|
1
|
+
export const ACTIVITY_SEND_WEBHOOK = '__internal#sendWebhook';
|
|
3
2
|
export const METADATA_ACCESS_SYMBOL = Symbol( '__metadata' );
|
|
4
3
|
export const WORKFLOWS_INDEX_FILENAME = '__workflows_entrypoint.js';
|
|
5
|
-
export const
|
|
6
|
-
export const TraceEvent = {
|
|
7
|
-
WORKFLOW_START: 'workflow_start',
|
|
8
|
-
WORKFLOW_END: 'workflow_end',
|
|
9
|
-
STEP_START: 'step_start',
|
|
10
|
-
STEP_END: 'step_end',
|
|
11
|
-
EVALUATOR_START: 'evaluator_start',
|
|
12
|
-
EVALUATOR_END: 'evaluator_end'
|
|
13
|
-
};
|
|
4
|
+
export const WORKFLOW_CATALOG = '$catalog';
|
|
14
5
|
export const ComponentType = {
|
|
15
6
|
EVALUATOR: 'evaluator',
|
|
16
7
|
INTERNAL_STEP: 'internal_step',
|
|
@@ -48,7 +48,8 @@ export class EvaluationResult {
|
|
|
48
48
|
* @param {string} [args.reasoning] - The reasoning behind the result
|
|
49
49
|
*/
|
|
50
50
|
constructor( args ) {
|
|
51
|
-
|
|
51
|
+
const result = EvaluationResult.#schema.safeParse( args );
|
|
52
|
+
if ( result.error ) {
|
|
52
53
|
throw new EvaluationResultValidationError( z.prettifyError( result.error ) );
|
|
53
54
|
}
|
|
54
55
|
this.value = args.value;
|
|
@@ -73,7 +74,8 @@ export class EvaluationStringResult extends EvaluationResult {
|
|
|
73
74
|
* @param {string} [args.reasoning] - The reasoning behind the result
|
|
74
75
|
*/
|
|
75
76
|
constructor( args ) {
|
|
76
|
-
|
|
77
|
+
const result = EvaluationStringResult.#valueSchema.safeParse( args.value );
|
|
78
|
+
if ( result.error ) {
|
|
77
79
|
throw new EvaluationResultValidationError( z.prettifyError( result.error ) );
|
|
78
80
|
}
|
|
79
81
|
super( args );
|
|
@@ -96,7 +98,8 @@ export class EvaluationBooleanResult extends EvaluationResult {
|
|
|
96
98
|
* @param {string} [args.reasoning] - The reasoning behind the result
|
|
97
99
|
*/
|
|
98
100
|
constructor( args ) {
|
|
99
|
-
|
|
101
|
+
const result = EvaluationBooleanResult.#valueSchema.safeParse( args.value );
|
|
102
|
+
if ( result.error ) {
|
|
100
103
|
throw new EvaluationResultValidationError( z.prettifyError( result.error ) );
|
|
101
104
|
}
|
|
102
105
|
super( args );
|
|
@@ -119,7 +122,8 @@ export class EvaluationNumberResult extends EvaluationResult {
|
|
|
119
122
|
* @param {string} [args.reasoning] - The reasoning behind the result
|
|
120
123
|
*/
|
|
121
124
|
constructor( args ) {
|
|
122
|
-
|
|
125
|
+
const result = EvaluationNumberResult.#valueSchema.safeParse( args.value );
|
|
126
|
+
if ( result.error ) {
|
|
123
127
|
throw new EvaluationResultValidationError( z.prettifyError( result.error ) );
|
|
124
128
|
}
|
|
125
129
|
super( args );
|
package/src/interface/step.js
CHANGED
|
@@ -16,6 +16,6 @@ export function step( { name, description, inputSchema, outputSchema, fn } ) {
|
|
|
16
16
|
return output;
|
|
17
17
|
};
|
|
18
18
|
|
|
19
|
-
setMetadata( wrapper, { name, description, inputSchema, outputSchema, type: ComponentType.
|
|
19
|
+
setMetadata( wrapper, { name, description, inputSchema, outputSchema, type: ComponentType.STEP } );
|
|
20
20
|
return wrapper;
|
|
21
21
|
};
|
package/src/interface/webhook.js
CHANGED
|
@@ -1,18 +1,30 @@
|
|
|
1
1
|
// THIS RUNS IN THE TEMPORAL'S SANDBOX ENVIRONMENT
|
|
2
|
-
import { defineSignal, setHandler, proxyActivities, workflowInfo } from '@temporalio/workflow';
|
|
3
|
-
import {
|
|
2
|
+
import { defineSignal, setHandler, proxyActivities, workflowInfo, proxySinks } from '@temporalio/workflow';
|
|
3
|
+
import { ACTIVITY_SEND_WEBHOOK } from '#consts';
|
|
4
|
+
import { FatalError } from '#errors';
|
|
4
5
|
import { validateCreateWebhook } from './validations/static.js';
|
|
5
6
|
|
|
6
7
|
export async function createWebhook( { url, payload } ) {
|
|
7
8
|
validateCreateWebhook( { url, payload } );
|
|
8
|
-
const workflowId = workflowInfo();
|
|
9
|
+
const { workflowId } = workflowInfo();
|
|
9
10
|
|
|
10
|
-
await proxyActivities(
|
|
11
|
+
await proxyActivities( {
|
|
12
|
+
startToCloseTimeout: '3m',
|
|
13
|
+
retry: {
|
|
14
|
+
initialInterval: '15s',
|
|
15
|
+
maximumAttempts: 5,
|
|
16
|
+
nonRetryableErrorTypes: [ FatalError.name ]
|
|
17
|
+
}
|
|
18
|
+
} )[ACTIVITY_SEND_WEBHOOK]( { url, workflowId, payload } );
|
|
11
19
|
|
|
20
|
+
const sinks = await proxySinks();
|
|
12
21
|
const resumeSignal = defineSignal( 'resume' );
|
|
13
22
|
|
|
23
|
+
const traceId = `${workflowId}-${url}-${Date.now()}`;
|
|
24
|
+
sinks.trace.addEventStart( { id: traceId, name: 'resume', kind: 'webhook' } );
|
|
14
25
|
return new Promise( resolve =>
|
|
15
26
|
setHandler( resumeSignal, responsePayload => {
|
|
27
|
+
sinks.trace.addEventEnd( { id: traceId, details: responsePayload } );
|
|
16
28
|
resolve( responsePayload );
|
|
17
29
|
} )
|
|
18
30
|
);
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
// THIS RUNS IN THE TEMPORAL'S SANDBOX ENVIRONMENT
|
|
2
|
-
import { proxyActivities, inWorkflowContext, executeChild, workflowInfo
|
|
2
|
+
import { proxyActivities, inWorkflowContext, executeChild, workflowInfo } from '@temporalio/workflow';
|
|
3
3
|
import { getInvocationDir } from './utils.js';
|
|
4
4
|
import { setMetadata } from './metadata.js';
|
|
5
5
|
import { FatalError, ValidationError } from '#errors';
|
|
6
6
|
import { validateWorkflow } from './validations/static.js';
|
|
7
7
|
import { validateWithSchema } from './validations/runtime.js';
|
|
8
|
-
import { READ_TRACE_FILE, TraceEvent } from '#consts';
|
|
9
8
|
|
|
10
9
|
const temporalActivityConfigs = {
|
|
11
10
|
startToCloseTimeout: '20 minute',
|
|
@@ -23,62 +22,43 @@ export function workflow( { name, description, inputSchema, outputSchema, fn } )
|
|
|
23
22
|
const workflowPath = getInvocationDir();
|
|
24
23
|
|
|
25
24
|
const steps = proxyActivities( temporalActivityConfigs );
|
|
26
|
-
const sinks = proxySinks();
|
|
27
25
|
|
|
28
26
|
const wrapper = async input => {
|
|
29
|
-
|
|
30
|
-
if ( inWorkflowContext() ) {
|
|
31
|
-
sinks.log.trace( { event: TraceEvent.WORKFLOW_START, input } );
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
validateWithSchema( inputSchema, input, `Workflow ${name} input` );
|
|
35
|
-
|
|
36
|
-
// this returns a plain function, for example, in unit tests
|
|
37
|
-
if ( !inWorkflowContext() ) {
|
|
38
|
-
const output = await fn( input );
|
|
39
|
-
validateWithSchema( outputSchema, output, `Workflow ${name} output` );
|
|
40
|
-
return output;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
const { memo, workflowId } = workflowInfo();
|
|
44
|
-
|
|
45
|
-
Object.assign( workflowInfo().memo, { workflowPath } );
|
|
46
|
-
|
|
47
|
-
// binds the methods called in the code that Webpack loader will add, they will exposed via "this"
|
|
48
|
-
const output = await fn.call( {
|
|
49
|
-
invokeStep: async ( stepName, input ) => steps[`${workflowPath}#${stepName}`]( input ),
|
|
50
|
-
invokeEvaluator: async ( evaluatorName, input ) => steps[`${workflowPath}#${evaluatorName}`]( input ),
|
|
27
|
+
validateWithSchema( inputSchema, input, `Workflow ${name} input` );
|
|
51
28
|
|
|
52
|
-
|
|
29
|
+
// this returns a plain function, for example, in unit tests
|
|
30
|
+
if ( !inWorkflowContext() ) {
|
|
31
|
+
const output = await fn( input );
|
|
32
|
+
validateWithSchema( outputSchema, output, `Workflow ${name} output` );
|
|
33
|
+
return output;
|
|
34
|
+
}
|
|
53
35
|
|
|
54
|
-
|
|
55
|
-
// Then it sets the memory for the child execution passing along who's the original workflow is and its type
|
|
56
|
-
const workflowMemory = memo.rootWorkflowId ?
|
|
57
|
-
{ parentWorkflowId: workflowId, rootWorkflowType: memo.rootWorkflowType, rootWorkflowId: memo.rootWorkflowId } :
|
|
58
|
-
{ parentWorkflowId: workflowId, rootWorkflowId: workflowId, rootWorkflowType: name };
|
|
36
|
+
const { workflowId, memo, startTime } = workflowInfo();
|
|
59
37
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
38
|
+
// Create the execution context object or preserve if it already exists:
|
|
39
|
+
// It will always contains the information about the root workflow
|
|
40
|
+
// It will be used to as context for tracing (connecting events)
|
|
41
|
+
const executionContext = memo.executionContext ?? {
|
|
42
|
+
workflowId,
|
|
43
|
+
workflowName: name,
|
|
44
|
+
startTime: startTime.getTime()
|
|
45
|
+
};
|
|
63
46
|
|
|
64
|
-
|
|
47
|
+
Object.assign( memo, { executionContext } );
|
|
65
48
|
|
|
66
|
-
|
|
49
|
+
// binds the methods called in the code that Webpack loader will add, they will exposed via "this"
|
|
50
|
+
const output = await fn.call( {
|
|
51
|
+
invokeStep: async ( stepName, input ) => steps[`${workflowPath}#${stepName}`]( input ),
|
|
52
|
+
invokeEvaluator: async ( evaluatorName, input ) => steps[`${workflowPath}#${evaluatorName}`]( input ),
|
|
67
53
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
const trace = await steps[READ_TRACE_FILE]( { workflowType: name, workflowId } );
|
|
71
|
-
return { output, trace };
|
|
54
|
+
startWorkflow: async ( childName, input ) => {
|
|
55
|
+
return executeChild( childName, { args: input ? [ input ] : [], memo: { executionContext, parentId: workflowId } } );
|
|
72
56
|
}
|
|
57
|
+
}, input );
|
|
73
58
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
* Any errors in the workflow will interrupt its execution since the workflow is designed to orchestrate and
|
|
78
|
-
* IOs should be made in steps
|
|
79
|
-
*/
|
|
80
|
-
throw new ApplicationFailure( error.message, error.constructor.name );
|
|
81
|
-
}
|
|
59
|
+
validateWithSchema( outputSchema, output, `Workflow ${name} output` );
|
|
60
|
+
|
|
61
|
+
return output;
|
|
82
62
|
};
|
|
83
63
|
|
|
84
64
|
setMetadata( wrapper, { name, description, inputSchema, outputSchema } );
|
|
@@ -1,11 +1,7 @@
|
|
|
1
1
|
import { FatalError } from '#errors';
|
|
2
|
-
import { existsSync, readFileSync, readdirSync } from 'node:fs';
|
|
3
|
-
import { join } from 'node:path';
|
|
4
2
|
import { setMetadata } from '../interface/metadata.js';
|
|
5
3
|
import { ComponentType } from '#consts';
|
|
6
4
|
|
|
7
|
-
const callerDir = process.argv[2];
|
|
8
|
-
|
|
9
5
|
/**
|
|
10
6
|
* Send a post to a given URL
|
|
11
7
|
*
|
|
@@ -15,7 +11,7 @@ const callerDir = process.argv[2];
|
|
|
15
11
|
* @param {any} options.payload - The payload to send url
|
|
16
12
|
* @throws {FatalError}
|
|
17
13
|
*/
|
|
18
|
-
export const
|
|
14
|
+
export const sendWebhook = async ( { url, workflowId, payload } ) => {
|
|
19
15
|
const request = fetch( url, {
|
|
20
16
|
method: 'POST',
|
|
21
17
|
headers: {
|
|
@@ -33,41 +29,11 @@ export const sendWebhookPost = async ( { url, workflowId, payload } ) => {
|
|
|
33
29
|
}
|
|
34
30
|
} )();
|
|
35
31
|
|
|
36
|
-
console.log( '[Core.
|
|
32
|
+
console.log( '[Core.SendWebhook]', res.status, res.statusText );
|
|
37
33
|
|
|
38
34
|
if ( !res.ok ) {
|
|
39
35
|
throw new FatalError( `Webhook fail: ${res.status}` );
|
|
40
36
|
}
|
|
41
37
|
};
|
|
42
38
|
|
|
43
|
-
setMetadata(
|
|
44
|
-
|
|
45
|
-
/**
|
|
46
|
-
* Read the trace file of a given execution and returns the content
|
|
47
|
-
*
|
|
48
|
-
* @param {object} options
|
|
49
|
-
* @param {string} options.workflowType - The type of the workflow
|
|
50
|
-
* @param {string} options.workflowId - The workflow execution id
|
|
51
|
-
* @returns {string[]} Each line of the trace file
|
|
52
|
-
*/
|
|
53
|
-
export const readTraceFile = async ( { workflowType, workflowId } ) => {
|
|
54
|
-
const dir = join( callerDir, 'logs', 'runs', workflowType );
|
|
55
|
-
|
|
56
|
-
if ( !existsSync( dir ) ) {
|
|
57
|
-
console.log( '[Core.ReadTraceFile]', 'Trace folder not found', dir );
|
|
58
|
-
return [];
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
const suffix = `-${workflowId}.raw`;
|
|
62
|
-
const matchingFile = readdirSync( dir ).find( f => f.endsWith( suffix ) );
|
|
63
|
-
|
|
64
|
-
if ( !matchingFile ) {
|
|
65
|
-
console.log( '[Core.ReadTraceFile]', 'Trace file not found', dir, suffix );
|
|
66
|
-
return [];
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
const file = join( dir, matchingFile );
|
|
70
|
-
return existsSync( file ) ? readFileSync( file, 'utf-8' ).split( '\n' ) : [];
|
|
71
|
-
};
|
|
72
|
-
|
|
73
|
-
setMetadata( readTraceFile, { type: ComponentType.INTERNAL_STEP, skipTrace: true } );
|
|
39
|
+
setMetadata( sendWebhook, { type: ComponentType.INTERNAL_STEP } );
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/*
|
|
2
|
+
╭───────────╮
|
|
3
|
+
│ T R A C E │╮
|
|
4
|
+
╰───────────╯│
|
|
5
|
+
╰───────────╯
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The public namespace for tracing
|
|
10
|
+
*
|
|
11
|
+
* @namespace
|
|
12
|
+
*/
|
|
13
|
+
export declare const Tracing: {
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Adds the start phase of a new event at the default trace for the current workflow.
|
|
17
|
+
*
|
|
18
|
+
* @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
19
|
+
* @param {string} kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
|
|
20
|
+
* @param {string} name - The human friendly name of the Event: query, request, create.
|
|
21
|
+
* @param {any} details - All details attached to this Event Phase. DB queried records, HTTP response body.
|
|
22
|
+
* @returns {void}
|
|
23
|
+
*/
|
|
24
|
+
addEventStart( args: { id: string; kind: string; name: string; details: any } ): void; // eslint-disable-line @typescript-eslint/no-explicit-any
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Adds the end phase at an event at the default trace for the current workflow.
|
|
28
|
+
*
|
|
29
|
+
* It needs to use the same id of the start phase.
|
|
30
|
+
*
|
|
31
|
+
* @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
32
|
+
* @param {any} details - All details attached to this Event Phase. DB queried records, HTTP response body.
|
|
33
|
+
* @returns {void}
|
|
34
|
+
*/
|
|
35
|
+
addEventEnd( args: { id: string; details: any } ): void; // eslint-disable-line @typescript-eslint/no-explicit-any
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Adds the error phase at an event as error at the default trace for the current workflow.
|
|
39
|
+
*
|
|
40
|
+
* It needs to use the same id of the start phase.
|
|
41
|
+
*
|
|
42
|
+
* @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
43
|
+
* @param {any} details - All details attached to this Event Phase. DB queried records, HTTP response body.
|
|
44
|
+
* @returns {void}
|
|
45
|
+
*/
|
|
46
|
+
addEventError( args: { id: string; details: any } ): void; // eslint-disable-line @typescript-eslint/no-explicit-any
|
|
47
|
+
};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { addEventPhaseWithContext } from './trace_engine.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The public namespace for tracing
|
|
5
|
+
*
|
|
6
|
+
* @namespace
|
|
7
|
+
*/
|
|
8
|
+
export const Tracing = {
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Adds the start phase of a new event at the default trace for the current workflow.
|
|
12
|
+
*
|
|
13
|
+
* @param {object} args
|
|
14
|
+
* @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
15
|
+
* @param {string} args.kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
|
|
16
|
+
* @param {string} args.name - The human friendly name of the Event: query, request, create.
|
|
17
|
+
* @param {object} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
|
|
18
|
+
* @returns {void}
|
|
19
|
+
*/
|
|
20
|
+
addEventStart: ( { id, kind, name, details } ) => addEventPhaseWithContext( 'start', { kind, name, details, id } ),
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Adds the end phase at an event at the default trace for the current workflow.
|
|
24
|
+
*
|
|
25
|
+
* It needs to use the same id of the start phase.
|
|
26
|
+
*
|
|
27
|
+
* @param {object} args
|
|
28
|
+
* @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
29
|
+
* @param {object} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
|
|
30
|
+
* @returns {void}
|
|
31
|
+
*/
|
|
32
|
+
addEventEnd: ( { id, details } ) => addEventPhaseWithContext( 'end', { id, details } ),
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Adds the error phase at an event as error at the default trace for the current workflow.
|
|
36
|
+
*
|
|
37
|
+
* It needs to use the same id of the start phase.
|
|
38
|
+
*
|
|
39
|
+
* @param {object} args
|
|
40
|
+
* @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
41
|
+
* @param {object} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
|
|
42
|
+
* @returns {void}
|
|
43
|
+
*/
|
|
44
|
+
addEventError: ( { id, details } ) => addEventPhaseWithContext( 'error', { id, details } )
|
|
45
|
+
};
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { addEventPhase, init } from './trace_engine.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Init method, if not called, no processors are attached and trace functions are dummy
|
|
5
|
+
*/
|
|
6
|
+
export { init };
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Trace nomenclature
|
|
10
|
+
*
|
|
11
|
+
* Trace - The collection of Events;
|
|
12
|
+
* Event - Any entry in the Trace file, must have the two phases START and END or ERROR;
|
|
13
|
+
* Phase - An specific part of an Event, either START or the conclusive END or ERROR;
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Internal use only
|
|
18
|
+
*
|
|
19
|
+
* Adds the start phase of a new event at the default trace for the current workflow.
|
|
20
|
+
*
|
|
21
|
+
* @param {object} args
|
|
22
|
+
* @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
23
|
+
* @param {string} args.kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
|
|
24
|
+
* @param {string} args.name - The human friendly name of the Event: query, request, create.
|
|
25
|
+
* @param {any} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
|
|
26
|
+
* @param {string} args.parentId - The parent Event, used to build a three.
|
|
27
|
+
* @param {object} args.executionContext - The original execution context from the workflow
|
|
28
|
+
* @returns {void}
|
|
29
|
+
*/
|
|
30
|
+
export const addEventStart = options => addEventPhase( 'start', options );
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Internal use only
|
|
34
|
+
*
|
|
35
|
+
* Adds the end phase at an event at the default trace for the current workflow.
|
|
36
|
+
*
|
|
37
|
+
* It needs to use the same id of the start phase.
|
|
38
|
+
*
|
|
39
|
+
* @param {object} args
|
|
40
|
+
* @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
41
|
+
* @param {string} args.kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
|
|
42
|
+
* @param {string} args.name - The human friendly name of the Event: query, request, create.
|
|
43
|
+
* @param {any} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
|
|
44
|
+
* @param {string} args.parentId - The parent Event, used to build a three.
|
|
45
|
+
* @param {object} args.executionContext - The original execution context from the workflow
|
|
46
|
+
* @returns {void}
|
|
47
|
+
*/
|
|
48
|
+
export const addEventEnd = options => addEventPhase( 'end', options );
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Internal use only
|
|
52
|
+
*
|
|
53
|
+
* Adds the error phase at an event as error at the default trace for the current workflow.
|
|
54
|
+
*
|
|
55
|
+
* It needs to use the same id of the start phase.
|
|
56
|
+
*
|
|
57
|
+
* @param {object} args
|
|
58
|
+
* @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
|
|
59
|
+
* @param {string} args.kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
|
|
60
|
+
* @param {string} args.name - The human friendly name of the Event: query, request, create.
|
|
61
|
+
* @param {any} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
|
|
62
|
+
* @param {string} args.parentId - The parent Event, used to build a three.
|
|
63
|
+
* @param {object} args.executionContext - The original execution context from the workflow
|
|
64
|
+
* @returns {void}
|
|
65
|
+
*/
|
|
66
|
+
export const addEventError = options => addEventPhase( 'error', options );
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { appendFileSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
3
|
+
import { fileURLToPath } from 'url';
|
|
4
|
+
import buildTraceTree from '../../tools/build_trace_tree.js';
|
|
5
|
+
import { EOL } from 'node:os';
|
|
6
|
+
|
|
7
|
+
const oneWeekInMS = 1000 * 60 * 60 * 24 * 7;
|
|
8
|
+
const __dirname = dirname( fileURLToPath( import.meta.url ) );
|
|
9
|
+
const tempDir = join( __dirname, 'temp', 'traces' );
|
|
10
|
+
|
|
11
|
+
const accumulate = ( { entry, executionContext: { workflowId, startTime } } ) => {
|
|
12
|
+
const path = join( tempDir, `${startTime}_${workflowId}.trace` );
|
|
13
|
+
appendFileSync( path, JSON.stringify( entry ) + EOL, 'utf-8' );
|
|
14
|
+
return readFileSync( path, 'utf-8' ).split( EOL ).slice( 0, -1 ).map( v => JSON.parse( v ) );
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
const cleanupOldTempFiles = ( threshold = Date.now() - oneWeekInMS ) =>
|
|
18
|
+
readdirSync( tempDir )
|
|
19
|
+
.filter( f => +f.split( '_' )[0] < threshold )
|
|
20
|
+
.forEach( f => rmSync( join( tempDir, f ) ) );
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Init this processor
|
|
24
|
+
*/
|
|
25
|
+
export const init = () => {
|
|
26
|
+
mkdirSync( tempDir, { recursive: true } );
|
|
27
|
+
cleanupOldTempFiles();
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Execute this processor:
|
|
32
|
+
*
|
|
33
|
+
* Persist a trace tree file to local file system, updating upon each new entry
|
|
34
|
+
*
|
|
35
|
+
* @param {object} args
|
|
36
|
+
* @param {object} entry - Trace event phase
|
|
37
|
+
* @param {object} executionContext - Execution info: workflowId, workflowName, startTime
|
|
38
|
+
* @returns
|
|
39
|
+
*/
|
|
40
|
+
export const exec = ( { entry, executionContext } ) => {
|
|
41
|
+
const { workflowId, workflowName, startTime } = executionContext;
|
|
42
|
+
const content = buildTraceTree( accumulate( { entry, executionContext } ) );
|
|
43
|
+
|
|
44
|
+
const timestamp = new Date( startTime ).toISOString().replace( /[:T.]/g, '-' );
|
|
45
|
+
const dir = join( process.argv[2], 'logs', 'runs', workflowName );
|
|
46
|
+
const path = join( dir, `${timestamp}_${workflowId}.json` );
|
|
47
|
+
|
|
48
|
+
mkdirSync( dir, { recursive: true } );
|
|
49
|
+
writeFileSync( path, JSON.stringify( content, undefined, 2 ) + EOL, 'utf-8' );
|
|
50
|
+
};
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
2
|
+
|
|
3
|
+
// In-memory fs mock store
|
|
4
|
+
const store = { files: new Map() };
|
|
5
|
+
const mkdirSyncMock = vi.fn();
|
|
6
|
+
const writeFileSyncMock = vi.fn();
|
|
7
|
+
const appendFileSyncMock = vi.fn( ( path, data ) => {
|
|
8
|
+
const prev = store.files.get( path ) ?? '';
|
|
9
|
+
store.files.set( path, prev + data );
|
|
10
|
+
} );
|
|
11
|
+
const readFileSyncMock = vi.fn( path => store.files.get( path ) ?? '' );
|
|
12
|
+
const readdirSyncMock = vi.fn( () => [] );
|
|
13
|
+
const rmSyncMock = vi.fn();
|
|
14
|
+
|
|
15
|
+
vi.mock( 'node:fs', () => ( {
|
|
16
|
+
mkdirSync: mkdirSyncMock,
|
|
17
|
+
writeFileSync: writeFileSyncMock,
|
|
18
|
+
appendFileSync: appendFileSyncMock,
|
|
19
|
+
readFileSync: readFileSyncMock,
|
|
20
|
+
readdirSync: readdirSyncMock,
|
|
21
|
+
rmSync: rmSyncMock
|
|
22
|
+
} ) );
|
|
23
|
+
|
|
24
|
+
const buildTraceTreeMock = vi.fn( entries => ( { count: entries.length } ) );
|
|
25
|
+
vi.mock( '../../tools/build_trace_tree.js', () => ( { default: buildTraceTreeMock } ) );
|
|
26
|
+
|
|
27
|
+
describe( 'tracing/processors/local', () => {
|
|
28
|
+
beforeEach( () => {
|
|
29
|
+
vi.clearAllMocks();
|
|
30
|
+
store.files.clear();
|
|
31
|
+
process.argv[2] = '/tmp/project';
|
|
32
|
+
} );
|
|
33
|
+
|
|
34
|
+
it( 'init(): creates temp dir and cleans up old files', async () => {
|
|
35
|
+
const { init } = await import( './index.js' );
|
|
36
|
+
|
|
37
|
+
const now = Date.now();
|
|
38
|
+
readdirSyncMock.mockReturnValue( [ `${now - ( 8 * 24 * 60 * 60 * 1000 )}_old.trace`, `${now}_new.trace` ] );
|
|
39
|
+
|
|
40
|
+
init();
|
|
41
|
+
|
|
42
|
+
expect( mkdirSyncMock ).toHaveBeenCalledWith( expect.stringMatching( /temp\/traces$/ ), { recursive: true } );
|
|
43
|
+
expect( rmSyncMock ).toHaveBeenCalledTimes( 1 );
|
|
44
|
+
} );
|
|
45
|
+
|
|
46
|
+
it( 'exec(): accumulates entries and writes aggregated tree', async () => {
|
|
47
|
+
const { exec, init } = await import( './index.js' );
|
|
48
|
+
init();
|
|
49
|
+
|
|
50
|
+
const startTime = Date.parse( '2020-01-02T03:04:05.678Z' );
|
|
51
|
+
const ctx = { executionContext: { workflowId: 'id1', workflowName: 'WF', startTime } };
|
|
52
|
+
|
|
53
|
+
exec( { ...ctx, entry: { name: 'A', phase: 'start', timestamp: startTime } } );
|
|
54
|
+
exec( { ...ctx, entry: { name: 'A', phase: 'tick', timestamp: startTime + 1 } } );
|
|
55
|
+
exec( { ...ctx, entry: { name: 'A', phase: 'end', timestamp: startTime + 2 } } );
|
|
56
|
+
|
|
57
|
+
// buildTraceTree called with 1, 2, 3 entries respectively
|
|
58
|
+
expect( buildTraceTreeMock ).toHaveBeenCalledTimes( 3 );
|
|
59
|
+
expect( buildTraceTreeMock.mock.calls.at( -1 )[0].length ).toBe( 3 );
|
|
60
|
+
|
|
61
|
+
expect( writeFileSyncMock ).toHaveBeenCalledTimes( 3 );
|
|
62
|
+
const [ writtenPath, content ] = writeFileSyncMock.mock.calls.at( -1 );
|
|
63
|
+
expect( writtenPath ).toMatch( /\/logs\/runs\/WF\// );
|
|
64
|
+
expect( JSON.parse( content.trim() ).count ).toBe( 3 );
|
|
65
|
+
} );
|
|
66
|
+
} );
|
|
67
|
+
|