@output.ai/core 0.0.16 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +16 -22
  2. package/package.json +19 -7
  3. package/src/consts.js +2 -11
  4. package/src/interface/evaluator.js +8 -4
  5. package/src/interface/step.js +1 -1
  6. package/src/interface/webhook.js +16 -4
  7. package/src/interface/workflow.js +28 -48
  8. package/src/internal_activities/index.js +3 -37
  9. package/src/tracing/index.d.ts +47 -0
  10. package/src/tracing/index.js +45 -0
  11. package/src/tracing/internal_interface.js +66 -0
  12. package/src/tracing/processors/local/index.js +50 -0
  13. package/src/tracing/processors/local/index.spec.js +67 -0
  14. package/src/tracing/processors/s3/index.js +51 -0
  15. package/src/tracing/processors/s3/index.spec.js +64 -0
  16. package/src/tracing/processors/s3/redis_client.js +19 -0
  17. package/src/tracing/processors/s3/redis_client.spec.js +50 -0
  18. package/src/tracing/processors/s3/s3_client.js +33 -0
  19. package/src/tracing/processors/s3/s3_client.spec.js +67 -0
  20. package/src/tracing/tools/build_trace_tree.js +76 -0
  21. package/src/tracing/tools/build_trace_tree.spec.js +99 -0
  22. package/src/tracing/tools/utils.js +28 -0
  23. package/src/tracing/tools/utils.spec.js +14 -0
  24. package/src/tracing/trace_engine.js +63 -0
  25. package/src/tracing/trace_engine.spec.js +91 -0
  26. package/src/utils.js +8 -0
  27. package/src/worker/catalog_workflow/index.js +2 -1
  28. package/src/worker/catalog_workflow/index.spec.js +6 -10
  29. package/src/worker/configs.js +24 -0
  30. package/src/worker/index.js +7 -8
  31. package/src/worker/interceptors/activity.js +15 -17
  32. package/src/worker/interceptors/workflow.js +18 -1
  33. package/src/worker/loader.js +40 -31
  34. package/src/worker/loader.spec.js +22 -29
  35. package/src/worker/loader_tools.js +63 -0
  36. package/src/worker/loader_tools.spec.js +85 -0
  37. package/src/worker/sinks.js +60 -10
  38. package/src/configs.js +0 -36
  39. package/src/configs.spec.js +0 -379
  40. package/src/worker/internal_utils.js +0 -60
  41. package/src/worker/internal_utils.spec.js +0 -134
  42. package/src/worker/tracer/index.js +0 -75
  43. package/src/worker/tracer/index.test.js +0 -102
  44. package/src/worker/tracer/tracer_tree.js +0 -85
  45. package/src/worker/tracer/tracer_tree.test.js +0 -115
  46. /package/src/{worker/async_storage.js → async_storage.js} +0 -0
package/README.md CHANGED
@@ -23,26 +23,18 @@ Think that workflows is the orchestrator and steps are executors. So the workflo
23
23
  ### workflow.js
24
24
 
25
25
  ```js
26
- import { workflow } from '@output.ai/workflow';
26
+ import { workflow, z } from '@output.ai/workflow';
27
27
  import { guessByName } from './steps.js';
28
28
 
29
29
  export default workflow( {
30
30
  name: 'guessMyProfession',
31
31
  description: 'Guess a person profession by its name',
32
- inputSchema: {
33
- type: 'object',
34
- required: [ 'name' ],
35
- properties: {
36
- name: { type: 'string'}
37
- }
38
- },
39
- outputSchema: {
40
- type: 'object',
41
- required: [ 'profession' ],
42
- properties: {
43
- profession: { type: 'string'}
44
- }
45
- },
32
+ inputSchema: z.object( {
33
+ name: z.string()
34
+ } ),
35
+ outputSchema: z.object( {
36
+ profession: z.string()
37
+ } ),
46
38
  fn: async input => {
47
39
  const profession = await guessByName( input.name );
48
40
  return { profession };
@@ -57,12 +49,8 @@ import { api } from './api.js'
57
49
 
58
50
  export const guessByName = step( {
59
51
  name: 'guessByName',
60
- inputSchema: {
61
- type: 'string'
62
- },
63
- outputSchema: {
64
- type: 'string'
65
- },
52
+ inputSchema: z.string(),
53
+ outputSchema: z.string(),
66
54
  fn: async name => {
67
55
  const res = await api.consumer( name );
68
56
  return res.body;
@@ -138,4 +126,10 @@ Necessary env variables to run the worker locally:
138
126
  - `TEMPORAL_API_KEY`: The API key to access remote temporal. If using local temporal, leave it blank;
139
127
  - `CATALOG_ID`: The name of the local catalog, always set this. Use your email;
140
128
  - `API_AUTH_KEY`: The API key to access the Framework API. Local can be blank, remote use the proper API Key;
141
- - `TRACING_ENABLED`: A "stringbool" value indicating if traces should be generated or not;
129
+ - `TRACE_LOCAL_ON`: A "stringbool" value indicating if traces should be saved locally, needs REDIS_URL;
130
+ - `TRACE_REMOTE_ON`: A "stringbool" value indicating if traces should be saved remotely, needs REDIS_URL and AWS_* secrets;
131
+ - `REDIS_URL`: The redis address to connect. Only necessary when any type of trace is enabled;
132
+ - `TRACE_REMOTE_S3_BUCKET`: The AWS S3 bucket to send the traces. Only necessary when remote trace is enabled;
133
+ - `AWS_REGION`: AWS region to connect to send the traces, must match the bucket region. Only necessary when remote trace is enabled;
134
+ - `AWS_ACCESS_KEY_ID`: AWS key id. Only necessary when remote trace is enabled;
135
+ - `AWS_SECRET_ACCESS_KEY`: AWS secrete. Only necessary when remote trace is enabled;
package/package.json CHANGED
@@ -1,10 +1,18 @@
1
1
  {
2
2
  "name": "@output.ai/core",
3
- "version": "0.0.16",
3
+ "version": "0.1.1",
4
4
  "description": "The core module of the output framework",
5
5
  "type": "module",
6
- "main": "src/index.js",
7
- "types": "src/index.d.ts",
6
+ "exports": {
7
+ ".": {
8
+ "types": "./src/index.d.ts",
9
+ "import": "./src/index.js"
10
+ },
11
+ "./tracing": {
12
+ "types": "./src/tracing/index.d.ts",
13
+ "import": "./src/tracing/index.js"
14
+ }
15
+ },
8
16
  "files": [
9
17
  "./src",
10
18
  "./bin"
@@ -17,19 +25,23 @@
17
25
  "worker": "node ./src/worker/index.js"
18
26
  },
19
27
  "dependencies": {
28
+ "@aws-sdk/client-s3": "3.913.0",
20
29
  "@babel/generator": "7.28.3",
21
30
  "@babel/parser": "7.28.4",
22
31
  "@babel/traverse": "7.28.4",
23
32
  "@babel/types": "7.28.4",
24
- "@temporalio/worker": "1.13.0",
25
- "@temporalio/workflow": "1.13.0",
26
- "zod": "4.1.9"
33
+ "@temporalio/worker": "1.13.1",
34
+ "@temporalio/workflow": "1.13.1",
35
+ "redis": "5.8.3",
36
+ "zod": "4.1.12"
27
37
  },
28
38
  "license": "UNLICENSED",
29
39
  "imports": {
30
40
  "#consts": "./src/consts.js",
31
- "#configs": "./src/configs.js",
32
41
  "#errors": "./src/errors.js",
42
+ "#utils": "./src/utils.js",
43
+ "#tracing": "./src/tracing/internal_interface.js",
44
+ "#async_storage": "./src/async_storage.js",
33
45
  "#internal_activities": "./src/internal_activities/index.js"
34
46
  }
35
47
  }
package/src/consts.js CHANGED
@@ -1,16 +1,7 @@
1
- export const SEND_WEBHOOK_ACTIVITY_NAME = '__internal#sendWebhookPost';
2
- export const READ_TRACE_FILE = '__internal#readTraceFile';
1
+ export const ACTIVITY_SEND_WEBHOOK = '__internal#sendWebhook';
3
2
  export const METADATA_ACCESS_SYMBOL = Symbol( '__metadata' );
4
3
  export const WORKFLOWS_INDEX_FILENAME = '__workflows_entrypoint.js';
5
- export const THIS_LIB_NAME = 'core';
6
- export const TraceEvent = {
7
- WORKFLOW_START: 'workflow_start',
8
- WORKFLOW_END: 'workflow_end',
9
- STEP_START: 'step_start',
10
- STEP_END: 'step_end',
11
- EVALUATOR_START: 'evaluator_start',
12
- EVALUATOR_END: 'evaluator_end'
13
- };
4
+ export const WORKFLOW_CATALOG = '$catalog';
14
5
  export const ComponentType = {
15
6
  EVALUATOR: 'evaluator',
16
7
  INTERNAL_STEP: 'internal_step',
@@ -48,7 +48,8 @@ export class EvaluationResult {
48
48
  * @param {string} [args.reasoning] - The reasoning behind the result
49
49
  */
50
50
  constructor( args ) {
51
- if ( !EvaluationResult.#schema.safeParse( args ) ) {
51
+ const result = EvaluationResult.#schema.safeParse( args );
52
+ if ( result.error ) {
52
53
  throw new EvaluationResultValidationError( z.prettifyError( result.error ) );
53
54
  }
54
55
  this.value = args.value;
@@ -73,7 +74,8 @@ export class EvaluationStringResult extends EvaluationResult {
73
74
  * @param {string} [args.reasoning] - The reasoning behind the result
74
75
  */
75
76
  constructor( args ) {
76
- if ( !EvaluationStringResult.#valueSchema.safeParse( args.value ) ) {
77
+ const result = EvaluationStringResult.#valueSchema.safeParse( args.value );
78
+ if ( result.error ) {
77
79
  throw new EvaluationResultValidationError( z.prettifyError( result.error ) );
78
80
  }
79
81
  super( args );
@@ -96,7 +98,8 @@ export class EvaluationBooleanResult extends EvaluationResult {
96
98
  * @param {string} [args.reasoning] - The reasoning behind the result
97
99
  */
98
100
  constructor( args ) {
99
- if ( !EvaluationBooleanResult.#valueSchema.safeParse( args.value ) ) {
101
+ const result = EvaluationBooleanResult.#valueSchema.safeParse( args.value );
102
+ if ( result.error ) {
100
103
  throw new EvaluationResultValidationError( z.prettifyError( result.error ) );
101
104
  }
102
105
  super( args );
@@ -119,7 +122,8 @@ export class EvaluationNumberResult extends EvaluationResult {
119
122
  * @param {string} [args.reasoning] - The reasoning behind the result
120
123
  */
121
124
  constructor( args ) {
122
- if ( !EvaluationNumberResult.#valueSchema.safeParse( args.value ) ) {
125
+ const result = EvaluationNumberResult.#valueSchema.safeParse( args.value );
126
+ if ( result.error ) {
123
127
  throw new EvaluationResultValidationError( z.prettifyError( result.error ) );
124
128
  }
125
129
  super( args );
@@ -16,6 +16,6 @@ export function step( { name, description, inputSchema, outputSchema, fn } ) {
16
16
  return output;
17
17
  };
18
18
 
19
- setMetadata( wrapper, { name, description, inputSchema, outputSchema, type: ComponentType.EVALUATOR } );
19
+ setMetadata( wrapper, { name, description, inputSchema, outputSchema, type: ComponentType.STEP } );
20
20
  return wrapper;
21
21
  };
@@ -1,18 +1,30 @@
1
1
  // THIS RUNS IN THE TEMPORAL'S SANDBOX ENVIRONMENT
2
- import { defineSignal, setHandler, proxyActivities, workflowInfo } from '@temporalio/workflow';
3
- import { SEND_WEBHOOK_ACTIVITY_NAME } from '#consts';
2
+ import { defineSignal, setHandler, proxyActivities, workflowInfo, proxySinks } from '@temporalio/workflow';
3
+ import { ACTIVITY_SEND_WEBHOOK } from '#consts';
4
+ import { FatalError } from '#errors';
4
5
  import { validateCreateWebhook } from './validations/static.js';
5
6
 
6
7
  export async function createWebhook( { url, payload } ) {
7
8
  validateCreateWebhook( { url, payload } );
8
- const workflowId = workflowInfo();
9
+ const { workflowId } = workflowInfo();
9
10
 
10
- await proxyActivities( temporalActivityConfigs )[SEND_WEBHOOK_ACTIVITY_NAME]( { url, workflowId, payload } );
11
+ await proxyActivities( {
12
+ startToCloseTimeout: '3m',
13
+ retry: {
14
+ initialInterval: '15s',
15
+ maximumAttempts: 5,
16
+ nonRetryableErrorTypes: [ FatalError.name ]
17
+ }
18
+ } )[ACTIVITY_SEND_WEBHOOK]( { url, workflowId, payload } );
11
19
 
20
+ const sinks = await proxySinks();
12
21
  const resumeSignal = defineSignal( 'resume' );
13
22
 
23
+ const traceId = `${workflowId}-${url}-${Date.now()}`;
24
+ sinks.trace.addEventStart( { id: traceId, name: 'resume', kind: 'webhook' } );
14
25
  return new Promise( resolve =>
15
26
  setHandler( resumeSignal, responsePayload => {
27
+ sinks.trace.addEventEnd( { id: traceId, details: responsePayload } );
16
28
  resolve( responsePayload );
17
29
  } )
18
30
  );
@@ -1,11 +1,10 @@
1
1
  // THIS RUNS IN THE TEMPORAL'S SANDBOX ENVIRONMENT
2
- import { proxyActivities, inWorkflowContext, executeChild, workflowInfo, ApplicationFailure, proxySinks } from '@temporalio/workflow';
2
+ import { proxyActivities, inWorkflowContext, executeChild, workflowInfo } from '@temporalio/workflow';
3
3
  import { getInvocationDir } from './utils.js';
4
4
  import { setMetadata } from './metadata.js';
5
5
  import { FatalError, ValidationError } from '#errors';
6
6
  import { validateWorkflow } from './validations/static.js';
7
7
  import { validateWithSchema } from './validations/runtime.js';
8
- import { READ_TRACE_FILE, TraceEvent } from '#consts';
9
8
 
10
9
  const temporalActivityConfigs = {
11
10
  startToCloseTimeout: '20 minute',
@@ -23,62 +22,43 @@ export function workflow( { name, description, inputSchema, outputSchema, fn } )
23
22
  const workflowPath = getInvocationDir();
24
23
 
25
24
  const steps = proxyActivities( temporalActivityConfigs );
26
- const sinks = proxySinks();
27
25
 
28
26
  const wrapper = async input => {
29
- try {
30
- if ( inWorkflowContext() ) {
31
- sinks.log.trace( { event: TraceEvent.WORKFLOW_START, input } );
32
- }
33
-
34
- validateWithSchema( inputSchema, input, `Workflow ${name} input` );
35
-
36
- // this returns a plain function, for example, in unit tests
37
- if ( !inWorkflowContext() ) {
38
- const output = await fn( input );
39
- validateWithSchema( outputSchema, output, `Workflow ${name} output` );
40
- return output;
41
- }
42
-
43
- const { memo, workflowId } = workflowInfo();
44
-
45
- Object.assign( workflowInfo().memo, { workflowPath } );
46
-
47
- // binds the methods called in the code that Webpack loader will add, they will exposed via "this"
48
- const output = await fn.call( {
49
- invokeStep: async ( stepName, input ) => steps[`${workflowPath}#${stepName}`]( input ),
50
- invokeEvaluator: async ( evaluatorName, input ) => steps[`${workflowPath}#${evaluatorName}`]( input ),
27
+ validateWithSchema( inputSchema, input, `Workflow ${name} input` );
51
28
 
52
- startWorkflow: async ( childName, input ) => {
29
+ // this returns a plain function, for example, in unit tests
30
+ if ( !inWorkflowContext() ) {
31
+ const output = await fn( input );
32
+ validateWithSchema( outputSchema, output, `Workflow ${name} output` );
33
+ return output;
34
+ }
53
35
 
54
- // Checks if current memo has rootWorkflowId, which means current execution is already a child
55
- // Then it sets the memory for the child execution passing along who's the original workflow is and its type
56
- const workflowMemory = memo.rootWorkflowId ?
57
- { parentWorkflowId: workflowId, rootWorkflowType: memo.rootWorkflowType, rootWorkflowId: memo.rootWorkflowId } :
58
- { parentWorkflowId: workflowId, rootWorkflowId: workflowId, rootWorkflowType: name };
36
+ const { workflowId, memo, startTime } = workflowInfo();
59
37
 
60
- return executeChild( childName, { args: input ? [ input ] : [], memo: workflowMemory } );
61
- }
62
- }, input );
38
+ // Create the execution context object or preserve if it already exists:
39
+ // It will always contains the information about the root workflow
40
+ // It will be used to as context for tracing (connecting events)
41
+ const executionContext = memo.executionContext ?? {
42
+ workflowId,
43
+ workflowName: name,
44
+ startTime: startTime.getTime()
45
+ };
63
46
 
64
- validateWithSchema( outputSchema, output, `Workflow ${name} output` );
47
+ Object.assign( memo, { executionContext } );
65
48
 
66
- sinks.log.trace( { event: TraceEvent.WORKFLOW_END, output } );
49
+ // binds the methods called in the code that Webpack loader will add, they will exposed via "this"
50
+ const output = await fn.call( {
51
+ invokeStep: async ( stepName, input ) => steps[`${workflowPath}#${stepName}`]( input ),
52
+ invokeEvaluator: async ( evaluatorName, input ) => steps[`${workflowPath}#${evaluatorName}`]( input ),
67
53
 
68
- // add trace if not child
69
- if ( !memo.rootWorkflowId ) {
70
- const trace = await steps[READ_TRACE_FILE]( { workflowType: name, workflowId } );
71
- return { output, trace };
54
+ startWorkflow: async ( childName, input ) => {
55
+ return executeChild( childName, { args: input ? [ input ] : [], memo: { executionContext, parentId: workflowId } } );
72
56
  }
57
+ }, input );
73
58
 
74
- return output;
75
- } catch ( error ) {
76
- /*
77
- * Any errors in the workflow will interrupt its execution since the workflow is designed to orchestrate and
78
- * IOs should be made in steps
79
- */
80
- throw new ApplicationFailure( error.message, error.constructor.name );
81
- }
59
+ validateWithSchema( outputSchema, output, `Workflow ${name} output` );
60
+
61
+ return output;
82
62
  };
83
63
 
84
64
  setMetadata( wrapper, { name, description, inputSchema, outputSchema } );
@@ -1,11 +1,7 @@
1
1
  import { FatalError } from '#errors';
2
- import { existsSync, readFileSync, readdirSync } from 'node:fs';
3
- import { join } from 'node:path';
4
2
  import { setMetadata } from '../interface/metadata.js';
5
3
  import { ComponentType } from '#consts';
6
4
 
7
- const callerDir = process.argv[2];
8
-
9
5
  /**
10
6
  * Send a post to a given URL
11
7
  *
@@ -15,7 +11,7 @@ const callerDir = process.argv[2];
15
11
  * @param {any} options.payload - The payload to send url
16
12
  * @throws {FatalError}
17
13
  */
18
- export const sendWebhookPost = async ( { url, workflowId, payload } ) => {
14
+ export const sendWebhook = async ( { url, workflowId, payload } ) => {
19
15
  const request = fetch( url, {
20
16
  method: 'POST',
21
17
  headers: {
@@ -33,41 +29,11 @@ export const sendWebhookPost = async ( { url, workflowId, payload } ) => {
33
29
  }
34
30
  } )();
35
31
 
36
- console.log( '[Core.SendWebhookPost]', res.status, res.statusText );
32
+ console.log( '[Core.SendWebhook]', res.status, res.statusText );
37
33
 
38
34
  if ( !res.ok ) {
39
35
  throw new FatalError( `Webhook fail: ${res.status}` );
40
36
  }
41
37
  };
42
38
 
43
- setMetadata( sendWebhookPost, { type: ComponentType.INTERNAL_STEP } );
44
-
45
- /**
46
- * Read the trace file of a given execution and returns the content
47
- *
48
- * @param {object} options
49
- * @param {string} options.workflowType - The type of the workflow
50
- * @param {string} options.workflowId - The workflow execution id
51
- * @returns {string[]} Each line of the trace file
52
- */
53
- export const readTraceFile = async ( { workflowType, workflowId } ) => {
54
- const dir = join( callerDir, 'logs', 'runs', workflowType );
55
-
56
- if ( !existsSync( dir ) ) {
57
- console.log( '[Core.ReadTraceFile]', 'Trace folder not found', dir );
58
- return [];
59
- }
60
-
61
- const suffix = `-${workflowId}.raw`;
62
- const matchingFile = readdirSync( dir ).find( f => f.endsWith( suffix ) );
63
-
64
- if ( !matchingFile ) {
65
- console.log( '[Core.ReadTraceFile]', 'Trace file not found', dir, suffix );
66
- return [];
67
- }
68
-
69
- const file = join( dir, matchingFile );
70
- return existsSync( file ) ? readFileSync( file, 'utf-8' ).split( '\n' ) : [];
71
- };
72
-
73
- setMetadata( readTraceFile, { type: ComponentType.INTERNAL_STEP, skipTrace: true } );
39
+ setMetadata( sendWebhook, { type: ComponentType.INTERNAL_STEP } );
@@ -0,0 +1,47 @@
1
+ /*
2
+ ╭───────────╮
3
+ │ T R A C E │╮
4
+ ╰───────────╯│
5
+ ╰───────────╯
6
+ */
7
+
8
+ /**
9
+ * The public namespace for tracing
10
+ *
11
+ * @namespace
12
+ */
13
+ export declare const Tracing: {
14
+
15
+ /**
16
+ * Adds the start phase of a new event at the default trace for the current workflow.
17
+ *
18
+ * @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
19
+ * @param {string} kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
20
+ * @param {string} name - The human friendly name of the Event: query, request, create.
21
+ * @param {any} details - All details attached to this Event Phase. DB queried records, HTTP response body.
22
+ * @returns {void}
23
+ */
24
+ addEventStart( args: { id: string; kind: string; name: string; details: any } ): void; // eslint-disable-line @typescript-eslint/no-explicit-any
25
+
26
+ /**
27
+ * Adds the end phase at an event at the default trace for the current workflow.
28
+ *
29
+ * It needs to use the same id of the start phase.
30
+ *
31
+ * @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
32
+ * @param {any} details - All details attached to this Event Phase. DB queried records, HTTP response body.
33
+ * @returns {void}
34
+ */
35
+ addEventEnd( args: { id: string; details: any } ): void; // eslint-disable-line @typescript-eslint/no-explicit-any
36
+
37
+ /**
38
+ * Adds the error phase at an event as error at the default trace for the current workflow.
39
+ *
40
+ * It needs to use the same id of the start phase.
41
+ *
42
+ * @param {string} id - A unique id for the Event, must be the same across all phases: start, end, error.
43
+ * @param {any} details - All details attached to this Event Phase. DB queried records, HTTP response body.
44
+ * @returns {void}
45
+ */
46
+ addEventError( args: { id: string; details: any } ): void; // eslint-disable-line @typescript-eslint/no-explicit-any
47
+ };
@@ -0,0 +1,45 @@
1
+ import { addEventPhaseWithContext } from './trace_engine.js';
2
+
3
+ /**
4
+ * The public namespace for tracing
5
+ *
6
+ * @namespace
7
+ */
8
+ export const Tracing = {
9
+
10
+ /**
11
+ * Adds the start phase of a new event at the default trace for the current workflow.
12
+ *
13
+ * @param {object} args
14
+ * @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
15
+ * @param {string} args.kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
16
+ * @param {string} args.name - The human friendly name of the Event: query, request, create.
17
+ * @param {object} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
18
+ * @returns {void}
19
+ */
20
+ addEventStart: ( { id, kind, name, details } ) => addEventPhaseWithContext( 'start', { kind, name, details, id } ),
21
+
22
+ /**
23
+ * Adds the end phase at an event at the default trace for the current workflow.
24
+ *
25
+ * It needs to use the same id of the start phase.
26
+ *
27
+ * @param {object} args
28
+ * @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
29
+ * @param {object} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
30
+ * @returns {void}
31
+ */
32
+ addEventEnd: ( { id, details } ) => addEventPhaseWithContext( 'end', { id, details } ),
33
+
34
+ /**
35
+ * Adds the error phase at an event as error at the default trace for the current workflow.
36
+ *
37
+ * It needs to use the same id of the start phase.
38
+ *
39
+ * @param {object} args
40
+ * @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
41
+ * @param {object} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
42
+ * @returns {void}
43
+ */
44
+ addEventError: ( { id, details } ) => addEventPhaseWithContext( 'error', { id, details } )
45
+ };
@@ -0,0 +1,66 @@
1
+ import { addEventPhase, init } from './trace_engine.js';
2
+
3
+ /**
4
+ * Init method, if not called, no processors are attached and trace functions are dummy
5
+ */
6
+ export { init };
7
+
8
+ /**
9
+ * Trace nomenclature
10
+ *
11
+ * Trace - The collection of Events;
12
+ * Event - Any entry in the Trace file, must have the two phases START and END or ERROR;
13
+ * Phase - An specific part of an Event, either START or the conclusive END or ERROR;
14
+ */
15
+
16
+ /**
17
+ * Internal use only
18
+ *
19
+ * Adds the start phase of a new event at the default trace for the current workflow.
20
+ *
21
+ * @param {object} args
22
+ * @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
23
+ * @param {string} args.kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
24
+ * @param {string} args.name - The human friendly name of the Event: query, request, create.
25
+ * @param {any} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
26
+ * @param {string} args.parentId - The parent Event, used to build a three.
27
+ * @param {object} args.executionContext - The original execution context from the workflow
28
+ * @returns {void}
29
+ */
30
+ export const addEventStart = options => addEventPhase( 'start', options );
31
+
32
+ /**
33
+ * Internal use only
34
+ *
35
+ * Adds the end phase at an event at the default trace for the current workflow.
36
+ *
37
+ * It needs to use the same id of the start phase.
38
+ *
39
+ * @param {object} args
40
+ * @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
41
+ * @param {string} args.kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
42
+ * @param {string} args.name - The human friendly name of the Event: query, request, create.
43
+ * @param {any} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
44
+ * @param {string} args.parentId - The parent Event, used to build a three.
45
+ * @param {object} args.executionContext - The original execution context from the workflow
46
+ * @returns {void}
47
+ */
48
+ export const addEventEnd = options => addEventPhase( 'end', options );
49
+
50
+ /**
51
+ * Internal use only
52
+ *
53
+ * Adds the error phase at an event as error at the default trace for the current workflow.
54
+ *
55
+ * It needs to use the same id of the start phase.
56
+ *
57
+ * @param {object} args
58
+ * @param {string} args.id - A unique id for the Event, must be the same across all phases: start, end, error.
59
+ * @param {string} args.kind - The kind of Event, like HTTP, DiskWrite, DBOp, etc.
60
+ * @param {string} args.name - The human friendly name of the Event: query, request, create.
61
+ * @param {any} args.details - All details attached to this Event Phase. DB queried records, HTTP response body.
62
+ * @param {string} args.parentId - The parent Event, used to build a three.
63
+ * @param {object} args.executionContext - The original execution context from the workflow
64
+ * @returns {void}
65
+ */
66
+ export const addEventError = options => addEventPhase( 'error', options );
@@ -0,0 +1,50 @@
1
+ import { appendFileSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
2
+ import { dirname, join } from 'node:path';
3
+ import { fileURLToPath } from 'url';
4
+ import buildTraceTree from '../../tools/build_trace_tree.js';
5
+ import { EOL } from 'node:os';
6
+
7
+ const oneWeekInMS = 1000 * 60 * 60 * 24 * 7;
8
+ const __dirname = dirname( fileURLToPath( import.meta.url ) );
9
+ const tempDir = join( __dirname, 'temp', 'traces' );
10
+
11
+ const accumulate = ( { entry, executionContext: { workflowId, startTime } } ) => {
12
+ const path = join( tempDir, `${startTime}_${workflowId}.trace` );
13
+ appendFileSync( path, JSON.stringify( entry ) + EOL, 'utf-8' );
14
+ return readFileSync( path, 'utf-8' ).split( EOL ).slice( 0, -1 ).map( v => JSON.parse( v ) );
15
+ };
16
+
17
+ const cleanupOldTempFiles = ( threshold = Date.now() - oneWeekInMS ) =>
18
+ readdirSync( tempDir )
19
+ .filter( f => +f.split( '_' )[0] < threshold )
20
+ .forEach( f => rmSync( join( tempDir, f ) ) );
21
+
22
+ /**
23
+ * Init this processor
24
+ */
25
+ export const init = () => {
26
+ mkdirSync( tempDir, { recursive: true } );
27
+ cleanupOldTempFiles();
28
+ };
29
+
30
+ /**
31
+ * Execute this processor:
32
+ *
33
+ * Persist a trace tree file to local file system, updating upon each new entry
34
+ *
35
+ * @param {object} args
36
+ * @param {object} entry - Trace event phase
37
+ * @param {object} executionContext - Execution info: workflowId, workflowName, startTime
38
+ * @returns
39
+ */
40
+ export const exec = ( { entry, executionContext } ) => {
41
+ const { workflowId, workflowName, startTime } = executionContext;
42
+ const content = buildTraceTree( accumulate( { entry, executionContext } ) );
43
+
44
+ const timestamp = new Date( startTime ).toISOString().replace( /[:T.]/g, '-' );
45
+ const dir = join( process.argv[2], 'logs', 'runs', workflowName );
46
+ const path = join( dir, `${timestamp}_${workflowId}.json` );
47
+
48
+ mkdirSync( dir, { recursive: true } );
49
+ writeFileSync( path, JSON.stringify( content, undefined, 2 ) + EOL, 'utf-8' );
50
+ };
@@ -0,0 +1,67 @@
1
+ import { describe, it, expect, vi, beforeEach } from 'vitest';
2
+
3
+ // In-memory fs mock store
4
+ const store = { files: new Map() };
5
+ const mkdirSyncMock = vi.fn();
6
+ const writeFileSyncMock = vi.fn();
7
+ const appendFileSyncMock = vi.fn( ( path, data ) => {
8
+ const prev = store.files.get( path ) ?? '';
9
+ store.files.set( path, prev + data );
10
+ } );
11
+ const readFileSyncMock = vi.fn( path => store.files.get( path ) ?? '' );
12
+ const readdirSyncMock = vi.fn( () => [] );
13
+ const rmSyncMock = vi.fn();
14
+
15
+ vi.mock( 'node:fs', () => ( {
16
+ mkdirSync: mkdirSyncMock,
17
+ writeFileSync: writeFileSyncMock,
18
+ appendFileSync: appendFileSyncMock,
19
+ readFileSync: readFileSyncMock,
20
+ readdirSync: readdirSyncMock,
21
+ rmSync: rmSyncMock
22
+ } ) );
23
+
24
+ const buildTraceTreeMock = vi.fn( entries => ( { count: entries.length } ) );
25
+ vi.mock( '../../tools/build_trace_tree.js', () => ( { default: buildTraceTreeMock } ) );
26
+
27
+ describe( 'tracing/processors/local', () => {
28
+ beforeEach( () => {
29
+ vi.clearAllMocks();
30
+ store.files.clear();
31
+ process.argv[2] = '/tmp/project';
32
+ } );
33
+
34
+ it( 'init(): creates temp dir and cleans up old files', async () => {
35
+ const { init } = await import( './index.js' );
36
+
37
+ const now = Date.now();
38
+ readdirSyncMock.mockReturnValue( [ `${now - ( 8 * 24 * 60 * 60 * 1000 )}_old.trace`, `${now}_new.trace` ] );
39
+
40
+ init();
41
+
42
+ expect( mkdirSyncMock ).toHaveBeenCalledWith( expect.stringMatching( /temp\/traces$/ ), { recursive: true } );
43
+ expect( rmSyncMock ).toHaveBeenCalledTimes( 1 );
44
+ } );
45
+
46
+ it( 'exec(): accumulates entries and writes aggregated tree', async () => {
47
+ const { exec, init } = await import( './index.js' );
48
+ init();
49
+
50
+ const startTime = Date.parse( '2020-01-02T03:04:05.678Z' );
51
+ const ctx = { executionContext: { workflowId: 'id1', workflowName: 'WF', startTime } };
52
+
53
+ exec( { ...ctx, entry: { name: 'A', phase: 'start', timestamp: startTime } } );
54
+ exec( { ...ctx, entry: { name: 'A', phase: 'tick', timestamp: startTime + 1 } } );
55
+ exec( { ...ctx, entry: { name: 'A', phase: 'end', timestamp: startTime + 2 } } );
56
+
57
+ // buildTraceTree called with 1, 2, 3 entries respectively
58
+ expect( buildTraceTreeMock ).toHaveBeenCalledTimes( 3 );
59
+ expect( buildTraceTreeMock.mock.calls.at( -1 )[0].length ).toBe( 3 );
60
+
61
+ expect( writeFileSyncMock ).toHaveBeenCalledTimes( 3 );
62
+ const [ writtenPath, content ] = writeFileSyncMock.mock.calls.at( -1 );
63
+ expect( writtenPath ).toMatch( /\/logs\/runs\/WF\// );
64
+ expect( JSON.parse( content.trim() ).count ).toBe( 3 );
65
+ } );
66
+ } );
67
+