@output.ai/core 0.0.7

package/README.md

@@ -0,0 +1,104 @@
+# Core
+
+Provides tools to run a workflow, which is a well defined logical unit of work.
+
+## step()
+
+Defines a step, which is a enclosed unit of work which can import external dependencies.
+
+```ts
+import { step } from 'flow-core';
+
+export const aSingleStep = step( {
+  name: 'aSingleStep',
+  fn: async (): Promise<string> => {
+    // do stuff
+  }
+} );
+```
+
+All step needs a name, and the name will be used in the context of the workflow (see below).
+
+## workflow()
+
+Defines a logical structure of step invocations. No external dependencies can be imported in the file containing the workflow.
+
+```ts
+import { workflow } from 'flow-core';
+import type { WorkflowContext } from 'flow-core';
+import type { PromptWorkflowInput, PromptWorkflowOutput } from './types';
+
+export default workflow( {
+  name: 'prompt',
+  description: 'A workflow to demonstrate the prompt feature',
+  fn: async ( input: PromptWorkflowInput, context: WorkflowContext ): Promise<PromptWorkflowOutput> => {
+    const result = await context.steps.aSingleStep();
+
+    return { result };
+  }
+} );
+```
+
+The workflow Context has the following properties:
+
+### .steps.*()
+
+Each step defined in the step file is available here as js function with the "name" it was defined there. The input arguments are the input arguments of its `fn` function.
+
+### .tools.webhook()
+
+Provides a webhook feature where a POST is made to a given url and the workflow is stopped until another POST is made back to the workflows API:
+
+```js
+const result = await context.tools.webhook( {
+  name: 'needFeedback',
+  description: 'Get a feedback',
+  url: 'http://xxx.xxx/feedback',
+  payload: { /* a given payload */ }
+} );
+```
+
+the url will receive the following payload:
+
+```js
+{
+  workflowId: '', // alphanumerical id of the workflow execution,
+  payload: { }, // the payload sent using tools.webhook()
+}
+```
+
+To resume the workflow, a POST has to be made to a given url, with a response payload and the workflowId.
+
+The host is localhost:3001 if running locally, there is not remote host yet.
+
+The format is:
+```
+POST https://locahost:3001/workflow/feedback
+  {
+    workflowId,
+    payload: {}
+  }
+```
+
+## Folder structure:
+
+```
+└ workflows
+  └ my workflow
+    ├ index.ts // contains the workflow()
+    ├ types.ts // contains input, output types, and other misc types
+    └ steps.ts // contains all steps
+```
+
+## Booting
+
+The project with workflows will be a "worker", meaning its workflows can be invoked individually by their names. To do so it has to be started using a npx command.
+
+_package.json_
+```js
+...
+  "scripts": {
+    "start": "npx flow-worker"
+  },
+...
+```

package/bin/worker.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+invocation_dir="$(pwd)"
+
+# Follow symlinks to get the real script path
+if [ -L "${BASH_SOURCE[0]}" ]; then
+  real_script="$(readlink "${BASH_SOURCE[0]}")"
+  # If relative path, resolve from symlink directory
+  if [[ "$real_script" != /* ]]; then
+    real_script="$(dirname "${BASH_SOURCE[0]}")/$real_script"
+  fi
+else
+  real_script="${BASH_SOURCE[0]}"
+fi
+
+# Get the real script directory (should be node_modules/<pkg>/bin)
+script_dir="$(cd "$(dirname "$real_script")" && pwd)"
+
+# SDK dir is the parent (node_modules/flow-core)
+sdk_dir="$(dirname "$script_dir")"
+
+cd ${sdk_dir}
+
+exec npm run worker -- ${invocation_dir} "${@:2}"  # Pass remaining args

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@output.ai/core",
+  "version": "0.0.7",
+  "description": "The core module of the output framework",
+  "type": "module",
+  "main": "src/index.js",
+  "types": "src/index.d.ts",
+  "files": [
+    "./src",
+    "./bin"
+  ],
+  "scripts": {
+    "worker": "node ./src/worker/index.js"
+  },
+  "bin": {
+    "flow-worker": "./bin/worker.sh"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/growthxai/flow-sdk"
+  },
+  "dependencies": {
+    "@temporalio/worker": "1.13.0",
+    "@temporalio/workflow": "1.13.0",
+    "undici": "7.15.0"
+  },
+  "imports": {
+    "#consts": "./src/consts.js",
+    "#configs": "./src/configs.js",
+    "#internal_activities": "./src/internal_activities/index.js"
+  }
+}

package/src/configs.js

@@ -0,0 +1,17 @@
+export const worker = {
+  address: process.env.TEMPORAL_ADDRESS,
+  apiKey: process.env.TEMPORAL_API_KEY,
+  executionTimeout: '1m',
+  maxActivities: 100,
+  maxWorkflows: 100,
+  namespace: process.env.TEMPORAL_NAMESPACE ?? 'default',
+  taskQueue: process.env.TEMPORAL_TASK_QUEUE ?? 'main'
+};
+
+export const api = {
+  authKey: process.env.API_AUTH_KEY
+};
+
+export const tracing = {
+  enabled: [ '1', 'true' ].includes( process.env.TRACING_ENABLED )
+};

package/src/consts.js

@@ -0,0 +1,3 @@
+export const nameSymbol = Symbol( '__name' );
+export const sendWebhookPostName = '__internal#sendWebhookPost';
+export const workflowsIndexFileName = '__workflows_entrypoint.js';

package/src/errors.d.ts

@@ -0,0 +1,3 @@
+export declare class FatalError extends Error { }
+
+export declare class ValidationError extends Error { }

package/src/errors.js

@@ -0,0 +1,3 @@
+export class FatalError extends Error { }
+
+export class ValidationError extends Error { }

package/src/index.d.ts

@@ -0,0 +1,38 @@
+export interface StepDefinition<TInput = unknown, TOutput = unknown> {
+  name: string;
+  description?: string;
+  fn: ( input: TInput ) => Promise<TOutput>;
+}
+
+export interface WorkflowContext {
+  steps: {
+    [stepName: string]: ( input?: unknown ) => Promise<unknown>;
+  };
+  workflowId: string | null;
+  tools: {
+    webhook: ( options: {
+      name: string;
+      description?: string;
+      url: string;
+      payload: object;
+    } ) => Promise<unknown>;
+  };
+}
+
+export interface WorkflowDefinition<TInput = unknown, TOutput = unknown> {
+  name: string;
+  description?: string;
+  fn: ( input: TInput, context: WorkflowContext ) => Promise<TOutput>;
+}
+
+export function step<TInput = unknown, TOutput = unknown>(
+  definition: StepDefinition<TInput, TOutput>
+): ( input: TInput ) => Promise<TOutput>;
+
+export function workflow<TInput = unknown, TOutput = unknown>(
+  definition: WorkflowDefinition<TInput, TOutput>
+): ( input: TInput ) => Promise<TOutput>;
+
+export function startWorkflow<TOutput = unknown>( name: string, options?: { input?: TInput } ): Promise<TOutput>;
+
+export { FatalError, ValidationError } from './errors.js';

package/src/index.js

@@ -0,0 +1,5 @@
+import { step } from './interface/step.js';
+import { startWorkflow, workflow } from './interface/workflow.js';
+import { FatalError, ValidationError } from './errors.js';
+
+export { step, startWorkflow, workflow, FatalError, ValidationError };

package/src/interface/metadata.js

@@ -0,0 +1,4 @@
+import { nameSymbol } from '#consts';
+
+export const setName = ( target, name ) =>
+  Object.defineProperty( target, nameSymbol, { value: name, writable: false, enumerable: false, configurable: false } );

package/src/interface/step.js

@@ -0,0 +1,6 @@
+import { setName } from './metadata.js';
+
+export function step( { name, description: _description, fn } ) {
+  setName( fn, name );
+  return fn;
+};

package/src/interface/utils.js

@@ -0,0 +1,9 @@
+// This is rigged to return which folder had the source of calls for both interface methods (step workflow) functions
+// Important, if to refactor, pay attention to the depth in the stack trace to extract the info
+// now is 3 cause (1 line is name, 2 line is this function, 3 line is step/workflow, 4 line is caller)
+export const getInvocationDir = _ => new Error()
+  .stack.split( '\n' )[3]
+  .split( ' ' )
+  .at( -1 )
+  .replace( /\((.+):\d+:\d+\)/, '$1' )
+  .split( '/' ).slice( 0, -1 ).join( '/' );

package/src/interface/webhook.js

@@ -0,0 +1,18 @@
+import { defineSignal, setHandler } from '@temporalio/workflow';
+
+export function createWebhook( workflowId, requestDispatcher ) {
+  return async function webhook( { name: _name, description: _description, url, payload } ) {
+    const { error } = await requestDispatcher( { url, workflowId, payload } );
+    if ( error ) {
+      throw new Error( 'Webhook call failed' );
+    }
+
+    const resumeSignal = defineSignal( 'resume' );
+
+    return new Promise( resolve => {
+      setHandler( resumeSignal, responsePayload => {
+        resolve( responsePayload );
+      } );
+    } );
+  };
+};

package/src/interface/workflow.js

@@ -0,0 +1,69 @@
+// THIS RUNS IN THE TEMPORAL'S SANDBOX ENVIRONMENT
+import { proxyActivities, IllegalStateError, executeChild, workflowInfo } from '@temporalio/workflow';
+import { getInvocationDir } from './utils.js';
+import { createWebhook } from './webhook.js';
+import { setName } from './metadata.js';
+import { sendWebhookPostName } from '#consts';
+import { FatalError, ValidationError } from '../errors.js';
+
+const temporalActivityConfigs = {
+  startToCloseTimeout: '20 minute',
+  retry: {
+    initialInterval: '10s',
+    backoffCoefficient: 2.0,
+    maximumInterval: '2 minutes',
+    maximumAttempts: 3,
+    nonRetryableErrorTypes: [ ValidationError.name, FatalError.name ]
+  }
+};
+
+export function workflow( { name, description: _description, fn } ) {
+  const workflowPath = getInvocationDir();
+
+  // This wraps the actually function call so before being invoked it creates the activity proxies and export the context
+  const wrapper = async input => {
+    try {
+      const { workflowId } = workflowInfo();
+
+      // enrich context information needed for tracking
+      Object.assign( workflowInfo().memo, { workflowPath } );
+
+      const proxies = proxyActivities( temporalActivityConfigs );
+      const context = {
+        steps: new Proxy( {}, {
+          get( _target, name ) {
+            return proxies[`${workflowPath}#${name}`];
+          }
+        } ),
+        tools: {
+          webhook: createWebhook( workflowId, proxies[sendWebhookPostName] )
+        }
+      };
+
+      return fn( input, context );
+
+    } catch ( error ) {
+      // IllegalStateError is thrown when temporal is not running and one try to access workflowInfo()
+      if ( error instanceof IllegalStateError ) {
+        return fn( input, { steps: null, workflowId: null } );
+      }
+
+      throw error;
+    }
+  };
+
+  setName( wrapper, name );
+  return wrapper;
+};
+
+export async function startWorkflow( name, { input } = {} ) {
+  const { memo, workflowId, workflowType } = workflowInfo();
+
+  // Checks if current memo has rootWorkflowId, which means current execution is already a child
+  // Then it sets the memory for the child execution passing along who's the original workflow is and its type
+  const workflowMemory = memo.rootWorkflowId ?
+    { parentWorkflowId: workflowId, rootWorkflowType: memo.rootWorkflowType, rootWorkflowId: memo.rootWorkflowId } :
+    { parentWorkflowId: workflowId, rootWorkflowId: workflowId, rootWorkflowType: workflowType };
+
+  return executeChild( name, { args: input ? [ input ] : [], memo: workflowMemory } );
+}

package/src/internal_activities/index.js

@@ -0,0 +1,17 @@
+import { fetch } from 'undici';
+import { api as apiConfig } from '#configs';
+
+export async function sendWebhookPost( { url, workflowId, payload } ) {
+  const res = await fetch( url, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      Authentication: `Basic ${apiConfig.authKey}`
+    },
+    body: JSON.stringify( { workflowId, payload } )
+  } );
+
+  console.log( '[Core.SendWebhookPost]', res.status, res.statusText );
+
+  return { error: !res.ok };
+};

package/src/worker/async_storage.js

@@ -0,0 +1,19 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+
+export const store = new AsyncLocalStorage();
+
+export const Storage = {
+  /**
+   * Execute a code piece wrapped in a function binding a given arbitrary object as context around it so it can be retrieve later
+   * @param {Function|AsyncFunction} fn The code to execute wrapped around a function without arguments
+   * @param {Object} context The context to bind
+   * @returns {any} The result of the `fn` execution
+   */
+  runWithContext: ( fn, context ) => store.run( context, fn ),
+
+  /**
+   * Load the context stored upstream in the chain of calls that lead to this point with `runWithContext`.
+   * @returns {any} Stored context
+   */
+  load: () => store.getStore()
+};

package/src/worker/index.js

@@ -0,0 +1,49 @@
+import { Worker, NativeConnection } from '@temporalio/worker';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'node:url';
+import { worker as workerConfig } from '#configs';
+import { loadActivities, loadWorkflows } from './loader.js';
+import { ActivityExecutionInterceptor } from './interceptors/activity.js';
+import { setupGlobalTracer } from './tracer/index.js';
+import { sinks } from './sinks.js';
+
+const __dirname = dirname( fileURLToPath( import.meta.url ) );
+
+// expose the coreTracker so other parts of the SDK can use it
+setupGlobalTracer();
+
+const { address, apiKey, maxActivities, maxWorkflows, namespace, taskQueue } = workerConfig;
+
+// Get caller directory from command line arguments
+const callerDir = process.argv[2];
+
+( async () => {
+  console.log( '[Core]', 'Loading workflows...', { callerDir } );
+  const workflowsPath = await loadWorkflows( callerDir );
+
+  console.log( '[Core]', 'Loading activities...', { callerDir } );
+  const activities = await loadActivities( callerDir );
+
+  console.log( '[Core]', 'Connecting Temporal...' );
+  // enable TLS only when connecting remove (api key is present)
+  const connection = await NativeConnection.connect( { address, tls: Boolean( apiKey ), apiKey } );
+
+  console.log( '[Core]', 'Creating Worker...' );
+  const worker = await Worker.create( {
+    connection,
+    namespace,
+    taskQueue,
+    workflowsPath,
+    activities,
+    sinks,
+    interceptors: {
+      workflowModules: [ join( __dirname, './interceptors/workflow.js' ) ],
+      activityInbound: [ () => new ActivityExecutionInterceptor() ]
+    },
+    maxConcurrentWorkflowTaskExecutions: maxWorkflows,
+    maxConcurrentActivityTaskExecutions: maxActivities
+  } );
+
+  console.log( '[Core] Starting...' );
+  worker.run();
+} )();

package/src/worker/interceptors/activity.js

@@ -0,0 +1,28 @@
+import { Context } from '@temporalio/activity';
+import { Storage } from '../async_storage.js';
+import { trace } from '../tracer/index.js';
+import { TraceEvent } from '../tracer/types.js';
+import { headersToObject } from '../sandboxed_utils.js';
+
+/*
+  This interceptor is called for every activity execution
+
+  It will have access to the activity scope.
+
+  What it does is to wrap the execution using Node's AsyncLocalStorage to save context info for tracing.
+
+  Some information it needs for its context comes from Temporal's Activity Context others are injected in the headers
+*/
+export class ActivityExecutionInterceptor {
+  async execute( input, next ) {
+    const { workflowExecution: { workflowId }, activityId, activityType, workflowType } = Context.current().info;
+    const context = { workflowId, workflowType, activityId, activityType, ...headersToObject( input.headers ) };
+
+    return Storage.runWithContext( async _ => {
+      trace( { lib: 'core', event: TraceEvent.STEP_START, input: input.args } );
+      const output = await next( input );
+      trace( { lib: 'core', event: TraceEvent.STEP_END, output } );
+      return output;
+    }, context );
+  }
+};

package/src/worker/interceptors/workflow.js

@@ -0,0 +1,42 @@
+// THIS RUNS IN THE TEMPORAL'S SANDBOX ENVIRONMENT
+import { proxySinks, workflowInfo } from '@temporalio/workflow';
+import { TraceEvent } from '../tracer/types.js';
+import { memoToHeaders } from '../sandboxed_utils.js';
+
+/*
+  This is not an AI comment!
+
+  This interceptor adds information value from workflowInfo().memo as Activity invocation headers.
+
+  This is a strategy to share values between the workflow context and activity context.
+
+  We also want to preserve existing headers that might have been inject somewhere else and
+*/
+class HeadersInjectionInterceptor {
+  async scheduleActivity( input, next ) {
+    Object.assign( input.headers, memoToHeaders( workflowInfo().memo ) );
+    return next( input );
+  }
+};
+
+/*
+This interceptor captures the workflow execution start and stop to log these event for the internal tracing
+  This is not an AI comment!
+
+  It uses sinks to share them.
+  - https://docs.temporal.io/develop/typescript/observability
+*/
+class WorkflowExecutionInterceptor {
+  async execute( input, next ) {
+    const sinks = proxySinks();
+    sinks.log.trace( { event: TraceEvent.WORKFLOW_START, input: input.args } );
+    const output = await next( input );
+    sinks.log.trace( { event: TraceEvent.WORKFLOW_END, output } );
+    return output;
+  }
+};
+
+export const interceptors = () => ( {
+  outbound: [ new HeadersInjectionInterceptor( workflowInfo().workflowType ) ],
+  inbound: [ new WorkflowExecutionInterceptor( workflowInfo().workflowType ) ]
+} );

package/src/worker/loader.js

@@ -0,0 +1,64 @@
+import { readdirSync, writeFileSync, existsSync, mkdirSync } from 'fs';
+import { dirname, join, resolve } from 'path';
+import { pathToFileURL, fileURLToPath } from 'url';
+import { nameSymbol, sendWebhookPostName, workflowsIndexFileName } from '#consts';
+import { sendWebhookPost } from '#internal_activities';
+
+const ignoreDirNames = [ 'vendor', 'node_modules' ];
+
+const __dirname = dirname( fileURLToPath( import.meta.url ) );
+
+// recursive run directories and when find files with given name, save their path, pathname and URI
+const recursiveNavigateWhileCollecting = ( path, filenames, collection = [] ) => {
+  for ( const entry of readdirSync( path, { withFileTypes: true } ) ) {
+    if ( ignoreDirNames.includes( entry.name ) ) { continue; }
+
+    const pathname = resolve( path, entry.name );
+    if ( entry.isDirectory() ) {
+      recursiveNavigateWhileCollecting( pathname, filenames, collection );
+    } else if ( filenames.includes( entry.name ) ) {
+      collection.push( { pathname, path, url: pathToFileURL( pathname ).href } );
+    }
+  }
+
+  return collection;
+};
+
+export async function loadActivities( path ) {
+  const activityPaths = recursiveNavigateWhileCollecting( path, [ 'steps.js' ] );
+  const activities = [];
+  for ( const { path, pathname, url } of activityPaths ) {
+    const exported = await import( url );
+    for ( const [ name, fn ] of Object.values( exported ).map( v => [ v[nameSymbol], v ] ).filter( v => v[0] ) ) {
+      console.log( '[Core.Scanner]', 'Activity loaded', name, pathname );
+      activities[`${path}#${name}`] = fn;
+    }
+  }
+
+  // system activities
+  activities[sendWebhookPostName] = sendWebhookPost;
+  return activities;
+};
+
+export async function loadWorkflows( path ) {
+  const workflowPaths = recursiveNavigateWhileCollecting( path, [ 'index.js' ] );
+  const workflows = [];
+  for ( const { pathname, url } of workflowPaths ) {
+    const exported = await import( url );
+
+    for ( const name of Object.values( exported ).map( v => v[nameSymbol] ).filter( n => n ) ) {
+      console.log( '[Core.Scanner]', 'Workflow loaded', name, pathname );
+      workflows.push( { name, pathname } );
+    }
+  }
+
+  const tempFolder = join( __dirname, 'temp' );
+  const entryPoint = join( tempFolder, workflowsIndexFileName );
+  const workflowsIndex = workflows.map( ( { name, pathname } ) => `export { default as ${name} } from '${pathname}';` ).join( '\n' );
+
+  if ( !existsSync( tempFolder ) ) {
+    mkdirSync( tempFolder );
+  }
+  writeFileSync( entryPoint, workflowsIndex, 'utf-8' );
+  return entryPoint;
+};

package/src/worker/sandboxed_utils.js

@@ -0,0 +1,18 @@
+// THIS IS SAFE TO RUN AT TEMPORAL'S SANDBOX ENVIRONMENT
+import { defaultPayloadConverter } from '@temporalio/common';
+
+/*
+  @important: They plain JS values need to be converted to "payload":
+  - https://typescript.temporal.io/api/namespaces/common/#headers
+  - https://community.temporal.io/t/specify-temporal-headers-when-starting-workflow/6712
+*/
+export const memoToHeaders = memo =>
+  Object.fromEntries(
+    Object.entries( memo ?? {} ).map( ( [ k, v ] ) => [ k, defaultPayloadConverter.toPayload( v ) ] )
+  );
+
+// And the opposite of the function above
+export const headersToObject = headers =>
+  Object.fromEntries(
+    Object.entries( headers ?? {} ).map( ( [ k, v ] ) => [ k, defaultPayloadConverter.fromPayload( v ) ] )
+  );

package/src/worker/sinks.js

@@ -0,0 +1,15 @@
+import { Storage } from './async_storage.js';
+import { trace } from './tracer/index.js';
+
+export const sinks = {
+  // This sink allow for sandbox Temporal environment to send trace logs back to the main thread.
+  log: {
+    trace: {
+      fn( workflowInfo, args ) {
+        const { workflowId, workflowType, memo } = workflowInfo;
+        Storage.runWithContext( _ => trace( { lib: 'core', ...args } ), { workflowId, workflowType, ...memo } );
+      },
+      callDuringReplay: false
+    }
+  }
+};

package/src/worker/temp/__workflows_entrypoint.js

@@ -0,0 +1,6 @@
+export { default as articleDraftAgentic } from '/app/test-workflows/dist/article-draft-agentic/index.js';
+export { default as httpClientDemo } from '/app/test-workflows/dist/http/index.js';
+export { default as nested } from '/app/test-workflows/dist/nested/index.js';
+export { default as local_prompt } from '/app/test-workflows/dist/prompt/index.js';
+export { default as simple } from '/app/test-workflows/dist/simple/index.js';
+export { default as webhook } from '/app/test-workflows/dist/webhook/index.js';

package/src/worker/tracer/index.js

@@ -0,0 +1,43 @@
+import { Storage } from '../async_storage.js';
+import { mkdirSync, existsSync, readdirSync, appendFileSync } from 'node:fs';
+import { join } from 'path';
+import { EOL } from 'os';
+import { buildLogTree } from './tracer_tree.js';
+import { tracing as tracingConfig } from '#configs';
+
+const callerDir = process.argv[2];
+
+const flushEntry = ( path, json ) => appendFileSync( path, JSON.stringify( json ) + EOL, 'utf-8' );
+
+export function trace( { lib, event, input, output } ) {
+  const now = Date.now();
+
+  if ( !tracingConfig.enabled ) { return; }
+
+  const {
+    activityId: stepId,
+    activityType: stepName,
+    workflowId,
+    workflowType,
+    workflowPath,
+    parentWorkflowId,
+    rootWorkflowId,
+    rootWorkflowType
+  } = Storage.load();
+
+  const entry = { lib, event, input, output, timestamp: now, stepId, stepName, workflowId, workflowType, workflowPath, parentWorkflowId };
+
+  // test for rootWorkflow to append to the same file as the parent/grandparent
+  const outputDir = join( callerDir, 'logs', 'runs', rootWorkflowType ?? workflowType );
+  if ( !existsSync( outputDir ) ) { mkdirSync( outputDir, { recursive: true } ); }
+
+  const suffix = `-${rootWorkflowId ?? workflowId}.raw`;
+  const logFile = readdirSync( outputDir ).find( f => f.endsWith( suffix ) ) ?? `${new Date( now ).toISOString()}-${suffix}`;
+  const logPath = join( outputDir, logFile );
+
+  flushEntry( logPath, entry );
+  buildLogTree( logPath );
+};
+
+export const setupGlobalTracer = () =>
+  Object.defineProperty( globalThis, Symbol.for( '__trace' ), { value: trace, writable: false, enumerable: false, configurable: false } );

package/src/worker/tracer/tracer_tree.js

@@ -0,0 +1,61 @@
+import { readFileSync, writeFileSync } from 'node:fs';
+import { EOL } from 'os';
+import { TraceEvent } from './types.js';
+
+const timestampAscSort = ( a, b ) => a.timestamp > b.timestamp ? -1 : 1;
+
+const pushSort = ( arr, entry, sorter ) => {
+  arr.push( entry );
+  arr.sort( sorter );
+};
+
+const coreLib = 'core';
+
+export const buildLogTree = src => {
+  const content = readFileSync( src, 'utf-8' );
+  const entries = content.split( EOL ).slice( 0, -1 ).map( c => JSON.parse( c ) );
+
+  const stepsMap = new Map();
+  const workflowsMap = new Map();
+
+  // close steps/workflows
+  for ( const entry of entries.filter( e => e.lib === coreLib ) ) {
+    const { event, workflowId, workflowType, workflowPath, parentWorkflowId, stepId, stepName, input, output, timestamp } = entry;
+
+    const baseEntry = { children: [], startedAt: timestamp, workflowId };
+    if ( event === TraceEvent.STEP_START ) {
+      stepsMap.set( `${workflowId}:${stepId}`, { event: 'step', input, stepId, stepName, ...baseEntry } );
+    }
+
+    if ( event === TraceEvent.STEP_END ) {
+      Object.assign( stepsMap.get( `${workflowId}:${stepId}` ) ?? {}, { output, endedAt: timestamp } );
+    }
+
+    if ( event === TraceEvent.WORKFLOW_START ) {
+      workflowsMap.set( workflowId, { event: 'workflow', input, parentWorkflowId, workflowPath, workflowType, ...baseEntry } );
+    }
+
+    if ( event === TraceEvent.WORKFLOW_END ) {
+      Object.assign( workflowsMap.get( workflowId ) ?? {}, { output, endedAt: timestamp } );
+    }
+  }
+
+  // insert operations inside steps
+  for ( const entry of entries.filter( e => e.lib !== coreLib ) ) {
+    pushSort( stepsMap.get( `${entry.workflowId}:${entry.stepId}` ).children, entry, timestampAscSort );
+  }
+
+  // insert steps into workflows
+  for ( const step of stepsMap.values() ) {
+    pushSort( workflowsMap.get( step.workflowId ).children, step, timestampAscSort );
+  }
+
+  // insert children workflows
+  for ( const workflow of [ ...workflowsMap.values() ].filter( w => w.parentWorkflowId ) ) {
+    pushSort( workflowsMap.get( workflow.parentWorkflowId ).children, workflow, timestampAscSort );
+  }
+
+  const rootWorkflow = [ ...workflowsMap.values() ].find( w => !w.parentWorkflowId );
+
+  writeFileSync( src.replace( /\.raw$/, '.json' ), JSON.stringify( rootWorkflow, undefined, 2 ), 'utf-8' );
+};

package/src/worker/tracer/types.js

@@ -0,0 +1,6 @@
+export const TraceEvent = {
+  WORKFLOW_START: 'workflow_start',
+  WORKFLOW_END: 'workflow_end',
+  STEP_START: 'step_start',
+  STEP_END: 'step_end'
+};