@output.ai/core 0.1.0 → 0.1.1

package/src/worker/loader.js

@@ -1,51 +1,60 @@
-import { dirname, join } from 'path';
+import { dirname, join } from 'node:path';
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { EOL } from 'node:os';
 import { fileURLToPath } from 'url';
-import { sendWebhook, readTraceFile } from '#internal_activities';
-import { ACTIVITY_SEND_WEBHOOK, ACTIVITY_READ_TRACE_FILE, WORKFLOWS_INDEX_FILENAME } from '#consts';
-import {
-  iteratorOverImportedComponents,
-  recursiveNavigateWhileCollecting,
-  writeFileOnLocationSync
-} from './internal_utils.js';
+import { sendWebhook } from '#internal_activities';
+import { ACTIVITY_SEND_WEBHOOK, WORKFLOWS_INDEX_FILENAME, WORKFLOW_CATALOG } from '#consts';
+import { importComponents } from './loader_tools.js';
 
 const __dirname = dirname( fileURLToPath( import.meta.url ) );
 
-// returns a map of activities, where the key is they path + name and the value is the function with metadata
-export async function loadActivities( path ) {
-  const activityPaths = recursiveNavigateWhileCollecting( path, [ 'steps.js', 'evaluators.js' ] );
-  const activities = [];
-  for await ( const { component, metadata, pathname, path } of iteratorOverImportedComponents( activityPaths ) ) {
-    console.log( '[Core.Scanner]', 'Component loaded:', metadata.type, metadata.name, 'at', pathname );
-    activities[`${path}#${metadata.name}`] = component;
+/**
+ * Builds a map of activities, where the key is their path and name and the value is the function
+ *
+ * @param {string} target
+ * @returns {object}
+ */
+export async function loadActivities( target ) {
+  const activities = {};
+  for await ( const { fn, metadata, path } of importComponents( target, [ 'steps.js', 'evaluators.js' ] ) ) {
+    console.log( '[Core.Scanner]', 'Component loaded:', metadata.type, metadata.name, 'at', path );
+    activities[`${dirname( path )}#${metadata.name}`] = fn;
   }
 
   // system activities
   activities[ACTIVITY_SEND_WEBHOOK] = sendWebhook;
-  activities[ACTIVITY_READ_TRACE_FILE] = readTraceFile;
   return activities;
 };
 
-// Returns array of workflow object, which are function with metadata attached
-export async function loadWorkflows( path ) {
-  const workflowPaths = recursiveNavigateWhileCollecting( path, [ 'workflow.js' ] );
+/**
+ * Builds an array of workflow objects
+ *
+ * @param {string} target
+ * @returns {object[]}
+ */
+export async function loadWorkflows( target ) {
   const workflows = [];
-  for await ( const { metadata, pathname, path } of iteratorOverImportedComponents( workflowPaths ) ) {
-    workflows.push( { ...metadata, pathname, path } );
-    console.log( '[Core.Scanner]', 'Workflow loaded:', metadata.name, 'at', pathname );
+  for await ( const { metadata, path } of importComponents( target, [ 'workflow.js' ] ) ) {
+    console.log( '[Core.Scanner]', 'Workflow loaded:', metadata.name, 'at', path );
+    workflows.push( { ...metadata, path } );
   }
   return workflows;
 };
 
-// Creates a temporary index file importing all workflows
+/**
+ * Creates a temporary index file importing all workflows
+ *
+ * @param {object[]} workflows
+ * @returns
+ */
 export function createWorkflowsEntryPoint( workflows ) {
-  const entryPoint = join( __dirname, 'temp', WORKFLOWS_INDEX_FILENAME );
+  const path = join( __dirname, 'temp', WORKFLOWS_INDEX_FILENAME );
 
   // default system catalog workflow
-  const catalogWorkflow = { name: 'catalog', pathname: join( __dirname, './catalog_workflow/workflow.js' ) };
-  const workflowsIndex = [ ... workflows, catalogWorkflow ]
-    .map( ( { name, pathname } ) => `export { default as ${name} } from '${pathname}';` )
-    .join( '\n' );
+  const catalog = { name: WORKFLOW_CATALOG, path: join( __dirname, './catalog_workflow/workflow.js' ) };
+  const content = [ ... workflows, catalog ].map( ( { name, path } ) => `export { default as ${name} } from '${path}';` ).join( EOL );
 
-  writeFileOnLocationSync( entryPoint, workflowsIndex );
-  return entryPoint;
+  mkdirSync( dirname( path ), { recursive: true } );
+  writeFileSync( path, content, 'utf-8' );
+  return path;
 };

package/src/worker/loader.spec.js

@@ -2,25 +2,23 @@ import { describe, it, expect, vi, beforeEach } from 'vitest';
 
 vi.mock( '#consts', () => ( {
   ACTIVITY_SEND_WEBHOOK: '__internal#sendWebhook',
-  ACTIVITY_READ_TRACE_FILE: '__internal#readTraceFile',
-  WORKFLOWS_INDEX_FILENAME: '__workflows_entrypoint.js'
+  WORKFLOWS_INDEX_FILENAME: '__workflows_entrypoint.js',
+  WORKFLOW_CATALOG: 'catalog'
 } ) );
 
 const sendWebhookMock = vi.fn();
-const readTraceFileMock = vi.fn();
 vi.mock( '#internal_activities', () => ( {
-  sendWebhook: sendWebhookMock,
-  readTraceFile: readTraceFileMock
+  sendWebhook: sendWebhookMock
 } ) );
 
-// Mock internal_utils to control filesystem-independent behavior
-const iteratorMock = vi.fn();
-const recursiveMock = vi.fn();
-const writeFileMock = vi.fn();
-vi.mock( './internal_utils.js', () => ( {
-  iteratorOverImportedComponents: iteratorMock,
-  recursiveNavigateWhileCollecting: recursiveMock,
-  writeFileOnLocationSync: writeFileMock
+const importComponentsMock = vi.fn();
+vi.mock( './loader_tools.js', () => ( { importComponents: importComponentsMock } ) );
+
+const mkdirSyncMock = vi.fn();
+const writeFileSyncMock = vi.fn();
+vi.mock( 'node:fs', () => ( {
+  mkdirSync: mkdirSyncMock,
+  writeFileSync: writeFileSyncMock
 } ) );
 
 describe( 'worker/loader', () => {
@@ -31,39 +29,37 @@ describe( 'worker/loader', () => {
   it( 'loadActivities returns map including system activity', async () => {
     const { loadActivities } = await import( './loader.js' );
 
-    recursiveMock.mockReturnValue( [ { pathname: '/a/steps.js', path: '/a', url: 'file:///a/steps.js' } ] );
-    iteratorMock.mockImplementation( async function *() {
-      yield { component: () => {}, metadata: { name: 'Act1' }, pathname: '/a/steps.js', path: '/a' };
+    importComponentsMock.mockImplementationOnce( async function *() {
+      yield { fn: () => {}, metadata: { name: 'Act1' }, path: '/a/steps.js' };
     } );
 
     const activities = await loadActivities( '/root' );
     expect( activities['/a#Act1'] ).toBeTypeOf( 'function' );
     expect( activities['__internal#sendWebhook'] ).toBe( sendWebhookMock );
-    expect( activities['__internal#readTraceFile'] ).toBe( readTraceFileMock );
   } );
 
   it( 'loadWorkflows returns array of workflows with metadata', async () => {
     const { loadWorkflows } = await import( './loader.js' );
 
-    recursiveMock.mockReturnValue( [ { pathname: '/b/workflow.js', path: '/b', url: 'file:///b/workflow.js' } ] );
-    iteratorMock.mockImplementation( async function *() {
-      yield { metadata: { name: 'Flow1', description: 'd' }, pathname: '/b/workflow.js', path: '/b' };
+    importComponentsMock.mockImplementationOnce( async function *() {
+      yield { metadata: { name: 'Flow1', description: 'd' }, path: '/b/workflow.js' };
     } );
 
     const workflows = await loadWorkflows( '/root' );
-    expect( workflows ).toEqual( [ { name: 'Flow1', description: 'd', pathname: '/b/workflow.js', path: '/b' } ] );
+    expect( workflows ).toEqual( [ { name: 'Flow1', description: 'd', path: '/b/workflow.js' } ] );
   } );
 
   it( 'createWorkflowsEntryPoint writes index and returns its path', async () => {
     const { createWorkflowsEntryPoint } = await import( './loader.js' );
 
-    const workflows = [ { name: 'W', pathname: '/abs/wf.js' } ];
+    const workflows = [ { name: 'W', path: '/abs/wf.js' } ];
     const entry = createWorkflowsEntryPoint( workflows );
 
-    expect( writeFileMock ).toHaveBeenCalledTimes( 1 );
-    const [ writtenPath, contents ] = writeFileMock.mock.calls[0];
+    expect( writeFileSyncMock ).toHaveBeenCalledTimes( 1 );
+    const [ writtenPath, contents ] = writeFileSyncMock.mock.calls[0];
     expect( entry ).toBe( writtenPath );
     expect( contents ).toContain( 'export { default as W } from \'/abs/wf.js\';' );
     expect( contents ).toContain( 'export { default as catalog }' );
+    expect( mkdirSyncMock ).toHaveBeenCalledTimes( 1 );
   } );
 } );

package/src/worker/loader_tools.js

@@ -0,0 +1,63 @@
+import { resolve } from 'path';
+import { pathToFileURL } from 'url';
+import { METADATA_ACCESS_SYMBOL } from '#consts';
+import { readdirSync } from 'fs';
+
+/**
+ * @typedef {object} CollectedFile
+ * @property {string} path - The file path
+ * @property {string} url - The resolved url of the file, ready to be imported
+ */
+/**
+ * @typedef {object} Component
+ * @property {Function} fn - The loaded component function
+ * @property {object} metadata - Associated metadata with the component
+ * @property {string} path - Associated metadata with the component
+ */
+
+/**
+ * Recursive traverse directories looking for files with given name.
+ *
+ * @param {string} path - The path to scan
+ * @param {string[]} filenames  - The filenames to look for
+ * @returns {CollectedFile[]} An array containing the collected files
+ * */
+const findByNameRecursively = ( parentPath, filenames, collection = [], ignoreDirNames = [ 'vendor', 'node_modules' ] ) => {
+  for ( const entry of readdirSync( parentPath, { withFileTypes: true } ) ) {
+    if ( ignoreDirNames.includes( entry.name ) ) {
+      continue;
+    }
+
+    const path = resolve( parentPath, entry.name );
+    if ( entry.isDirectory() ) {
+      findByNameRecursively( path, filenames, collection );
+    } else if ( filenames.includes( entry.name ) ) {
+      collection.push( { path, url: pathToFileURL( path ).href } );
+    }
+  }
+
+  return collection;
+};
+
+/**
+ * For each path, dynamic import it, and for each exported component with metadata (step, workflow), yields it.
+ *
+ * @generator
+ * @async
+ * @function importComponents
+ * @param {string} target - Place to look for files
+ * @param {string[]} filenames - File names to load recursively from target
+ * @yields {Component}
+ */
+export async function *importComponents( target, filenames ) {
+  for ( const { url, path } of findByNameRecursively( target, filenames ) ) {
+    const imported = await import( url );
+    for ( const fn of Object.values( imported ) ) {
+      const metadata = fn[METADATA_ACCESS_SYMBOL];
+      if ( !metadata ) {
+        continue;
+      }
+      yield { fn, metadata, path };
+    }
+  }
+};

package/src/worker/loader_tools.spec.js

@@ -0,0 +1,85 @@
+import { describe, it, expect } from 'vitest';
+import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { importComponents } from './loader_tools.js';
+
+describe( '.importComponents', () => {
+  it( 'imports modules and yields metadata from exports tagged with METADATA_ACCESS_SYMBOL', async () => {
+    const root = join( process.cwd(), 'sdk/core/temp_test_modules', `meta-${Date.now()}` );
+    mkdirSync( root, { recursive: true } );
+    const file = join( root, 'meta.module.js' );
+    writeFileSync( file, [
+      'import { METADATA_ACCESS_SYMBOL } from "#consts";',
+      'export const StepA = () => {};',
+      'StepA[METADATA_ACCESS_SYMBOL] = { kind: "step", name: "a" };',
+      'export const FlowB = () => {};',
+      'FlowB[METADATA_ACCESS_SYMBOL] = { kind: "workflow", name: "b" };'
+    ].join( '\n' ) );
+
+    const collected = [];
+    for await ( const m of importComponents( root, [ 'meta.module.js' ] ) ) {
+      collected.push( m );
+    }
+
+    expect( collected.length ).toBe( 2 );
+    expect( collected.map( m => m.metadata.name ).sort() ).toEqual( [ 'a', 'b' ] );
+    expect( collected.map( m => m.metadata.kind ).sort() ).toEqual( [ 'step', 'workflow' ] );
+    for ( const m of collected ) {
+      expect( m.path ).toBe( file );
+      expect( typeof m.fn ).toBe( 'function' );
+    }
+
+    rmSync( root, { recursive: true, force: true } );
+  } );
+
+  it( 'ignores exports without metadata symbol', async () => {
+    const root = join( process.cwd(), 'sdk/core/temp_test_modules', `meta-${Date.now()}-nometa` );
+    mkdirSync( root, { recursive: true } );
+    const file = join( root, 'meta.module.js' );
+    writeFileSync( file, [
+      'export const Plain = () => {};',
+      'export const AlsoPlain = {}'
+    ].join( '\n' ) );
+
+    const collected = [];
+    for await ( const m of importComponents( root, [ 'meta.module.js' ] ) ) {
+      collected.push( m );
+    }
+
+    expect( collected.length ).toBe( 0 );
+    rmSync( root, { recursive: true, force: true } );
+  } );
+
+  it( 'skips files inside ignored directories (node_modules, vendor)', async () => {
+    const root = join( process.cwd(), 'sdk/core/temp_test_modules', `meta-${Date.now()}-ignoredirs` );
+    const okDir = join( root, 'ok' );
+    const nmDir = join( root, 'node_modules' );
+    const vendorDir = join( root, 'vendor' );
+    mkdirSync( okDir, { recursive: true } );
+    mkdirSync( nmDir, { recursive: true } );
+    mkdirSync( vendorDir, { recursive: true } );
+
+    const okFile = join( okDir, 'meta.module.js' );
+    const nmFile = join( nmDir, 'meta.module.js' );
+    const vendorFile = join( vendorDir, 'meta.module.js' );
+
+    const fileContents = [
+      'import { METADATA_ACCESS_SYMBOL } from "#consts";',
+      'export const C = () => {};',
+      'C[METADATA_ACCESS_SYMBOL] = { kind: "step", name: "c" };'
+    ].join( '\n' );
+    writeFileSync( okFile, fileContents );
+    writeFileSync( nmFile, fileContents );
+    writeFileSync( vendorFile, fileContents );
+
+    const collected = [];
+    for await ( const m of importComponents( root, [ 'meta.module.js' ] ) ) {
+      collected.push( m );
+    }
+
+    expect( collected.length ).toBe( 1 );
+    expect( collected[0].path ).toBe( okFile );
+
+    rmSync( root, { recursive: true, force: true } );
+  } );
+} );

package/src/worker/sinks.js

@@ -1,3 +1,4 @@
+import { WORKFLOW_CATALOG } from '#consts';
 import { addEventStart, addEventEnd, addEventError } from '#tracing';
 
 /**
@@ -8,8 +9,11 @@ import { addEventStart, addEventEnd, addEventError } from '#tracing';
  * @param {object} details - Teh details to attach to the event
  */
 const addWorkflowEvent = ( method, workflowInfo, details ) => {
-  const { workflowId: id, workflowType: name, memo: { parentId, traceId, traceHelm } } = workflowInfo;
-  method( { id, kind: 'workflow', name, details, parentId, traceId, traceHelm } );
+  const { workflowId: id, workflowType: name, memo: { parentId, executionContext } } = workflowInfo;
+  if ( name === WORKFLOW_CATALOG ) {
+    return;
+  } // ignore internal catalog events
+  method( { id, kind: 'workflow', name, details, parentId, executionContext } );
 };
 
 /**
@@ -21,8 +25,8 @@ const addWorkflowEvent = ( method, workflowInfo, details ) => {
  */
 const addEvent = ( method, workflowInfo, options ) => {
   const { id, name, kind, details } = options;
-  const { workflowId, memo: { traceId, traceHelm } } = workflowInfo;
-  method( { id, kind, name, details, parentId: workflowId, traceId, traceHelm } );
+  const { workflowId, memo: { executionContext } } = workflowInfo;
+  method( { id, kind, name, details, parentId: workflowId, executionContext } );
 };
 
 // This sink allow for sandbox Temporal environment to send trace logs back to the main thread.

package/src/configs.js

@@ -1,31 +0,0 @@
-import * as z from 'zod';
-
-class InvalidEnvVarsErrors extends Error { }
-
-const envVarSchema = z.object( {
-  TEMPORAL_ADDRESS: z.string().optional().default( 'localhost:7233' ),
-  TEMPORAL_NAMESPACE: z.string().optional().default( 'default' ),
-  TEMPORAL_API_KEY: z.string().optional(),
-  CATALOG_ID: z.string().regex( /^[a-z0-9_.@-]+$/i ),
-  API_AUTH_KEY: z.string().optional()
-} );
-
-const { data: safeEnvVar, error } = envVarSchema.safeParse( process.env );
-if ( error ) {
-  throw new InvalidEnvVarsErrors( z.prettifyError( error ) );
-}
-
-export const worker = {
-  address: safeEnvVar.TEMPORAL_ADDRESS,
-  apiKey: safeEnvVar.TEMPORAL_API_KEY,
-  executionTimeout: '1m',
-  maxActivities: 100,
-  maxWorkflows: 100,
-  namespace: safeEnvVar.TEMPORAL_NAMESPACE ?? 'default',
-  taskQueue: safeEnvVar.CATALOG_ID,
-  catalogId: safeEnvVar.CATALOG_ID
-};
-
-export const api = {
-  authKey: safeEnvVar.API_AUTH_KEY
-};