@output.ai/core 0.1.6 → 0.1.9-dev.pr156.0

package/README.md

@@ -48,6 +48,28 @@ export default workflow( {
 })
 ```
 
+Workflows can only import the following files
+
+#### Components
+- `evaluators.js`
+- `shared_steps.js`
+- `steps.js`
+- `workflow.js`
+
+#### Core library
+- `@output.ai.core`
+
+#### Whitelisted files
+- `types.js`
+- `consts.js`
+- `constants.js`
+- `vars.js`
+- `variables.js`
+- `utils.js`
+- `tools.js`
+- `functions.js`
+- `shared.js`
+
 ### Step
 
 Re-usable units of work that can contain IO, used by the workflow.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@output.ai/core",
-  "version": "0.1.6",
+  "version": "0.1.9-dev.pr156.0",
   "description": "The core module of the output framework",
   "type": "module",
   "exports": {

package/src/index.d.ts

@@ -144,7 +144,7 @@ export async function step( options: {
  */
 
 /**
- * Creates a workflow orchestrator defined schema for both input and output.
+ * Creates a workflow orchestrator with defined schema for both input and output.
  *
  * @param {object} options - Workflow options
  * @param {string} options.name - Unique workflow name

package/src/tracing/processors/local/index.js

@@ -29,6 +29,8 @@ export const init = () => {
 
 const getOutputDir = workflowName => join( process.argv[2], 'logs', 'runs', workflowName );
 
+const getRelativeOutputDir = workflowName => join( 'logs', 'runs', workflowName );
+
 const buildOutputFileName = ( { startTime, workflowId } ) => {
   const timestamp = new Date( startTime ).toISOString().replace( /[:T.]/g, '-' );
   return `${timestamp}_${workflowId}.json`;
@@ -56,12 +58,12 @@ export const exec = ( { entry, executionContext } ) => {
 };
 
 /**
- * Returns where the trace is saved
+ * Returns where the trace is saved (as a relative path)
  * @param {object} args
  * @param {string} args.startTime - The start time of the workflow
  * @param {string} args.workflowId - The id of the workflow execution
  * @param {string} args.workflowName - The name of the workflow
- * @returns {string} The path where the trace will be saved
+ * @returns {string} The relative path where the trace will be saved
  */
 export const getDestination = ( { startTime, workflowId, workflowName } ) =>
-  join( getOutputDir( workflowName ), buildOutputFileName( { workflowId, startTime } ) );
+  join( getRelativeOutputDir( workflowName ), buildOutputFileName( { workflowId, startTime } ) );

package/src/tracing/processors/local/index.spec.js

@@ -63,5 +63,19 @@ describe( 'tracing/processors/local', () => {
     expect( writtenPath ).toMatch( /\/logs\/runs\/WF\// );
     expect( JSON.parse( content.trim() ).count ).toBe( 3 );
   } );
+
+  it( 'getDestination(): returns relative path', async () => {
+    const { getDestination } = await import( './index.js' );
+
+    const startTime = Date.parse( '2020-01-02T03:04:05.678Z' );
+    const workflowId = 'workflow-id-123';
+    const workflowName = 'test-workflow';
+
+    const destination = getDestination( { startTime, workflowId, workflowName } );
+
+    // Should return a relative path, not an absolute path
+    expect( destination ).not.toMatch( /^\/|^[A-Z]:\\/i ); // Not starting with / or Windows drive letter
+    expect( destination ).toBe( 'logs/runs/test-workflow/2020-01-02-03-04-05-678Z_workflow-id-123.json' );
+  } );
 } );
 

package/src/worker/webpack_loaders/consts.js

@@ -3,15 +3,24 @@ export const NodeType = {
 };
 
 export const ComponentFile = {
+  EVALUATORS: 'evaluators',
   STEPS: 'steps',
   SHARED_STEPS: 'shared_steps',
-  EVALUATORS: 'evaluators',
   WORKFLOW: 'workflow'
 };
 
-export const ExtraneousFile = {
-  TYPES: 'types'
-};
+export const EXTRANEOUS_FILE = 'extraneous';
+export const ExtraneousFileList = [
+  'types',
+  'consts',
+  'constants',
+  'vars',
+  'variables',
+  'utils',
+  'tools',
+  'functions',
+  'shared'
+];
 
 export const CoreModule = {
   LOCAL: 'local_core',

package/src/worker/webpack_loaders/tools.js

@@ -20,7 +20,7 @@ import {
   thisExpression,
   isExportDefaultDeclaration
 } from '@babel/types';
-import { ComponentFile, ExtraneousFile, NodeType } from './consts.js';
+import { ComponentFile, EXTRANEOUS_FILE, ExtraneousFileList, NodeType } from './consts.js';
 
 /**
  * Resolve a relative module specifier against a base directory.
@@ -134,6 +134,8 @@ export const isWorkflowPath = value => /(^|\/)workflow\.js$/.test( value );
  */
 export const isTypesPath = value => /(^|\/)types\.js$/.test( value );
 
+export const isExtraneousFile = value => ExtraneousFileList.map( t => new RegExp( `(^|\/)${t}\\.js$` ) ).find( rx => rx.test( value ) );
+
 /**
  * Determine file kind based on its path.
  * @param {string} filename
@@ -152,8 +154,8 @@ export const getFileKind = path => {
   if ( isWorkflowPath( path ) ) {
     return ComponentFile.WORKFLOW;
   }
-  if ( isTypesPath( path ) ) {
-    return ExtraneousFile.TYPES;
+  if ( isExtraneousFile( path ) ) {
+    return EXTRANEOUS_FILE;
   }
   return null;
 };

package/src/worker/webpack_loaders/tools.spec.js

@@ -20,6 +20,7 @@ import {
   buildSharedStepsNameMap,
   buildWorkflowNameMap,
   buildEvaluatorsNameMap,
+  isExtraneousFile,
   getFileKind
 } from './tools.js';
 
@@ -169,6 +170,49 @@ describe( 'workflow_rewriter tools', () => {
     expect( isEvaluatorsPath( 'steps.js' ) ).toBe( false );
   } );
 
+  it( 'isExtraneousFile: returns truthy for extraneous files (types/consts/constants/vars/variables)', () => {
+    const ok = [
+      'types.js',
+      './types.js',
+      '/a/b/types.js',
+      'consts.js',
+      './consts.js',
+      '/a/b/consts.js',
+      'constants.js',
+      './constants.js',
+      '/a/b/constants.js',
+      'vars.js',
+      './vars.js',
+      '/a/b/vars.js',
+      'variables.js',
+      './variables.js',
+      '/a/b/variables.js'
+    ];
+    for ( const p of ok ) {
+      expect( Boolean( isExtraneousFile( p ) ) ).toBe( true );
+    }
+  } );
+
+  it( 'isExtraneousFile: returns falsy for non-extraneous or non-.js files', () => {
+    const bad = [
+      'types.ts',
+      '/a/b/types.ts',
+      'types.mjs',
+      'variables.jsx',
+      'variables.mjs',
+      'myconstants.js',
+      'steps.js',
+      'evaluators.js',
+      'shared_steps.js',
+      'workflow.js',
+      'typess.js',
+      '/a/b/c/variables.json'
+    ];
+    for ( const p of bad ) {
+      expect( isExtraneousFile( p ) ).toBeFalsy();
+    }
+  } );
+
   it( 'createThisMethodCall: builds this.method(\'name\', ...args) call', () => {
     const call = createThisMethodCall( 'invoke', 'n', [ t.numericLiteral( 1 ), t.identifier( 'x' ) ] );
     expect( t.isCallExpression( call ) ).toBe( true );

package/src/worker/webpack_loaders/workflow_validator/index.mjs

@@ -1,7 +1,7 @@
 import traverseModule from '@babel/traverse';
 import { dirname } from 'node:path';
 import { parse, toAbsolutePath, getFileKind } from '../tools.js';
-import { ComponentFile, CoreModule, ExtraneousFile } from '../consts.js';
+import { ComponentFile, CoreModule, EXTRANEOUS_FILE, ExtraneousFileList } from '../consts.js';
 import {
   isCallExpression,
   isFunctionExpression,
@@ -23,10 +23,10 @@ const traverse = traverseModule.default ?? traverseModule;
 const validateWorkflowImports = ( { specifier, filename } ) => {
   const isCore = Object.values( CoreModule ).includes( specifier );
   const isComponent = Object.values( ComponentFile ).includes( getFileKind( specifier ) );
-  const isAllowedExtraneous = getFileKind( specifier ) === ExtraneousFile.TYPES;
+  const isAllowedExtraneous = getFileKind( specifier ) === EXTRANEOUS_FILE;
   if ( !isCore && !isComponent && !isAllowedExtraneous ) {
     throw new Error( `Invalid dependency in workflow.js: '${specifier}'. \
-Only evaluators, steps, shared_steps, types, workflows or @output.ai/* imports are allowed in ${filename}` );
+Only components (${Object.values( ComponentFile ) } ), @output.ai/core, or whitelisted (${ExtraneousFileList}) imports are allowed in ${filename}` );
   }
 };
 

package/src/worker/webpack_loaders/workflow_validator/index.spec.js

@@ -38,7 +38,7 @@ describe( 'workflow_validator loader', () => {
 
   it( 'workflow.js: rejects external dependencies', async () => {
     const dir = mkdtempSync( join( tmpdir(), 'wf-reject-' ) );
-    const src = 'import x from "./utils.js";';
+    const src = 'import x from "./foo.js";';
     await expect( runLoader( join( dir, 'workflow.js' ), src ) ).rejects.toThrow( /Invalid (import|dependency) in workflow\.js/ );
     rmSync( dir, { recursive: true, force: true } );
   } );
@@ -218,6 +218,17 @@ describe( 'workflow_validator loader', () => {
     rmSync( dir, { recursive: true, force: true } );
   } );
 
+  it( 'workflow.js: allows importing extraneous files (consts/constants/vars/variables)', async () => {
+    const dir = mkdtempSync( join( tmpdir(), 'wf-extra-allow-' ) );
+    const bases = [ 'consts', 'constants', 'vars', 'variables' ];
+    for ( const base of bases ) {
+      writeFileSync( join( dir, `${base}.js` ), 'export const X = 1\n' );
+      const src = `import x from "./${base}.js";`;
+      await expect( runLoader( join( dir, 'workflow.js' ), src ) ).resolves.toBeTruthy();
+    }
+    rmSync( dir, { recursive: true, force: true } );
+  } );
+
   it( 'steps.js: rejects require of steps/shared_steps/evaluators/workflow; allows other require', async () => {
     const dir = mkdtempSync( join( tmpdir(), 'steps-require-' ) );
     await expect( runLoader( join( dir, 'steps.js' ), 'const { S } = require("./steps.js");' ) )