@outputai/core 0.1.0

package/src/consts.js

@@ -0,0 +1,32 @@
+export const ACTIVITY_GET_TRACE_DESTINATIONS = '__internal#getTraceDestinations';
+export const ACTIVITY_OPTIONS_FILENAME = '__activity_options.js';
+export const ACTIVITY_SEND_HTTP_REQUEST = '__internal#sendHttpRequest';
+export const METADATA_ACCESS_SYMBOL = Symbol( '__metadata' );
+export const SHARED_STEP_PREFIX = '$shared';
+export const WORKFLOW_CATALOG = '$catalog';
+export const WORKFLOWS_INDEX_FILENAME = '__workflows_entrypoint.js';
+
+export const ComponentType = {
+  EVALUATOR: 'evaluator',
+  INTERNAL_STEP: 'internal_step',
+  STEP: 'step',
+  WORKFLOW: 'workflow'
+};
+
+export const LifecycleEvent = {
+  START: 'start',
+  END: 'end',
+  ERROR: 'error'
+};
+
+export const BusEventType = {
+  WORKFLOW_START: 'workflow:start',
+  WORKFLOW_END: 'workflow:end',
+  WORKFLOW_ERROR: 'workflow:error',
+
+  ACTIVITY_START: 'activity:start',
+  ACTIVITY_END: 'activity:end',
+  ACTIVITY_ERROR: 'activity:error',
+
+  RUNTIME_ERROR: 'runtime_error'
+};

package/src/errors.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Error indicating a non-recoverable failure.
+ *
+ * Throw this error to end the workflow execution altogether without retries.
+ */
+export class FatalError extends Error { }
+
+/**
+ * Error indicating invalid input or schema validation issues.
+ *
+ * This error is thrown when there are validation errors, either in the input or output, for steps, evaluators, and workflows.
+ *
+ * It will end the workflow execution without retries.
+ */
+export class ValidationError extends Error { }

package/src/errors.js

@@ -0,0 +1,14 @@
+/**
+ * These are errors exposed as tools for the user to break their flow
+ * They work in both steps and workflows
+ */
+
+/**
+ * Any generic fatal errors
+ */
+export class FatalError extends Error { }
+
+/**
+ * Any validation error
+ */
+export class ValidationError extends Error { }

package/src/hooks/index.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Payload passed to the onError handler when a workflow, activity or runtime error occurs.
+ */
+export interface ErrorHookPayload {
+  /** Origin of the error: workflow execution, activity execution, or runtime. */
+  source: 'workflow' | 'activity' | 'runtime';
+  /** Name of the workflow, when the error is scoped to a workflow or activity. */
+  workflowName?: string;
+  /** Name of the activity, when the error is from an activity. */
+  activityName?: string;
+  /** The error thrown. */
+  error: Error;
+}
+
+/**
+ * Register a handler to be invoked on workflow, activity or runtime errors.
+ *
+ * @param handler - Function called with the error payload.
+ */
+export declare function onError( handler: ( payload: ErrorHookPayload ) => void ): void;
+
+/**
+ * Register a handler to be invoked when a given event happens
+ *
+ * @param eventName - The name of the event to subscribe
+ * @param handler - Function called with the event payload
+ */
+export declare function on( eventName: string, handler: ( payload: object ) => void ): void;

package/src/hooks/index.js

@@ -0,0 +1,32 @@
+import { messageBus } from '#bus';
+import { BusEventType } from '#consts';
+import { createChildLogger } from '#logger';
+
+const log = createChildLogger( 'Hooks' );
+
+export const onError = handler => {
+  const invokeHandler = async args => {
+    try {
+      await handler( args );
+    } catch ( error ) {
+      log.error( 'onError hook error', { error } );
+    }
+  };
+
+  messageBus.on( BusEventType.ACTIVITY_ERROR, async ( { name, workflowName, error } ) =>
+    invokeHandler( { source: 'activity', activityName: name, workflowName, error } ) );
+  messageBus.on( BusEventType.WORKFLOW_ERROR, async ( { name, error } ) =>
+    invokeHandler( { source: 'workflow', workflowName: name, error } ) );
+  messageBus.on( BusEventType.RUNTIME_ERROR, async ( { error } ) =>
+    invokeHandler( { source: 'runtime', error } ) );
+};
+
+export const on = ( eventName, handler ) => {
+  messageBus.on( `external:${eventName}`, async payload => {
+    try {
+      await handler( payload );
+    } catch ( error ) {
+      log.error( `on(${eventName}) hook error`, { error } );
+    }
+  } );
+};

package/src/index.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Export all types from the interface
+ *
+ */
+export * from './interface/index.d.ts';
+
+/**
+ * Exports all errors
+ */
+export * from './errors.d.ts';
+
+/**
+ * Expose z from Zod as a convenience.
+ */
+export { z } from 'zod';
+
+/**
+ * Continue the workflow as a new execution with fresh history.
+ *
+ * Re-exported from Temporal for advanced use cases. Prefer using
+ * `context.control.continueAsNew()` within workflows for type-safe usage.
+ *
+ * @see {@link https://docs.temporal.io/develop/typescript/continue-as-new}
+ */
+export { continueAsNew } from '@temporalio/workflow';
+
+/**
+ * Exports Temporal's sleep() function for advanced use cases.
+ * Pause workflow execution for a specified duration.
+ *
+ * Use this for delay-based throttling when calling external APIs.
+ *
+ * @example
+ * ```ts
+ * import { sleep } from '@outputai/core';
+ *
+ * for ( const url of urls ) {
+ *   await fetchUrl( url );
+ *   await sleep( 100 ); // 100ms delay between calls
+ * }
+ * ```
+ *
+ * @see {@link https://docs.temporal.io/develop/typescript/timers}
+ *
+ * @param ms - Duration to sleep in milliseconds (or a string like '1s', '100ms')
+ * @returns A promise that resolves after the specified duration
+ *
+ */
+export function sleep( ms: number | string ): Promise<void>;

package/src/index.js

@@ -0,0 +1,4 @@
+export * from './interface/index.js';
+export * from './errors.js';
+export { z } from 'zod';
+export { continueAsNew, sleep } from '@temporalio/workflow';

package/src/interface/evaluation_result.d.ts

@@ -0,0 +1,173 @@
+import type { z } from 'zod';
+
+/**
+ * A single feedback for an EvaluationResult
+ */
+export class EvaluationFeedback {
+  /**
+   * Issue found
+   */
+  issue: string;
+
+  /**
+   * Improvement suggestion
+   */
+  suggestion?: string;
+
+  /**
+   * Reference for the issue
+   */
+  reference?: string;
+
+  /**
+   * Issue priority
+   */
+  priority?: 'low' | 'medium' | 'high' | 'critical';
+
+  /**
+   * @constructor
+   * @param args
+   * @param args.issue
+   * @param args.suggestion
+   * @param args.reference
+   * @param args.priority
+   */
+  constructor( args: {
+    issue: string;
+    suggestion?: string;
+    reference?: string;
+    priority?: 'low' | 'medium' | 'high' | 'critical';
+  } );
+
+  /**
+   * @returns The zod schema for this class
+   */
+  static get schema(): z.ZodType;
+}
+
+/**
+ * Base constructor arguments for EvaluationResult classes
+ */
+export type EvaluationResultArgs<TValue = any> = { // eslint-disable-line @typescript-eslint/no-explicit-any
+  /**
+   * The value of the evaluation
+   */
+  value: TValue;
+  /**
+   * The confidence in the evaluation
+   */
+  confidence: number;
+  /**
+   * The name of the evaluation
+   */
+  name?: string;
+  /**
+   * The reasoning behind the result
+   */
+  reasoning?: string;
+  /**
+   * Feedback for this evaluation
+   */
+  feedback?: EvaluationFeedback[];
+  /**
+   * Dimensions of this evaluation
+   */
+  dimensions?: Array<EvaluationStringResult | EvaluationNumberResult | EvaluationBooleanResult>;
+};
+
+/**
+ * Represents the result of an evaluation.
+ *
+ * Generic base class; evaluators must return an instance of an EvaluationResult subclass.
+ */
+export class EvaluationResult {
+  /**
+   * The name of the evaluation result
+   */
+  name?: string;
+
+  /**
+   * The evaluation result value
+   */
+  value: any; // eslint-disable-line @typescript-eslint/no-explicit-any
+
+  /**
+   * The evaluation result confidence
+   */
+  confidence: number;
+
+  /**
+   * The evaluation result reasoning
+   */
+  reasoning?: string;
+
+  /**
+   * Feedback for this evaluation
+   */
+  feedback: EvaluationFeedback[];
+
+  /**
+   * Dimensions of this evaluation
+   */
+  dimensions: Array<EvaluationStringResult | EvaluationNumberResult | EvaluationBooleanResult>;
+
+  /**
+   * @constructor
+   * @param args
+   */
+  constructor( args: EvaluationResultArgs );
+
+  /**
+   * @returns The zod schema for this class
+   */
+  static get schema(): z.ZodType;
+}
+
+/**
+ * An evaluation result where the value is a string
+ * @extends EvaluationResult
+ */
+export class EvaluationStringResult extends EvaluationResult {
+  /**
+   * @constructor
+   * @param args
+   */
+  constructor( args: EvaluationResultArgs<string> );
+}
+
+/**
+ * An evaluation result where the value is a number
+ * @extends EvaluationResult
+ */
+export class EvaluationNumberResult extends EvaluationResult {
+  /**
+   * @constructor
+   * @param args
+   */
+  constructor( args: EvaluationResultArgs<number> );
+}
+
+/**
+ * An evaluation result where the value is a boolean
+ * @extends EvaluationResult
+ */
+export class EvaluationBooleanResult extends EvaluationResult {
+  /**
+   * @constructor
+   * @param args
+   */
+  constructor( args: EvaluationResultArgs<boolean> );
+}
+
+/**
+ * An evaluation result where the value is a verdict (pass, partial, fail)
+ * @extends EvaluationResult
+ */
+export class EvaluationVerdictResult extends EvaluationResult {
+  /**
+   * @constructor
+   * @param args - See {@link EvaluationResultArgs} for full parameter documentation.
+   * @param args.value - The verdict: 'pass', 'partial', or 'fail'.
+   */
+  constructor( args: EvaluationResultArgs<'pass' | 'partial' | 'fail'> );
+}

package/src/interface/evaluation_result.js

@@ -0,0 +1,215 @@
+import { ValidationError } from '#errors';
+import * as z from 'zod';
+
+/**
+ * Error for when EvaluationResult are invalid
+ */
+export class EvaluationResultValidationError extends ValidationError {};
+
+/**
+ * A single feedback for an EvaluationResult
+ */
+export class EvaluationFeedback {
+
+  /**
+   * Issue found
+   * @type {string}
+   */
+  issue;
+
+  /**
+   * Improvement suggestion
+   * @type {string}
+   */
+  suggestion;
+
+  /**
+   * Reference for the issue
+   * @type {string}
+   */
+  reference;
+
+  /**
+   * Issue priority
+   * @type {'low' | 'medium' | 'high' | 'critical' | undefined}
+   */
+  priority;
+
+  /**
+   * The zod schema for this class
+   * @type {z.ZodType}
+   */
+  static get schema() {
+    return z.object( {
+      issue: z.string(),
+      suggestion: z.string().optional(),
+      reference: z.string().optional(),
+      priority: z.enum( [ 'low', 'medium', 'high', 'critical' ] ).optional()
+    } );
+  };
+
+  /**
+   * @constructor
+   * @param {object} args
+   * @param {string} args.issue
+   * @param {string} [args.suggestion]
+   * @param {string} [args.reference]
+   * @param {'low' | 'medium' | 'high' | 'critical'} [args.priority]
+   */
+  constructor( { issue, suggestion = undefined, reference = undefined, priority = undefined } ) {
+    const result = this.constructor.schema.safeParse( { issue, suggestion, reference, priority } );
+    if ( result.error ) {
+      throw new EvaluationResultValidationError( z.prettifyError( result.error ) );
+    }
+    this.issue = issue;
+    this.suggestion = suggestion;
+    this.reference = reference;
+    this.priority = priority;
+  }
+}
+
+/**
+ * Generic EvaluationResult class, represents the result of an evaluation.
+ */
+export class EvaluationResult {
+
+  /**
+   * The name of the evaluation result
+   * @type {string}
+   */
+  name;
+
+  /**
+   * The evaluation result value
+   * @type {any}
+   */
+  value = null;
+
+  /**
+   * The confidence value, between 0 and 1
+   * @type {number}
+   */
+  confidence;
+
+  /**
+   * The reasoning value
+   * @type {string}
+   */
+  reasoning;
+
+  /**
+   * Feedback for this evaluation
+   * @type {EvaluationFeedback[]}
+   */
+  feedback = [];
+
+  /**
+   * Dimensions of this evaluation
+   * @type {EvaluationResult[]}
+   */
+  dimensions = [];
+
+  /**
+   * The schema main field
+   * @type {z.ZodAny}
+   */
+  static valueSchema = z.any();
+
+  /**
+   * The zod schema for this class
+   * @type {z.ZodType}
+   */
+  static get schema() {
+    const baseSchema = z.object( {
+      value: this.valueSchema,
+      confidence: z.number(),
+      reasoning: z.string().optional(),
+      name: z.string().optional(),
+      feedback: z.array( EvaluationFeedback.schema ).optional()
+    } );
+
+    // Adds dimension but keep it only one level deep
+    return baseSchema.extend( {
+      dimensions: z.array(
+        baseSchema.extend( {
+          value: z.union( [ z.string(), z.number(), z.boolean() ] )
+        } )
+      ).optional()
+    } );
+  };
+
+  /**
+   * @constructor
+   * @param {object} args
+   * @param {any} args.value
+   * @param {number} args.confidence
+   * @param {string} [args.name]
+   * @param {EvaluationResult[]} [args.dimensions]
+   * @param {EvaluationFeedback[]} [args.feedback]
+   * @param {string} [args.reasoning]
+   */
+  constructor( { value, confidence, dimensions = [], feedback = [], name = undefined, reasoning = undefined } ) {
+    const result = this.constructor.schema.safeParse( { value, confidence, dimensions, feedback, name, reasoning } );
+    if ( result.error ) {
+      throw new EvaluationResultValidationError( z.prettifyError( result.error ) );
+    }
+    this.confidence = confidence;
+    this.value = value;
+    this.dimensions = dimensions;
+    this.feedback = feedback;
+    this.name = name;
+    this.reasoning = reasoning;
+  }
+};
+
+/**
+ * An evaluation result that uses a string value
+ * @extends EvaluationResult
+ * @property {string} value - The evaluation result value
+ * @constructor
+ * @param {object} args
+ * @param {string} args.value - The value of the evaluation (must be a string)
+ * @see EvaluationResult#constructor for other parameters (confidence, reasoning)
+ */
+export class EvaluationStringResult extends EvaluationResult {
+  static valueSchema = z.string();
+};
+
+/**
+ * An evaluation result that uses a boolean value
+ * @extends EvaluationResult
+ * @property {boolean} value - The evaluation result value
+ * @constructor
+ * @param {object} args
+ * @param {boolean} args.value - The value of the evaluation (must be a boolean)
+ * @see EvaluationResult#constructor for other parameters (confidence, reasoning)
+ */
+export class EvaluationBooleanResult extends EvaluationResult {
+  static valueSchema = z.boolean();
+};
+
+/**
+ * An evaluation result that uses a number value
+ * @extends EvaluationResult
+ * @property {number} value - The evaluation result value
+ * @constructor
+ * @param {object} args
+ * @param {number} args.value - The value of the evaluation (must be a number)
+ * @see EvaluationResult#constructor for other parameters (confidence, reasoning)
+ */
+export class EvaluationNumberResult extends EvaluationResult {
+  static valueSchema = z.number();
+};
+
+/**
+ * An evaluation result that uses a verdict value (pass, partial, fail)
+ * @extends EvaluationResult
+ * @property {'pass' | 'partial' | 'fail'} value - The evaluation verdict
+ * @constructor
+ * @param {object} args
+ * @param {'pass' | 'partial' | 'fail'} args.value - The verdict value
+ * @see EvaluationResult#constructor for other parameters (confidence, reasoning)
+ */
+export class EvaluationVerdictResult extends EvaluationResult {
+  static valueSchema = z.enum( [ 'pass', 'partial', 'fail' ] );
+};

package/src/interface/evaluator.d.ts

@@ -0,0 +1,70 @@
+import type { z } from 'zod';
+import type { AnyZodSchema, TemporalActivityOptions } from './types.d.ts';
+import type { EvaluationResult } from './evaluation_result.d.ts';
+
+/**
+ * Options for an evaluator.
+ */
+export type EvaluatorOptions = {
+
+  /**
+   * Temporal activity options for this evaluator.
+   */
+  activityOptions?: TemporalActivityOptions
+};
+
+/**
+ * The handler function of an evaluator.
+ *
+ * @param input - The evaluator input; it matches the schema defined by `inputSchema`.
+ *
+ * @returns The result of the evaluation.
+ */
+export type EvaluatorFunction<
+  InputSchema extends AnyZodSchema | undefined = undefined,
+  Result extends EvaluationResult
+> = InputSchema extends AnyZodSchema ?
+  ( input: z.infer<InputSchema> ) => Promise<Result> :
+  () => Promise<Result>;
+
+/**
+ * A wrapper around the user defined `fn` handler function.
+ *
+ * It has the same signature and returns the same value, calling the user function inside.
+ *
+ * It adds input validation based on the `inputSchema`.
+ */
+export type EvaluatorFunctionWrapper<EvaluatorFunction> =
+  Parameters<EvaluatorFunction> extends [infer Input] ?
+    ( input: Input ) => ReturnType<EvaluatorFunction> :
+    () => ReturnType<EvaluatorFunction>;
+
+/**
+ * Creates an evaluation function. It is similar to a step, but must return an EvaluationResult.
+ *
+ * It is translated to a Temporal Activity.
+ *
+ * @typeParam InputSchema - Zod schema of the fn's input.
+ * @typeParam Result - Return type of the fn, extends EvaluationResult.
+ *
+ * @throws {@link ValidationError}
+ * @throws {@link FatalError}
+ *
+ * @param params - Evaluator parameters
+ * @param params.name - Human-readable evaluator name (must start with a letter or underscore, followed by letters, numbers, or underscores)
+ * @param params.description - Description of the evaluator
+ * @param params.inputSchema - Zod schema for the `fn` input
+ * @param params.fn - A function containing the evaluator code
+ * @param params.options - Optional evaluator options.
+ * @returns A wrapper function around the `fn` function
+ */
+export declare function evaluator<
+  InputSchema extends AnyZodSchema,
+  Result extends EvaluationResult
+>( params: {
+  name: string;
+  description?: string;
+  inputSchema: InputSchema;
+  fn: EvaluatorFunction<InputSchema, Result>;
+  options?: EvaluatorOptions;
+} ): EvaluatorFunctionWrapper<EvaluatorFunction<InputSchema, Result>>;

package/src/interface/evaluator.js

@@ -0,0 +1,34 @@
+import { validateEvaluator } from './validations/static.js';
+import { validateWithSchema } from './validations/runtime.js';
+import { setMetadata } from '#utils';
+import { ValidationError } from '#errors';
+import { ComponentType } from '#consts';
+import { EvaluationResult } from './evaluation_result.js';
+/**
+ * Expose the function to create a new evaluator
+ * @param {object} opts
+ * @param {string} opts.name
+ * @param {string} opts.description
+ * @param {z.ZodType} opts.inputSchema
+ * @param {Function} opts.fn
+ * @param {object} opts.options
+ * @returns {Function}
+ */
+export function evaluator( { name, description, inputSchema, fn, options } ) {
+  validateEvaluator( { name, description, inputSchema, fn, options } );
+
+  const wrapper = async input => {
+    validateWithSchema( inputSchema, input, `Evaluator ${name} input` );
+
+    const output = await fn( input );
+
+    if ( !( output instanceof EvaluationResult ) ) {
+      throw new ValidationError( 'Evaluators must return an EvaluationResult' );
+    }
+
+    return output;
+  };
+
+  setMetadata( wrapper, { name, description, inputSchema, type: ComponentType.EVALUATOR, options } );
+  return wrapper;
+};