@elizaos/plugin-workflow 2.0.0-beta.1 → 2.0.3-beta.5

package/src/types/index.ts

@@ -5,7 +5,7 @@ import type {
   INodeProperties,
   INodeTypeDescription,
   IWorkflowSettings,
-} from '@elizaos/workflows';
+} from './workflow-contracts';
 
 // Core workflow types
 
@@ -111,6 +111,27 @@ export interface WorkflowDefinitionResponse extends WorkflowDefinition {
   versionId: string;
 }
 
+export type WorkflowRevisionOperation =
+  | 'update'
+  | 'activate'
+  | 'deactivate'
+  | 'tags'
+  | 'restore'
+  | 'delete';
+
+export interface WorkflowRevision {
+  id: string;
+  workflowId: string;
+  versionId: string;
+  name: string;
+  active: boolean;
+  workflow: WorkflowDefinition;
+  createdAt: string;
+  updatedAt: string;
+  capturedAt: string;
+  operation: WorkflowRevisionOperation;
+}
+
 export interface WorkflowCredential {
   id: string;
   name: string;
@@ -121,6 +142,18 @@ export interface WorkflowCredential {
   updatedAt: string;
 }
 
+export interface WorkflowExecutionEngineMetrics {
+  provider: 'smithers';
+  nodes: number;
+  levels: number;
+  maxConcurrency: number;
+  started: number;
+  finished: number;
+  failed: number;
+  skipped: number;
+  retries: number;
+}
+
 export interface WorkflowExecution {
   id: string;
   finished: boolean;
@@ -147,6 +180,7 @@ export interface WorkflowExecution {
     resultData?: {
       runData?: Record<string, unknown[]>;
       lastNodeExecuted?: string;
+      engine?: WorkflowExecutionEngineMetrics;
       error?: {
         message: string;
         stack?: string;
@@ -160,6 +194,64 @@ export interface WorkflowExecution {
   };
 }
 
+export interface WorkflowEvaluationSampleNode {
+  name: string;
+  status: 'success' | 'error' | 'unknown';
+  itemCount: number;
+  executionTimeMs?: number;
+  error?: string;
+  preview?: string;
+}
+
+export interface WorkflowEvaluationSample {
+  id: string;
+  workflowId: string;
+  workflowName: string;
+  workflowVersionId?: string;
+  executionId: string;
+  createdAt: string;
+  input: {
+    mode: WorkflowExecution['mode'];
+    triggerData?: Record<string, unknown>;
+  };
+  expected: {
+    status: WorkflowExecution['status'];
+    passed: boolean;
+    lastNodeExecuted?: string;
+    engine?: WorkflowExecutionEngineMetrics;
+    error?: string;
+    nodes: WorkflowEvaluationSampleNode[];
+  };
+  score: {
+    pass: boolean;
+    value: number;
+    reason: string;
+  };
+  tags: string[];
+}
+
+export interface WorkflowEvaluationSuite {
+  workflowId: string;
+  workflowName: string;
+  workflowVersionId?: string;
+  generatedAt: string;
+  sampleCount: number;
+  samples: WorkflowEvaluationSample[];
+  jsonl: string;
+  optimizer: {
+    engine: 'smithers-gepa';
+    target: 'workflow-generation';
+    suiteName: string;
+    caseFile: string;
+    recommendedCommand: string;
+    recommendedEvalCommand: string;
+    recommendedOptimizeCommand: string;
+    recommendedObservabilityCommand: string;
+    recommendedMetricsCommand: string;
+    notes: string[];
+  };
+}
+
 export interface WorkflowTag {
   id: string;
   name: string;
@@ -426,7 +518,7 @@ export interface ConnectorDisconnectedPayload extends EventPayload {
   userId: string;
   /**
    * Workflow credential type ids tied to the disconnected connector
-   * (e.g. `['gmailOAuth2', 'gmailOAuth2Api']`). Empty list ⇒ no-op.
+   * (e.g. `['gmailOAuth2', 'gmailOAuth2Api']`). Empty list skips deletion.
    */
   credTypes: readonly string[];
   /** Connector name (e.g. `'gmail'`). Informational. */

package/src/types/workflow-contracts.ts

@@ -0,0 +1,166 @@
+export type GenericValue = string | object | number | boolean | undefined | null;
+
+export type NodeParameterValue = string | number | boolean | undefined | null;
+
+export type NodeParameterValueType =
+  | NodeParameterValue
+  | INodeParameters
+  | NodeParameterValue[]
+  | INodeParameters[]
+  | Record<string, unknown>
+  | Array<Record<string, unknown>>;
+
+export interface INodeParameters {
+  [key: string]: NodeParameterValueType;
+}
+
+export type NodePropertyTypes =
+  | 'boolean'
+  | 'button'
+  | 'collection'
+  | 'color'
+  | 'dateTime'
+  | 'fixedCollection'
+  | 'hidden'
+  | 'json'
+  | 'notice'
+  | 'multiOptions'
+  | 'number'
+  | 'options'
+  | 'string'
+  | 'credentialsSelect'
+  | 'resourceLocator'
+  | 'curlImport'
+  | 'resourceMapper'
+  | 'filter'
+  | 'assignmentCollection'
+  | 'credentials'
+  | 'workflowSelector';
+
+export interface INodePropertyOptions {
+  name: string;
+  value?: string | number | boolean;
+  action?: string;
+  description?: string;
+  displayName?: string;
+}
+
+export interface INodePropertyCollection {
+  displayName?: string;
+  name: string;
+  values: INodeProperties[];
+}
+
+export interface INodeProperties {
+  displayName: string;
+  name: string;
+  type: NodePropertyTypes;
+  default: NodeParameterValueType;
+  description?: string;
+  hint?: string;
+  options?: Array<INodePropertyOptions | INodeProperties | INodePropertyCollection>;
+  placeholder?: string;
+  required?: boolean;
+  typeOptions?: Record<string, unknown>;
+  displayOptions?: unknown;
+  routing?: unknown;
+}
+
+export interface INodeCredentialsDetails {
+  id: string | null;
+  name: string;
+  __aiGatewayManaged?: boolean;
+}
+
+export interface INodeCredentials {
+  [key: string]: INodeCredentialsDetails;
+}
+
+export type OnError = 'continueErrorOutput' | 'continueRegularOutput' | 'stopWorkflow';
+
+export interface INode {
+  id: string;
+  name: string;
+  typeVersion: number;
+  type: string;
+  position: [number, number];
+  disabled?: boolean;
+  notes?: string;
+  notesInFlow?: boolean;
+  retryOnFail?: boolean;
+  maxTries?: number;
+  waitBetweenTries?: number;
+  alwaysOutputData?: boolean;
+  executeOnce?: boolean;
+  onError?: OnError;
+  continueOnFail?: boolean;
+  parameters: INodeParameters;
+  credentials?: INodeCredentials;
+  webhookId?: string;
+  extendsCredential?: string;
+  rewireOutputLogTo?: string;
+  forceCustomOperation?: {
+    resource: string;
+    operation: string;
+  };
+}
+
+export type NodeGroupType =
+  | 'input'
+  | 'output'
+  | 'organization'
+  | 'schedule'
+  | 'transform'
+  | 'trigger';
+
+export interface INodeTypeDescription {
+  displayName: string;
+  name: string;
+  group: NodeGroupType[];
+  version: number | number[];
+  description: string;
+  defaults: {
+    name: string;
+    color?: string;
+  };
+  inputs: unknown[] | string[] | string;
+  outputs: unknown[] | string[] | string;
+  credentials?: Array<{
+    name: string;
+    required?: boolean;
+    displayOptions?: unknown;
+  }>;
+  properties: INodeProperties[];
+  icon?: string;
+  iconUrl?: string;
+  polling?: boolean;
+  triggerPanel?: unknown;
+  webhooks?: Array<Record<string, unknown>>;
+}
+
+export namespace WorkflowSettings {
+  export type CallerPolicy = 'workflowsFromSameOwner' | 'workflowsFromAList' | 'any';
+  export type SaveDataExecution = 'DEFAULT' | 'all' | 'none';
+  export type RedactionPolicy = 'none' | 'mask';
+}
+
+export type WorkflowSettingsBinaryMode = 'separate' | 'combined';
+
+export interface IWorkflowSettings {
+  timezone?: 'DEFAULT' | string;
+  errorWorkflow?: 'DEFAULT' | string;
+  callerIds?: string;
+  callerPolicy?: WorkflowSettings.CallerPolicy;
+  saveDataErrorExecution?: WorkflowSettings.SaveDataExecution;
+  saveDataSuccessExecution?: WorkflowSettings.SaveDataExecution;
+  saveManualExecutions?: 'DEFAULT' | boolean;
+  saveExecutionProgress?: 'DEFAULT' | boolean;
+  executionTimeout?: number;
+  executionOrder?: 'v0' | 'v1';
+  binaryMode?: WorkflowSettingsBinaryMode;
+  timeSavedPerExecution?: number;
+  timeSavedMode?: 'fixed' | 'dynamic';
+  availableInMCP?: boolean;
+  credentialResolverId?: string;
+  redactionPolicy?: WorkflowSettings.RedactionPolicy;
+}

package/src/utils/catalog.ts

@@ -9,7 +9,7 @@ import type {
 /**
  * workflows node catalog with keyword-based search
  * @note Uses embedded catalog (457 nodes as of April 2025)
- * @todo Add dynamic refresh via GET /node-types in v2
+ * @note Dynamic refresh via GET /node-types belongs in a catalog-refresh pass.
  */
 
 const NODE_CATALOG = defaultNodesData as NodeDefinition[];
@@ -64,7 +64,7 @@ export function searchNodes(keywords: string[], limit = 15): NodeSearchResult[]
 
     const nodeName = node.name.toLowerCase();
     const nodeDisplayName = node.displayName.toLowerCase();
-    const nodeDescription = node.description?.toLowerCase() || '';
+    const nodeDescription = node.description.toLowerCase() || '';
     const nameTokens = tokenize(node.name);
     const displayTokens = tokenize(node.displayName);
 

package/src/utils/clarification.ts

@@ -18,6 +18,10 @@ export function isCatalogClarification(item: string | ClarificationRequest): boo
     : isCatalogClarificationString(item.question);
 }
 
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
 /**
  * Normalize a mixed-shape clarifications array into structured
  * `ClarificationRequest` objects. Legacy strings become `kind: 'free_text'`
@@ -25,7 +29,7 @@ export function isCatalogClarification(item: string | ClarificationRequest): boo
  * picker). Structured items pass through unchanged.
  */
 export function coerceClarificationRequests(
-  items: ReadonlyArray<string | ClarificationRequest> | undefined | null
+  items: ReadonlyArray<unknown> | undefined | null
 ): ClarificationRequest[] {
   if (!items || items.length === 0) {
     return [];
@@ -38,11 +42,21 @@ export function coerceClarificationRequests(
         continue;
       }
       out.push({ kind: 'free_text', question: trimmed, paramPath: '' });
-    } else if (item && typeof item === 'object' && typeof item.question === 'string') {
+    } else if (isRecord(item) && typeof item.question === 'string') {
       out.push({
-        kind: item.kind ?? 'free_text',
-        platform: item.platform,
-        scope: item.scope,
+        kind:
+          item.kind === 'target_channel' ||
+          item.kind === 'target_server' ||
+          item.kind === 'recipient' ||
+          item.kind === 'value' ||
+          item.kind === 'free_text'
+            ? item.kind
+            : 'free_text',
+        platform: typeof item.platform === 'string' ? item.platform : undefined,
+        scope:
+          isRecord(item.scope) && typeof item.scope.guildId === 'string'
+            ? { guildId: item.scope.guildId }
+            : undefined,
         question: item.question,
         paramPath: typeof item.paramPath === 'string' ? item.paramPath : '',
       });

package/src/utils/context.ts

@@ -3,7 +3,7 @@ import type { IAgentRuntime, Memory, State, UUID } from '@elizaos/core';
 export function buildConversationContext(message: Memory, state: State | undefined): string {
   const raw = state?.values?.recentMessages;
   const recentMessages = typeof raw === 'string' ? raw : '';
-  const currentText = message.content?.text ?? '';
+  const currentText = message.content.text ?? '';
 
   if (!recentMessages) {
     return currentText;

package/src/utils/evaluation-samples.ts

@@ -0,0 +1,239 @@
+import type {
+  WorkflowDefinitionResponse,
+  WorkflowEvaluationSample,
+  WorkflowEvaluationSampleNode,
+  WorkflowEvaluationSuite,
+  WorkflowExecution,
+} from '../types/index';
+
+const DEFAULT_LIMIT = 10;
+const MAX_LIMIT = 50;
+const MAX_PREVIEW_LENGTH = 240;
+const MAX_SAMPLE_VALUE_DEPTH = 4;
+const MAX_SAMPLE_OBJECT_KEYS = 20;
+const MAX_SAMPLE_ARRAY_ITEMS = 10;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function readString(value: unknown): string | undefined {
+  return typeof value === 'string' && value.trim() ? value.trim() : undefined;
+}
+
+function readNumber(value: unknown): number | undefined {
+  return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
+}
+
+function clampLimit(limit?: number): number {
+  if (typeof limit !== 'number' || !Number.isFinite(limit)) return DEFAULT_LIMIT;
+  return Math.min(Math.max(1, Math.floor(limit)), MAX_LIMIT);
+}
+
+function truncate(value: string, maxLength = MAX_PREVIEW_LENGTH): string {
+  return value.length > maxLength ? `${value.slice(0, maxLength - 3)}...` : value;
+}
+
+function compactValue(value: unknown, depth = 0): unknown {
+  if (
+    value === null ||
+    typeof value === 'boolean' ||
+    typeof value === 'number' ||
+    typeof value === 'undefined'
+  ) {
+    return value;
+  }
+  if (typeof value === 'string') {
+    return truncate(value, MAX_PREVIEW_LENGTH);
+  }
+  if (depth >= MAX_SAMPLE_VALUE_DEPTH) {
+    return '[truncated]';
+  }
+  if (Array.isArray(value)) {
+    return value.slice(0, MAX_SAMPLE_ARRAY_ITEMS).map((item) => compactValue(item, depth + 1));
+  }
+  if (!isRecord(value)) {
+    return String(value);
+  }
+  return Object.fromEntries(
+    Object.entries(value)
+      .slice(0, MAX_SAMPLE_OBJECT_KEYS)
+      .map(([key, item]) => [key, compactValue(item, depth + 1)])
+  );
+}
+
+function compactRecord(value: unknown): Record<string, unknown> | undefined {
+  if (!isRecord(value)) return undefined;
+  const compacted = compactValue(value);
+  return isRecord(compacted) ? compacted : undefined;
+}
+
+function countMainItems(data: unknown): number {
+  if (!isRecord(data) || !Array.isArray(data.main)) return 0;
+  return data.main.reduce((total, output) => {
+    if (!Array.isArray(output)) return total;
+    return total + output.length;
+  }, 0);
+}
+
+function previewMainData(data: unknown): string | undefined {
+  if (!isRecord(data) || !Array.isArray(data.main)) return undefined;
+  for (const output of data.main) {
+    if (!Array.isArray(output)) continue;
+    const first = output.find(isRecord);
+    if (!first) continue;
+    const json = isRecord(first.json) ? first.json : first;
+    try {
+      return truncate(JSON.stringify(compactValue(json)));
+    } catch {
+      return 'Output could not be previewed';
+    }
+  }
+  return undefined;
+}
+
+function readRunError(run: unknown): string | undefined {
+  if (!isRecord(run)) return undefined;
+  const error = run.error;
+  if (isRecord(error)) {
+    return readString(error.message) ?? readString(error.description);
+  }
+  return readString(error);
+}
+
+function readExecutionError(execution: WorkflowExecution): string | undefined {
+  const error = execution.data?.resultData?.error;
+  if (error?.message) return error.message;
+  for (const node of collectNodeSamples(execution)) {
+    if (node.error) return node.error;
+  }
+  return undefined;
+}
+
+function collectNodeSamples(execution: WorkflowExecution): WorkflowEvaluationSampleNode[] {
+  const runData = execution.data?.resultData?.runData;
+  if (!runData) return [];
+
+  return Object.entries(runData).flatMap(([name, runs]) => {
+    if (!Array.isArray(runs)) return [];
+    return runs.map((run): WorkflowEvaluationSampleNode => {
+      const record = isRecord(run) ? run : {};
+      const error = readRunError(record);
+      const preview = previewMainData(record.data);
+      return {
+        name,
+        status: error ? 'error' : execution.status === 'success' ? 'success' : 'unknown',
+        itemCount: countMainItems(record.data),
+        ...(readNumber(record.executionTime) !== undefined
+          ? { executionTimeMs: readNumber(record.executionTime) }
+          : {}),
+        ...(error ? { error } : {}),
+        ...(preview ? { preview } : {}),
+      };
+    });
+  });
+}
+
+function slugify(value: string): string {
+  return value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 48);
+}
+
+function buildSample(
+  workflow: WorkflowDefinitionResponse,
+  execution: WorkflowExecution
+): WorkflowEvaluationSample {
+  const passed = execution.status === 'success';
+  const workflowId = workflow.id;
+  const triggerData = compactRecord(execution.customData?.triggerData);
+  const error = readExecutionError(execution);
+  const nodes = collectNodeSamples(execution);
+  const scoreReason = passed
+    ? 'Execution completed successfully.'
+    : error
+      ? `Execution failed: ${error}`
+      : `Execution finished with status ${execution.status}.`;
+
+  return {
+    id: `${workflowId}:${execution.id}`,
+    workflowId,
+    workflowName: workflow.name,
+    workflowVersionId: workflow.versionId,
+    executionId: execution.id,
+    createdAt: execution.stoppedAt ?? execution.startedAt,
+    input: {
+      mode: execution.mode,
+      ...(triggerData ? { triggerData } : {}),
+    },
+    expected: {
+      status: execution.status,
+      passed,
+      ...(execution.data?.resultData?.lastNodeExecuted
+        ? { lastNodeExecuted: execution.data.resultData.lastNodeExecuted }
+        : {}),
+      ...(execution.data?.resultData?.engine ? { engine: execution.data.resultData.engine } : {}),
+      ...(error ? { error } : {}),
+      nodes,
+    },
+    score: {
+      pass: passed,
+      value: passed ? 1 : 0,
+      reason: scoreReason,
+    },
+    tags: [
+      'smithers',
+      'workflow-eval',
+      `workflow:${workflowId}`,
+      `status:${execution.status}`,
+      `mode:${execution.mode}`,
+    ],
+  };
+}
+
+export function buildWorkflowEvaluationSuite(
+  workflow: WorkflowDefinitionResponse,
+  executions: WorkflowExecution[],
+  options?: { limit?: number; generatedAt?: string }
+): WorkflowEvaluationSuite {
+  const limit = clampLimit(options?.limit);
+  const samples = executions.slice(0, limit).map((execution) => buildSample(workflow, execution));
+  const safeName = slugify(workflow.name) || slugify(workflow.id) || 'workflow';
+  const suiteName = safeName;
+  const caseFile = `evals/${suiteName}.jsonl`;
+  const recommendedEvalCommand = `bunx smithers-orchestrator eval <workflow.tsx> --cases ${caseFile} --suite ${suiteName}`;
+  const recommendedOptimizeCommand = 'bunx smithers-orchestrator optimize';
+  const recommendedObservabilityCommand = 'bunx smithers-orchestrator observability --detach';
+  const recommendedMetricsCommand =
+    'bunx smithers-orchestrator up <workflow.tsx> --serve --metrics';
+  const jsonl = samples.map((sample) => JSON.stringify(sample)).join('\n');
+  return {
+    workflowId: workflow.id,
+    workflowName: workflow.name,
+    workflowVersionId: workflow.versionId,
+    generatedAt: options?.generatedAt ?? new Date().toISOString(),
+    sampleCount: samples.length,
+    samples,
+    jsonl,
+    optimizer: {
+      engine: 'smithers-gepa',
+      target: 'workflow-generation',
+      suiteName,
+      caseFile,
+      recommendedCommand: recommendedEvalCommand,
+      recommendedEvalCommand,
+      recommendedOptimizeCommand,
+      recommendedObservabilityCommand,
+      recommendedMetricsCommand,
+      notes: [
+        `Copy jsonl to ${caseFile} before running eval.`,
+        'Run GEPA optimization after the eval suite is configured; Smithers writes an optimized prompt artifact only when score improves.',
+        'Use the observability and metrics commands when inspecting long-running or flaky workflows.',
+        'Successful executions are positive examples; failed executions remain useful regression cases.',
+        'Samples are generated from persisted workflow executions and compact large payloads.',
+      ],
+    },
+  };
+}