@etohq/connector-engine 1.5.1-alpha.4

package/src/engine/clean-connector-engine.ts

@@ -0,0 +1,726 @@
+/**
+ * @fileoverview Clean Connector Engine - First Principles Implementation
+ */
+
+import { ConnectorPlugin } from "../types/connector-plugin"
+import { RoutingConfig, RoutingContext } from "../types/routing-config"
+import { RoutingEngine } from "./routing-engine"
+import {
+  createStep,
+  createWorkflow,
+  StepResponse,
+  WorkflowResponse,
+  ReturnWorkflow,
+  WorkflowData,
+} from "@etohq/framework/workflows-sdk"
+
+// ===== ENGINE TYPES =====
+
+export interface CleanConnectorRegistry {
+  [connectorId: string]: ConnectorPlugin
+}
+
+export interface CleanOperationGroup {
+  connectors: string[]
+  operations: string[]
+  options: {
+    behavior: "execute_all" | "first_success" | "route_by_domain"
+    timeout?: number
+    retries?: number
+  }
+}
+
+export interface CleanOperationGroups {
+  [groupName: string]: CleanOperationGroup
+}
+
+export interface CleanEngineConfig {
+  configLoader: (connectorId: string) => Promise<any>
+  routingConfigLoader?: (groupId: string) => Promise<RoutingConfig> // Optional for gradual adoption
+}
+
+export interface CleanGroupResult<T> {
+  groupName: string
+  executedConnectors: string[]
+  results: Record<string, T>
+  metadata: {
+    behavior: string
+    executionTime: number
+    errors?: Record<string, Error>
+    routingDecision: any
+    algorithm: any
+    ruleApplied: string | undefined
+  }
+}
+
+// ===== TYPE GENERATION WITH AUTO-GROUPS =====
+
+// Simplified auto-group methods (domain-based routing)
+type AutoGroupMethods<TConnectors extends CleanConnectorRegistry> = {
+  [K in keyof TConnectors]: TConnectors[K] extends { getName(): infer Domain }
+    ? {
+        [Operation in keyof TConnectors[K]["operations"] as `execute${Capitalize<
+          Domain & string
+        >}${Capitalize<Operation & string>}`]: (
+          input: TConnectors[K]["operations"][Operation] extends (
+            input: infer I,
+            config: any
+          ) => any
+            ? I
+            : never,
+          context?: RoutingContext
+        ) => ReturnWorkflow<
+          TConnectors[K]["operations"][Operation] extends (
+            input: infer I,
+            config: any
+          ) => any
+            ? I
+            : never,
+          CleanGroupResult<
+            TConnectors[K]["operations"][Operation] extends (
+              input: any,
+              config: any
+            ) => Promise<infer O>
+              ? O
+              : never
+          >,
+          []
+        >
+      }
+    : {}
+}[keyof TConnectors]
+
+// Explicit group methods (existing)
+type GroupMethods<TGroups extends CleanOperationGroups> = {
+  [GroupName in keyof TGroups]: {
+    [Operation in TGroups[GroupName]["operations"][number] as `execute${Capitalize<
+      GroupName & string
+    >}${Capitalize<Operation & string>}`]: (
+      input: any,
+      context?: RoutingContext
+    ) => ReturnWorkflow<any, CleanGroupResult<any>, []>
+  }
+}[keyof TGroups]
+
+type ConnectorMethods<TConnectors extends CleanConnectorRegistry> = {
+  [ConnectorId in keyof TConnectors]: {
+    [Operation in keyof TConnectors[ConnectorId]["operations"] as `execute${Capitalize<
+      ConnectorId & string
+    >}${Capitalize<Operation & string>}`]: (
+      input: TConnectors[ConnectorId]["operations"][Operation] extends (
+        input: infer I,
+        config: any
+      ) => any
+        ? I
+        : never
+    ) => ReturnWorkflow<
+      TConnectors[ConnectorId]["operations"][Operation] extends (
+        input: infer I,
+        config: any
+      ) => any
+        ? I
+        : never,
+      TConnectors[ConnectorId]["operations"][Operation] extends (
+        input: any,
+        config: any
+      ) => Promise<infer O>
+        ? O
+        : never,
+      []
+    >
+  }
+}[keyof TConnectors]
+
+// ===== ENGINE IMPLEMENTATION =====
+
+class ConnectorEngineImpl<
+  TConnectors extends CleanConnectorRegistry,
+  TGroups extends CleanOperationGroups
+> {
+  private connectors: TConnectors
+  private groups: TGroups
+  private configLoader: (connectorId: string) => Promise<any>
+  private routingConfigLoader?: (groupId: string) => Promise<RoutingConfig>
+  private routingEngine: RoutingEngine
+
+  constructor(
+    connectors: TConnectors,
+    groups: TGroups,
+    config: CleanEngineConfig
+  ) {
+    this.connectors = connectors
+    this.groups = groups
+    this.configLoader = config.configLoader
+    this.routingConfigLoader = config.routingConfigLoader
+    this.routingEngine = new RoutingEngine()
+
+    this.generateDynamicMethods()
+  }
+
+  // ===== DYNAMIC METHOD GENERATION =====
+
+  private generateDynamicMethods(): void {
+    this.generateConnectorMethods()
+    this.generateGroupMethods()
+  }
+
+  private generateConnectorMethods(): void {
+    for (const [connectorId, plugin] of Object.entries(this.connectors)) {
+      for (const operationName of Object.keys(plugin.operations)) {
+        const methodName = `execute${this.capitalize(
+          connectorId
+        )}${this.capitalize(operationName)}`
+
+        ;(this as any)[methodName] = (): ReturnWorkflow<any, any, []> => {
+          return this.createConnectorWorkflow(connectorId, operationName)
+        }
+      }
+    }
+  }
+
+  private generateGroupMethods(): void {
+    // Auto-generate group methods even without explicit groups
+    this.generateAutoGroups()
+
+    // Generate methods for explicit groups
+    for (const [groupName, groupConfig] of Object.entries(this.groups)) {
+      for (const operationName of groupConfig.operations) {
+        const methodName = `execute${this.capitalize(
+          groupName
+        )}${this.capitalize(operationName)}`
+
+        ;(this as any)[methodName] = (
+          input: any,
+          context?: RoutingContext
+        ): ReturnWorkflow<any, CleanGroupResult<any>, []> => {
+          return this.createSmartGroupWorkflow(
+            groupName,
+            operationName,
+            input,
+            context
+          )
+        }
+      }
+    }
+  }
+
+  // Auto-generate groups for gradual adoption with type safety
+  private generateAutoGroups(): void {
+    const domainGroups: Record<
+      string,
+      { connectors: string[]; operations: Set<string> }
+    > = {}
+
+    // Group connectors by domain (plugin type)
+    for (const [connectorId, plugin] of Object.entries(this.connectors)) {
+      const domain = plugin.getName() // Use plugin name as domain
+
+      if (!domainGroups[domain]) {
+        domainGroups[domain] = { connectors: [], operations: new Set() }
+      }
+
+      domainGroups[domain].connectors.push(connectorId)
+
+      // Add all operations from this plugin
+      for (const op of Array.from(Object.keys(plugin.operations))) {
+        domainGroups[domain].operations.add(op)
+      }
+    }
+
+    // Generate group methods for each domain
+    for (const [domain, { connectors, operations }] of Object.entries(
+      domainGroups
+    )) {
+      for (const operationName of operations) {
+        const methodName = `execute${this.capitalize(domain)}${this.capitalize(
+          operationName
+        )}`
+
+        // Only generate if not already exists (explicit groups take precedence)
+        if (!(this as any)[methodName]) {
+          ;(this as any)[methodName] = (
+            input: any,
+            context?: RoutingContext
+          ): ReturnWorkflow<any, CleanGroupResult<any>, []> => {
+            return this.createAutoGroupWorkflow(
+              domain,
+              operationName,
+              input,
+              context,
+              connectors
+            )
+          }
+        }
+      }
+    }
+  }
+
+  // Create workflow for auto-generated groups
+  private createAutoGroupWorkflow<TInput, TOutput>(
+    domain: string,
+    operationName: string,
+    input: TInput,
+    context?: RoutingContext,
+    availableConnectors?: string[]
+  ): ReturnWorkflow<TInput, CleanGroupResult<TOutput>, []> {
+    return createWorkflow(
+      `${domain}-${operationName}-auto`,
+      (workflowInput: WorkflowData<TInput>) => {
+        const step = createStep(
+          `${domain}-${operationName}-auto-step`,
+          async (stepInput: TInput, { container }) => {
+            // Get connectors for this domain
+            const domainConnectors =
+              availableConnectors ||
+              Object.keys(this.connectors).filter(
+                (id) => this.connectors[id].getName() === domain
+              )
+
+            if (domainConnectors.length === 0) {
+              throw new Error(`No connectors available for domain: ${domain}`)
+            }
+
+            // Simple routing: try routing if available, otherwise use first connector
+            let selectedConnector: string
+
+            if (this.routingConfigLoader) {
+              try {
+                const routingConfig = await this.routingConfigLoader(domain)
+                const routingContext = this.extractRoutingContext(
+                  stepInput,
+                  context,
+                  operationName,
+                  domain
+                )
+                const routingDecision = await this.routingEngine.route(
+                  routingContext,
+                  routingConfig
+                )
+                selectedConnector = routingDecision.selected_connector
+              } catch {
+                // Fallback to simple selection if routing fails
+                selectedConnector = domainConnectors[0]
+              }
+            } else {
+              // No routing - use round robin or first available
+              selectedConnector = domainConnectors[0]
+            }
+
+            // Execute on selected connector
+            const config = await this.configLoader(selectedConnector)
+            const plugin = this.connectors[selectedConnector]
+            const operation =
+              plugin.operations[operationName as keyof typeof plugin.operations]
+
+            if (!operation) {
+              throw new Error(
+                `Operation ${operationName} not found on connector ${selectedConnector}`
+              )
+            }
+
+            const startTime = Date.now()
+            const result = await operation(stepInput, config)
+
+            return new StepResponse(
+              {
+                groupName: domain,
+                executedConnectors: [selectedConnector],
+                results: { [selectedConnector]: result },
+                metadata: {
+                  behavior: "auto_group",
+                  executionTime: Date.now() - startTime,
+                  routingDecision: { selected_connector: selectedConnector, algorithm_used: 'auto', rule_applied: undefined },
+                  algorithm: 'auto',
+                  ruleApplied: undefined
+                },
+              },
+              result
+            )
+          }
+        )
+
+        const result = step(workflowInput)
+        return new WorkflowResponse(result)
+      }
+    )
+  }
+
+  private capitalize(str: string): string {
+    return str.charAt(0).toUpperCase() + str.slice(1)
+  }
+
+  // ===== WORKFLOW CREATION =====
+
+  private createConnectorWorkflow<TInput, TOutput>(
+    connectorId: string,
+    operationName: string
+  ): ReturnWorkflow<TInput, TOutput, []> {
+    return createWorkflow(
+      `${connectorId}-${operationName}`,
+      (workflowInput: WorkflowData<TInput>) => {
+        const step = createStep(
+          `${connectorId}-${operationName}-step`,
+          async (stepInput: TInput, { container }) => {
+            const config = await this.configLoader(connectorId)
+            const plugin = this.connectors[connectorId]
+            const operation =
+              plugin.operations[operationName as keyof typeof plugin.operations]
+
+            if (!operation) {
+              throw new Error(
+                `Operation ${operationName} not found on connector ${connectorId}`
+              )
+            }
+
+            const result = await operation(stepInput, config)
+            return new StepResponse(result, result)
+          }
+        )
+
+        const result = step(workflowInput)
+        return new WorkflowResponse(result)
+      }
+    )
+  }
+
+  private createSmartGroupWorkflow<TInput, TOutput>(
+    groupName: string,
+    operationName: string,
+    input: TInput,
+    context?: RoutingContext
+  ): ReturnWorkflow<TInput, CleanGroupResult<TOutput>, []> {
+    return createWorkflow(
+      `${groupName}-${operationName}-smart`,
+      (workflowInput: WorkflowData<TInput>) => {
+        const step = createStep(
+          `${groupName}-${operationName}-smart-step`,
+          async (stepInput: TInput, { container }) => {
+            // Load routing config from DB
+            const routingConfig = this.routingConfigLoader
+              ? await this.routingConfigLoader(groupName)
+              : {
+                  profile_id: 'default',
+                  name: 'default',
+                  algorithm: 'priority' as any,
+                  connectors: [],
+                  rules: [],
+                  created_at: new Date().toISOString(),
+                  updated_at: new Date().toISOString(),
+                  modified_at: new Date().toISOString(),
+                  version: 1
+                }
+
+            // Create dynamic routing context from plugin input
+            const routingContext: RoutingContext = this.extractRoutingContext(
+              stepInput,
+              context,
+              operationName,
+              groupName
+            )
+
+            // Use smart routing to select connector
+            const routingDecision = await this.routingEngine.route(
+              routingContext,
+              routingConfig
+            )
+
+            // Execute on selected connector
+            const selectedConnector = routingDecision.selected_connector
+            const config = await this.configLoader(selectedConnector)
+            const plugin = this.connectors[selectedConnector]
+            const operation =
+              plugin.operations[operationName as keyof typeof plugin.operations]
+
+            if (!operation) {
+              throw new Error(
+                `Operation ${operationName} not found on connector ${selectedConnector}`
+              )
+            }
+
+            const startTime = Date.now()
+
+            try {
+              const result = await operation(stepInput, config)
+
+              // Update metrics for successful execution
+              await this.routingEngine.updateMetrics(
+                selectedConnector,
+                true,
+                Date.now() - startTime
+              )
+
+              return new StepResponse(
+                {
+                  groupName,
+                  executedConnectors: [selectedConnector],
+                  results: { [selectedConnector]: result },
+                  metadata: {
+                    behavior: "smart_routing",
+                    executionTime: Date.now() - startTime,
+                    routingDecision,
+                    algorithm: routingDecision.algorithm_used,
+                    ruleApplied: routingDecision.rule_applied,
+                  },
+                },
+                result
+              )
+            } catch (error) {
+              // Update metrics for failed execution
+              await this.routingEngine.updateMetrics(
+                selectedConnector,
+                false,
+                Date.now() - startTime
+              )
+
+              // Try fallback connectors if available
+              if (routingDecision.alternative_connectors.length > 0) {
+                return await this.tryFallbackConnectors(
+                  routingDecision.alternative_connectors,
+                  operationName,
+                  stepInput,
+                  groupName,
+                  startTime,
+                  routingContext
+                )
+              }
+
+              throw error
+            }
+          }
+        )
+
+        const result = step(workflowInput)
+        return new WorkflowResponse(result)
+      }
+    )
+  }
+
+  // ===== DYNAMIC CONTEXT EXTRACTION =====
+
+  private extractRoutingContext<TInput>(
+    input: TInput,
+    providedContext?: RoutingContext,
+    operationName?: string,
+    groupName?: string
+  ): RoutingContext {
+    const inputObj = input as any
+
+    // Start with provided context or defaults
+    const baseContext: RoutingContext = {
+      operation: operationName,
+      domain: groupName,
+      ...providedContext,
+    }
+
+    // Extract common fields from input (works for any domain)
+    if (inputObj.region !== undefined) baseContext.region = inputObj.region
+    if (inputObj.country !== undefined) baseContext.country = inputObj.country
+    if (inputObj.userId !== undefined) baseContext.userId = inputObj.userId
+    if (inputObj.user_id !== undefined) baseContext.userId = inputObj.user_id
+    if (inputObj.tenantId !== undefined)
+      baseContext.tenantId = inputObj.tenantId
+    if (inputObj.tenant_id !== undefined)
+      baseContext.tenantId = inputObj.tenant_id
+    if (inputObj.priority !== undefined)
+      baseContext.priority = inputObj.priority
+    if (inputObj.resourceType !== undefined)
+      baseContext.resourceType = inputObj.resourceType
+    if (inputObj.resource_type !== undefined)
+      baseContext.resourceType = inputObj.resource_type
+
+    // Extract size/amount fields (could be file size, payment amount, etc.)
+    if (inputObj.size !== undefined) baseContext.resourceSize = inputObj.size
+    if (inputObj.amount !== undefined)
+      baseContext.resourceSize = inputObj.amount
+    if (inputObj.fileSize !== undefined)
+      baseContext.resourceSize = inputObj.fileSize
+    if (inputObj.file_size !== undefined)
+      baseContext.resourceSize = inputObj.file_size
+
+    // Include all custom fields for rule evaluation
+    baseContext.metadata = {
+      ...baseContext.metadata,
+      operation: operationName,
+      domain: groupName,
+      inputFields: Object.keys(inputObj),
+      // Include any custom fields from input for rule evaluation
+      customFields: this.extractCustomFields(inputObj),
+    }
+
+    return baseContext
+  }
+
+  private extractCustomFields(input: any): Record<string, any> {
+    const customFields: Record<string, any> = {}
+    const standardFields = [
+      "region",
+      "country",
+      "userId",
+      "user_id",
+      "tenantId",
+      "tenant_id",
+      "priority",
+      "resourceType",
+      "resource_type",
+      "size",
+      "amount",
+      "fileSize",
+      "file_size",
+    ]
+
+    for (const [key, value] of Object.entries(input)) {
+      if (!standardFields.includes(key)) {
+        customFields[key] = value
+      }
+    }
+
+    return customFields
+  }
+
+  private async tryFallbackConnectors<TInput>(
+    fallbackConnectors: string[],
+    operationName: string,
+    input: TInput,
+    groupName: string,
+    originalStartTime: number,
+    context: RoutingContext
+  ): Promise<StepResponse<CleanGroupResult<any>, any>> {
+    const errors: Record<string, Error> = {}
+
+    for (const connectorId of fallbackConnectors) {
+      try {
+        const config = await this.configLoader(connectorId)
+        const plugin = this.connectors[connectorId]
+        const operation =
+          plugin.operations[operationName as keyof typeof plugin.operations]
+
+        if (!operation) continue
+
+        const startTime = Date.now()
+        const result = await operation(input, config)
+
+        // Update metrics for successful fallback
+        await this.routingEngine.updateMetrics(
+          connectorId,
+          true,
+          Date.now() - startTime
+        )
+
+        return new StepResponse(
+          {
+            groupName,
+            executedConnectors: [connectorId],
+            results: { [connectorId]: result },
+            metadata: {
+              behavior: "fallback_routing",
+              executionTime: Date.now() - originalStartTime,
+              routingDecision: { selected_connector: connectorId, algorithm_used: 'fallback', rule_applied: undefined },
+              algorithm: 'fallback',
+              ruleApplied: undefined
+            },
+          },
+          result
+        )
+      } catch (error) {
+        errors[connectorId] =
+          error instanceof Error ? error : new Error(String(error))
+
+        // Update metrics for failed fallback
+        await this.routingEngine.updateMetrics(
+          connectorId,
+          false,
+          Date.now() - originalStartTime
+        )
+      }
+    }
+
+    throw new Error(
+      `All connectors failed for ${groupName}.${operationName}. Errors: ${JSON.stringify(
+        errors
+      )}`
+    )
+  }
+
+  // ===== PUBLIC API =====
+
+  async execute<
+    ConnectorId extends keyof TConnectors,
+    Operation extends keyof TConnectors[ConnectorId]["operations"]
+  >(connectorId: ConnectorId, operation: Operation, input: any): Promise<any> {
+    const config = await this.configLoader(connectorId as string)
+    const plugin = this.connectors[connectorId]
+    const operationFn =
+      plugin.operations[operation as keyof typeof plugin.operations]
+
+    if (!operationFn) {
+      throw new Error(
+        `Operation ${String(operation)} not found on connector ${String(
+          connectorId
+        )}`
+      )
+    }
+
+    return await operationFn(input, config)
+  }
+
+  async executeGroup<GroupName extends keyof TGroups>(
+    groupName: GroupName,
+    operation: string,
+    input: any
+  ): Promise<CleanGroupResult<any>> {
+    // Use smart routing workflow and extract result
+    const workflow = this.createSmartGroupWorkflow(
+      groupName as string,
+      operation,
+      input
+    )
+    const result = await workflow.run({ input })
+
+    return result.result as CleanGroupResult<any>
+  }
+}
+
+// ===== ENGINE FACTORY =====
+
+export function ConnectorEngine<
+  TConnectors extends CleanConnectorRegistry,
+  TGroups extends CleanOperationGroups = {}
+>(
+  connectors: TConnectors,
+  groups?: TGroups,
+  config?: CleanEngineConfig
+): ConnectorEngineImpl<TConnectors, TGroups> &
+  ConnectorMethods<TConnectors> &
+  AutoGroupMethods<TConnectors> &
+  GroupMethods<TGroups> {
+  const engineConfig: CleanEngineConfig = config || {
+    configLoader: async () => ({}), // Default empty config
+    routingConfigLoader: async () => ({
+      profile_id: "default",
+      name: "default",
+      algorithm: { type: "priority", data: [] },
+      created_at: new Date().toISOString(),
+      modified_at: new Date().toISOString(),
+      version: 1,
+    }),
+  }
+
+  return new ConnectorEngineImpl(
+    connectors,
+    groups || ({} as TGroups),
+    engineConfig
+  ) as ConnectorEngineImpl<TConnectors, TGroups> &
+    ConnectorMethods<TConnectors> &
+    AutoGroupMethods<TConnectors> &
+    GroupMethods<TGroups>
+}
+
+// ===== EXPORTS =====
+
+export type {
+  CleanConnectorRegistry as ConnectorRegistry,
+  CleanOperationGroup as OperationGroup,
+  CleanOperationGroups as OperationGroups,
+  CleanEngineConfig as EngineConfig,
+  CleanGroupResult as GroupResult,
+}