conductor_ruby 0.1.0

data/examples/metrics_example.rb

@@ -0,0 +1,284 @@
+# frozen_string_literal: true
+
+#
+# Example demonstrating Prometheus metrics collection and HTTP endpoint exposure.
+#
+# This example shows how to:
+# - Enable Prometheus metrics collection for task execution
+# - Expose metrics via HTTP endpoint for scraping
+# - Track task poll times, execution times, errors, and more
+# - Integrate with Prometheus monitoring
+#
+# Metrics collected (canonical harmonized set):
+# - task_poll_total: Total number of task polls
+# - task_poll_time_seconds: Task poll duration (with status label)
+# - task_poll_error_total: Poll errors by exception type
+# - task_execution_started_total: Tasks dispatched to worker function
+# - task_execute_time_seconds: Task execution duration (with status label)
+# - task_execute_error_total: Execution errors by exception
+# - task_result_size_bytes: Task result payload size
+# - task_update_error_total: Failed task updates (CRITICAL)
+# - task_update_time_seconds: Task result update latency
+# - task_paused_total: Workers paused
+# - active_workers: Current active worker count (gauge)
+# - workflow_start_error_total: Workflow start failures
+# - workflow_input_size_bytes: Workflow input payload size
+# - http_api_client_request_seconds: HTTP API client latency
+# - thread_uncaught_exceptions_total: Uncaught thread exceptions
+#
+# Requirements:
+#   gem 'prometheus-client'
+#
+# Usage:
+#   1. Run this example: ruby metrics_example.rb
+#   2. View metrics: curl http://localhost:9090/metrics
+#   3. Configure Prometheus to scrape: http://localhost:9090/metrics
+#
+# Environment variables:
+#   CONDUCTOR_SERVER_URL  - Conductor server URL (default: http://localhost:8080/api)
+#   CONDUCTOR_AUTH_KEY    - Authentication key (optional)
+#   CONDUCTOR_AUTH_SECRET - Authentication secret (optional)
+#   METRICS_PORT          - Port for metrics HTTP server (default: 9090)
+#
+
+require 'bundler/setup'
+require 'conductor'
+
+# Check if prometheus-client is available
+begin
+  require 'prometheus/client'
+  PROMETHEUS_AVAILABLE = true
+rescue LoadError
+  PROMETHEUS_AVAILABLE = false
+  puts 'WARNING: prometheus-client gem not installed.'
+  puts 'Install with: gem install prometheus-client'
+  puts "Or add to Gemfile: gem 'prometheus-client'"
+  puts
+end
+
+#
+# Example worker tasks
+#
+
+# Async HTTP simulation worker
+Conductor::Worker.define('async_http_task', poll_interval: 100, thread_count: 10) do |task|
+  url = task.input_data['url'] || 'https://api.example.com/data'
+  delay = task.input_data['delay'] || 0.1
+
+  # Simulate async HTTP request
+  sleep(delay)
+
+  {
+    url: url,
+    status: 'success',
+    timestamp: Time.now.iso8601
+  }
+end
+
+# Data processor worker
+Conductor::Worker.define('async_data_processor', poll_interval: 100, thread_count: 10) do |task|
+  data = task.input_data['data'] || 'sample data'
+  process_time = task.input_data['process_time'] || 0.5
+
+  # Simulate data processing
+  sleep(process_time)
+
+  # Process the data
+  processed = data.upcase
+
+  {
+    original: data,
+    processed: processed,
+    length: processed.length,
+    processed_at: Time.now.iso8601
+  }
+end
+
+# Batch processor worker
+Conductor::Worker.define('async_batch_processor', poll_interval: 100, thread_count: 5) do |task|
+  items = task.input_data['items'] || []
+
+  # Process all items
+  results = items.map do |item|
+    sleep(0.01) # Simulate I/O operation
+    "processed_#{item}"
+  end
+
+  {
+    input_count: items.size,
+    results: results,
+    completed_at: Time.now.iso8601
+  }
+end
+
+# CPU-bound worker
+Conductor::Worker.define('sync_cpu_task', poll_interval: 100, thread_count: 5) do |task|
+  n = task.input_data['n'] || 100_000
+
+  # CPU-bound calculation
+  result = (0...n).sum { |i| i * i }
+
+  { result: result }
+end
+
+def main
+  metrics_port = (ENV['METRICS_PORT'] || 9090).to_i
+
+  # Configure Conductor connection
+  config = Conductor::Configuration.new(
+    server_api_url: ENV['CONDUCTOR_SERVER_URL'] || 'http://localhost:8080/api',
+    auth_key: ENV.fetch('CONDUCTOR_AUTH_KEY', nil),
+    auth_secret: ENV.fetch('CONDUCTOR_AUTH_SECRET', nil)
+  )
+
+  puts '=' * 80
+  puts 'Metrics Collection Example'
+  puts '=' * 80
+  puts
+
+  if PROMETHEUS_AVAILABLE
+    # Create metrics collector with Prometheus backend
+    metrics = Conductor::Worker::Telemetry::MetricsCollector.create(backend: :prometheus)
+
+    # Start metrics HTTP server
+    metrics_server = Conductor::Worker::Telemetry::MetricsServer.new(port: metrics_port)
+    metrics_server.start
+
+    puts 'Metrics mode: Prometheus HTTP'
+    puts "Metrics HTTP endpoint: http://localhost:#{metrics_port}/metrics"
+    puts "Health check: http://localhost:#{metrics_port}/health"
+  else
+    # Fall back to null metrics (logging only)
+    metrics = Conductor::Worker::Telemetry::MetricsCollector.create(backend: :null)
+    metrics_server = nil
+
+    puts 'Metrics mode: Null (prometheus-client gem not installed)'
+    puts 'To enable Prometheus metrics, install: gem install prometheus-client'
+  end
+
+  puts
+  puts 'Workers available:'
+  puts '  - async_http_task: Simulated HTTP requests (I/O-bound)'
+  puts '  - async_data_processor: Data processing'
+  puts '  - async_batch_processor: Batch processing'
+  puts '  - sync_cpu_task: CPU-bound calculations'
+  puts
+  puts 'Try these commands:'
+  puts "  curl http://localhost:#{metrics_port}/metrics"
+  puts "  watch -n 1 'curl -s http://localhost:#{metrics_port}/metrics | grep task_poll_total'"
+  puts
+  puts 'Press Ctrl+C to stop...'
+  puts '=' * 80
+  puts
+
+  begin
+    # Create task handler with metrics enabled
+    handler = Conductor::Worker::TaskHandler.new(
+      configuration: config,
+      scan_for_annotated_workers: true,
+      event_listeners: [metrics]
+    )
+
+    # Handle graceful shutdown
+    shutdown = false
+    Signal.trap('INT') do
+      puts "\nShutting down gracefully..."
+      shutdown = true
+      handler.stop
+    end
+
+    handler.start
+    handler.join unless shutdown
+  rescue Interrupt
+    puts 'Interrupted!'
+  rescue StandardError => e
+    puts "Error: #{e.message}"
+    puts e.backtrace.first(5).join("\n")
+  ensure
+    metrics_server&.stop
+  end
+
+  puts "\nWorkers stopped. Goodbye!"
+end
+
+# Alternative: Custom metrics backend example
+class CustomMetricsBackend
+  def initialize
+    @counters = Hash.new(0)
+    @histograms = Hash.new { |h, k| h[k] = [] }
+    @mutex = Mutex.new
+  end
+
+  def increment(name, labels: {})
+    key = "#{name}:#{labels.sort.to_h}"
+    @mutex.synchronize { @counters[key] += 1 }
+  end
+
+  def observe(name, value, labels: {})
+    key = "#{name}:#{labels.sort.to_h}"
+    @mutex.synchronize { @histograms[key] << value }
+  end
+
+  def set(name, value, labels: {})
+    key = "#{name}:#{labels.sort.to_h}"
+    @mutex.synchronize { @counters[key] = value }
+  end
+
+  def report
+    @mutex.synchronize do
+      puts "\n--- Custom Metrics Report ---"
+      puts 'Counters:'
+      @counters.each { |k, v| puts "  #{k}: #{v}" }
+      puts 'Histograms:'
+      @histograms.each do |k, values|
+        next if values.empty?
+
+        avg = values.sum / values.size
+        puts "  #{k}: count=#{values.size}, avg=#{avg.round(2)}, min=#{values.min.round(2)}, max=#{values.max.round(2)}"
+      end
+      puts '-----------------------------'
+    end
+  end
+end
+
+def main_with_custom_backend
+  # Example using a custom metrics backend
+  config = Conductor::Configuration.new(
+    server_api_url: ENV['CONDUCTOR_SERVER_URL'] || 'http://localhost:8080/api'
+  )
+
+  custom_backend = CustomMetricsBackend.new
+  metrics = Conductor::Worker::Telemetry::MetricsCollector.create(backend: custom_backend)
+
+  puts 'Using custom metrics backend...'
+
+  handler = Conductor::Worker::TaskHandler.new(
+    configuration: config,
+    scan_for_annotated_workers: true,
+    event_listeners: [metrics]
+  )
+
+  # Report metrics periodically
+  reporter = Thread.new do
+    loop do
+      sleep 30
+      custom_backend.report
+    end
+  end
+
+  Signal.trap('INT') do
+    reporter.kill
+    handler.stop
+  end
+
+  handler.start
+  handler.join
+
+  custom_backend.report
+end
+
+if __FILE__ == $PROGRAM_NAME
+  # Run with Prometheus by default
+  # Use main_with_custom_backend for custom backend example
+  main
+end

data/examples/new_dsl_demo.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+# Example demonstrating the new Ruby-idiomatic DSL for Conductor workflows
+#
+# This example shows how to use the new DSL which provides:
+# - Method-per-type approach: simple(), http(), parallel(), etc.
+# - Auto-generated reference names
+# - Hash-style [] for output references
+# - wf[:param] for workflow inputs
+# - Block-based control flow
+
+require 'bundler/setup'
+require 'conductor'
+
+# Define a workflow using the new DSL
+# Note: executor is optional - only needed for .register() and .execute()
+workflow = Conductor.workflow :order_processing_demo, version: 1 do
+  # Access workflow inputs with wf[:param_name]
+  # Returns: "${workflow.input.param_name}"
+  order_id_input = wf[:order_id]
+  user_email_input = wf[:user_email]
+
+  # Simple tasks return TaskRef objects
+  # Access task outputs with task[:field_name]
+  user = simple :get_user, user_id: order_id_input
+  order = simple :validate_order,
+                 order_id: order_id_input,
+                 user_email: user[:email]
+
+  # HTTP tasks with fluent syntax
+  api_response = http :call_payment_api,
+                      url: 'https://api.payment.example.com/charge',
+                      method: :post,
+                      body: {
+                        amount: order[:total],
+                        currency: 'USD',
+                        customer: user[:id]
+                      },
+                      headers: {
+                        'Authorization' => 'Bearer ${workflow.input.api_key}',
+                        'Content-Type' => 'application/json'
+                      }
+
+  # Parallel execution - tasks run concurrently
+  parallel do
+    simple :ship_order,
+           order_id: order[:id],
+           address: order[:shipping_address]
+
+    simple :send_confirmation_email,
+           to: user_email_input,
+           order_id: order[:id]
+
+    simple :update_inventory,
+           product_id: order[:product_id],
+           quantity: order[:quantity]
+  end
+
+  # Switch/decision based on output value
+  decide order[:region] do
+    on 'US' do
+      simple :calculate_us_tax, amount: order[:total]
+      simple :apply_us_shipping
+    end
+
+    on 'EU' do
+      simple :calculate_vat, amount: order[:total]
+      simple :apply_eu_shipping
+    end
+
+    on 'UK' do
+      simple :calculate_uk_tax, amount: order[:total]
+      simple :apply_uk_shipping
+    end
+
+    otherwise do
+      simple :apply_international_shipping
+      terminate :completed, 'International order processed with standard shipping'
+    end
+  end
+
+  # Set workflow variables
+  set(
+    order_status: 'completed',
+    processed_at: '${CPEWF_EPOCH}',
+    processor_id: wf[:worker_id]
+  )
+
+  # Define workflow outputs
+  output(
+    order_id: order[:id],
+    status: 'processed',
+    total: order[:total],
+    payment_result: api_response[:transaction_id]
+  )
+end
+
+# Display workflow information
+puts "Workflow created: #{workflow.name} (version #{workflow.version})"
+puts "Task count: #{workflow.builder.tasks.size}"
+
+# Convert to WorkflowDef to see the generated structure
+workflow_def = workflow.to_workflow_def
+puts "\nGenerated WorkflowDef:"
+puts "  Name: #{workflow_def.name}"
+puts "  Version: #{workflow_def.version}"
+puts "  Tasks: #{workflow_def.tasks.size}"
+puts "\nFirst 3 tasks:"
+workflow_def.tasks.take(3).each_with_index do |task, idx|
+  puts "  #{idx + 1}. #{task.type}: #{task.task_reference_name}"
+end
+
+puts "\nOutput parameters: #{workflow_def.output_parameters.keys.join(', ')}"
+
+puts "\n✓ DSL example completed successfully!"
+puts "\nTo register and execute this workflow, provide an executor:"
+puts <<~EXAMPLE
+  
+  # Create executor with configuration
+  config = Conductor::Configuration.new
+  config.server_url = 'http://localhost:8080/api'
+  executor = Conductor::Workflow::WorkflowExecutor.new(config)
+  
+  # Define workflow with executor
+  workflow = Conductor.workflow :order_processing_demo, version: 1, executor: executor do
+    # ... workflow definition ...
+  end
+  
+  # Register the workflow
+  workflow.register(overwrite: true)
+  
+  # Execute the workflow
+  result = workflow.execute(input: {
+    order_id: '12345',
+    user_email: 'customer@example.com',
+    api_key: 'secret_key',
+    worker_id: 'worker-001'
+  })
+  
+  puts "Workflow status: \#{result.status}"
+EXAMPLE

data/examples/orkes/http_poll.rb

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# HTTP Poll Task Example
+#
+# Demonstrates using HTTP_POLL task to poll an external API
+# until a condition is met or timeout occurs.
+#
+# Usage:
+#   bundle exec ruby examples/orkes/http_poll.rb
+
+require_relative '../../lib/conductor'
+
+include Conductor::Workflow
+
+def create_http_poll_workflow(workflow_client, workflow_executor)
+  workflow = ConductorWorkflow.new(
+    workflow_client,
+    'http_poll_workflow_ruby',
+    version: 1,
+    executor: workflow_executor
+  )
+  workflow.description('Workflow demonstrating HTTP polling')
+
+  # HTTP Poll task - polls until condition is met
+  poll_task = HttpPollTask.new('poll_status', {
+                                 'uri' => 'https://httpbin.org/json',
+                                 'method' => 'GET',
+                                 'connectionTimeOut' => 5000,
+                                 'readTimeOut' => 5000
+                               })
+  poll_task.input('terminalCondition', '$.slideshow != null')
+  poll_task.input('pollingInterval', 2)
+  poll_task.input('pollingStrategy', 'FIXED')
+
+  # Process the result
+  process = SimpleTask.new('process_result', 'process_ref')
+                      .input('data', poll_task.output('response'))
+
+  workflow >> poll_task >> process
+  workflow.output_parameter('result', process.output)
+
+  workflow
+end
+
+def main
+  config = Conductor::Configuration.new
+
+  puts '=' * 70
+  puts 'HTTP Poll Task Example'
+  puts '=' * 70
+  puts "Server: #{config.server_url}"
+  puts
+
+  clients = Conductor::Orkes::OrkesClients.new(config)
+  workflow_executor = clients.get_workflow_executor
+  workflow_client = clients.get_workflow_client
+
+  workflow = create_http_poll_workflow(workflow_client, workflow_executor)
+  workflow_executor.register_workflow(workflow, overwrite: true)
+  puts "Registered workflow: #{workflow.name}"
+
+  puts "\nExecuting HTTP poll workflow..."
+  result = workflow_executor.execute(
+    workflow.name,
+    input: {},
+    wait_for_seconds: 30
+  )
+
+  puts "Status: #{result.status}"
+  puts "Output: #{result.output.inspect}"
+  puts "\nView at: #{config.ui_host}/execution/#{result.workflow_id}"
+end
+
+if __FILE__ == $PROGRAM_NAME
+  begin
+    main
+  rescue Conductor::ApiError => e
+    puts "API Error: #{e.message}"
+  rescue StandardError => e
+    puts "Error: #{e.message}"
+  end
+end

data/examples/orkes/secrets_example.rb

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Secrets Management Example
+#
+# Demonstrates using the SecretClient to manage sensitive data
+# like API keys, passwords, and tokens.
+#
+# Usage:
+#   bundle exec ruby examples/orkes/secrets_example.rb
+
+require_relative '../../lib/conductor'
+
+def main
+  config = Conductor::Configuration.new
+  clients = Conductor::Orkes::OrkesClients.new(config)
+  secret_client = clients.get_secret_client
+
+  puts '=' * 70
+  puts 'Secrets Management Example'
+  puts '=' * 70
+  puts "Server: #{config.server_url}"
+  puts
+
+  secret_name = "test_api_key_ruby_#{Time.now.to_i}"
+
+  begin
+    # Create a secret
+    puts "Creating secret: #{secret_name}"
+    secret_client.put_secret(secret_name, 'super-secret-value-12345')
+    puts 'Secret created successfully'
+
+    # List all secrets (names only - values are never exposed)
+    puts "\nListing all secrets:"
+    secrets = secret_client.list_all_secret_names
+    secrets.first(5).each { |s| puts "  - #{s}" }
+    puts "  ... (#{secrets.length} total)"
+
+    # Check if secret exists
+    exists = secret_client.secret_exists?(secret_name)
+    puts "\nSecret '#{secret_name}' exists: #{exists}"
+
+    # Update the secret
+    puts "\nUpdating secret..."
+    secret_client.put_secret(secret_name, 'updated-secret-value-67890')
+    puts 'Secret updated'
+  ensure
+    # Clean up
+    puts "\nDeleting secret..."
+    begin
+      secret_client.delete_secret(secret_name)
+      puts 'Secret deleted'
+    rescue StandardError => e
+      puts "Could not delete: #{e.message}"
+    end
+  end
+
+  puts "\nSecrets example complete!"
+end
+
+if __FILE__ == $PROGRAM_NAME
+  begin
+    main
+  rescue Conductor::ApiError => e
+    puts "API Error: #{e.message}"
+  rescue StandardError => e
+    puts "Error: #{e.message}"
+  end
+end

data/examples/orkes/wait_for_webhook.rb

@@ -0,0 +1,90 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Wait for Webhook Example
+#
+# Demonstrates using the WAIT_FOR_WEBHOOK task to pause workflow
+# execution until an external webhook is received.
+#
+# Usage:
+#   bundle exec ruby examples/orkes/wait_for_webhook.rb
+
+require_relative '../../lib/conductor'
+
+include Conductor::Workflow
+
+def create_webhook_workflow(workflow_client, workflow_executor)
+  workflow = ConductorWorkflow.new(
+    workflow_client,
+    'webhook_workflow_ruby',
+    version: 1,
+    executor: workflow_executor
+  )
+  workflow.description('Workflow that waits for external webhook')
+
+  # Initial task
+  init = SimpleTask.new('init_process', 'init_ref')
+                   .input('order_id', workflow.input('order_id'))
+
+  # Wait for webhook - pauses until external signal received
+  wait_webhook = WaitForWebhookTask.new('wait_for_payment')
+                                   .input('matches', {
+                                            '$.[?(@.order_id == "${workflow.input.order_id}")]' => true
+                                          })
+
+  # Process after webhook received
+  process = SimpleTask.new('process_payment', 'process_ref')
+                      .input('payment_data', '${wait_for_payment.output}')
+
+  workflow >> init >> wait_webhook >> process
+
+  workflow.output_parameter('result', '${process_ref.output}')
+  workflow
+end
+
+def main
+  config = Conductor::Configuration.new
+
+  puts '=' * 70
+  puts 'Wait for Webhook Example'
+  puts '=' * 70
+  puts "Server: #{config.server_url}"
+  puts
+
+  clients = Conductor::Orkes::OrkesClients.new(config)
+  workflow_executor = clients.get_workflow_executor
+  workflow_client = clients.get_workflow_client
+
+  workflow = create_webhook_workflow(workflow_client, workflow_executor)
+  workflow_executor.register_workflow(workflow, overwrite: true)
+  puts "Registered workflow: #{workflow.name}"
+
+  # Start workflow
+  workflow_id = workflow_executor.start_workflow(
+    Conductor::Http::Models::StartWorkflowRequest.new(
+      name: workflow.name,
+      version: 1,
+      input: { 'order_id' => 'ORD-12345' }
+    )
+  )
+
+  puts "Started workflow: #{workflow_id}"
+  puts
+  puts 'Workflow is now waiting for webhook...'
+  puts
+  puts 'To complete the workflow, send a webhook:'
+  puts "  POST #{config.server_url}/webhook/#{workflow.name}"
+  puts '  Body: {"order_id": "ORD-12345", "payment_status": "completed"}'
+  puts
+  puts "Monitor at: #{config.ui_host}/execution/#{workflow_id}"
+end
+
+if __FILE__ == $PROGRAM_NAME
+  begin
+    main
+  rescue Conductor::ApiError => e
+    puts "API Error: #{e.message}"
+  rescue StandardError => e
+    puts "Error: #{e.message}"
+  end
+end