simple_flow 0.1.0

data/examples/07_real_world_etl.rb

@@ -0,0 +1,277 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/simple_flow'
+require 'json'
+require 'timecop'
+Timecop.travel(Time.local(2001, 9, 11, 7, 0, 0))
+
+# Real-world example: Data ETL (Extract, Transform, Load) pipeline
+
+puts "=" * 60
+puts "Real-World Example: Data ETL Pipeline"
+puts "=" * 60
+puts
+
+# Simulate data sources
+class DataSource
+  def self.fetch_users_csv
+    sleep 0.1
+    [
+      { id: 1, name: "Alice Johnson", email: "alice@example.com", signup_date: "2023-01-15" },
+      { id: 2, name: "Bob Smith", email: "bob@example.com", signup_date: "2023-02-20" },
+      { id: 3, name: "Charlie Brown", email: "CHARLIE@EXAMPLE.COM", signup_date: "2023-03-10" }
+    ]
+  end
+
+  def self.fetch_orders_json
+    sleep 0.1
+    [
+      { order_id: 101, user_id: 1, amount: 150.00, status: "completed" },
+      { order_id: 102, user_id: 2, amount: 75.50, status: "pending" },
+      { order_id: 103, user_id: 1, amount: 200.00, status: "completed" },
+      { order_id: 104, user_id: 3, amount: 50.00, status: "cancelled" }
+    ]
+  end
+
+  def self.fetch_products_api
+    sleep 0.1
+    [
+      { product_id: "A1", name: "Widget", category: "tools" },
+      { product_id: "B2", name: "Gadget", category: "electronics" },
+      { product_id: "C3", name: "Doohickey", category: "tools" }
+    ]
+  end
+end
+
+# Build the ETL pipeline
+etl_pipeline = SimpleFlow::Pipeline.new do
+  # Extract Phase: Load data from multiple sources in parallel
+  step :extract_users, ->(result) {
+    puts "  📥 Extracting users from CSV..."
+    users = DataSource.fetch_users_csv
+    result.with_context(:raw_users, users).continue(result.value)
+  }, depends_on: :none
+
+  step :extract_orders, ->(result) {
+    puts "  📥 Extracting orders from JSON..."
+    orders = DataSource.fetch_orders_json
+    result.with_context(:raw_orders, orders).continue(result.value)
+  }, depends_on: :none
+
+  step :extract_products, ->(result) {
+    puts "  📥 Extracting products from API..."
+    products = DataSource.fetch_products_api
+    result.with_context(:raw_products, products).continue(result.value)
+  }, depends_on: :none
+
+  # Transform Phase: Clean and normalize data in parallel
+  step :transform_users, ->(result) {
+    puts "  🔄 Transforming user data..."
+    raw_users = result.context[:raw_users]
+
+    transformed = raw_users.map do |user|
+      {
+        id: user[:id],
+        name: user[:name].downcase.split.map(&:capitalize).join(' '),
+        email: user[:email].downcase,
+        signup_year: user[:signup_date].split('-').first.to_i,
+        created_at: Time.now
+      }
+    end
+
+    result.with_context(:users, transformed).continue(result.value)
+  }, depends_on: [:extract_users]
+
+  step :transform_orders, ->(result) {
+    puts "  🔄 Transforming order data..."
+    raw_orders = result.context[:raw_orders]
+
+    # Filter out cancelled orders and add computed fields
+    transformed = raw_orders
+      .reject { |o| o[:status] == "cancelled" }
+      .map do |order|
+        {
+          id: order[:order_id],
+          user_id: order[:user_id],
+          amount: order[:amount],
+          status: order[:status].to_sym,
+          tax: (order[:amount] * 0.08).round(2),
+          total: (order[:amount] * 1.08).round(2)
+        }
+      end
+
+    result.with_context(:orders, transformed).continue(result.value)
+  }, depends_on: [:extract_orders]
+
+  step :transform_products, ->(result) {
+    puts "  🔄 Transforming product data..."
+    raw_products = result.context[:raw_products]
+
+    # Normalize and categorize
+    transformed = raw_products.map do |product|
+      {
+        id: product[:product_id],
+        name: product[:name],
+        category: product[:category].to_sym,
+        normalized_name: product[:name].downcase.gsub(/[^a-z0-9]/, '_')
+      }
+    end
+
+    result.with_context(:products, transformed).continue(result.value)
+  }, depends_on: [:extract_products]
+
+  # Aggregate Phase: Join and compute analytics
+  step :aggregate_user_stats, ->(result) {
+    puts "  📊 Aggregating user statistics..."
+    users = result.context[:users]
+    orders = result.context[:orders]
+
+    user_stats = users.map do |user|
+      user_orders = orders.select { |o| o[:user_id] == user[:id] }
+      {
+        user_id: user[:id],
+        name: user[:name],
+        email: user[:email],
+        total_orders: user_orders.size,
+        total_spent: user_orders.sum { |o| o[:total] },
+        avg_order_value: user_orders.size > 0 ? (user_orders.sum { |o| o[:total] } / user_orders.size).round(2) : 0
+      }
+    end
+
+    result.with_context(:user_stats, user_stats).continue(result.value)
+  }, depends_on: [:transform_users, :transform_orders]
+
+  step :aggregate_category_stats, ->(result) {
+    puts "  📊 Aggregating category statistics..."
+    products = result.context[:products]
+
+    category_stats = products
+      .group_by { |p| p[:category] }
+      .transform_values { |prods| { count: prods.size, products: prods.map { |p| p[:name] } } }
+
+    result.with_context(:category_stats, category_stats).continue(result.value)
+  }, depends_on: [:transform_products]
+
+  # Validation Phase: Check data quality
+  step :validate_data, ->(result) {
+    puts "  ✅ Validating data quality..."
+    users = result.context[:users]
+    orders = result.context[:orders]
+
+    issues = []
+
+    # Check for duplicate emails
+    emails = users.map { |u| u[:email] }
+    duplicates = emails.select { |e| emails.count(e) > 1 }.uniq
+    issues << "Duplicate emails found: #{duplicates.join(', ')}" if duplicates.any?
+
+    # Check for orphaned orders
+    user_ids = users.map { |u| u[:id] }
+    orphaned = orders.reject { |o| user_ids.include?(o[:user_id]) }
+    issues << "#{orphaned.size} orphaned orders found" if orphaned.any?
+
+    if issues.any?
+      result.with_context(:validation_warnings, issues).continue(result.value)
+    else
+      result.with_context(:validation_warnings, []).continue(result.value)
+    end
+  }, depends_on: [:aggregate_user_stats]
+
+  # Load Phase: Prepare final output
+  step :prepare_output, ->(result) {
+    puts "  💾 Preparing output..."
+
+    output = {
+      metadata: {
+        processed_at: Time.now,
+        pipeline_version: "1.0",
+        warnings: result.context[:validation_warnings]
+      },
+      users: result.context[:users],
+      orders: result.context[:orders],
+      products: result.context[:products],
+      analytics: {
+        user_stats: result.context[:user_stats],
+        category_stats: result.context[:category_stats],
+        summary: {
+          total_users: result.context[:users].size,
+          total_orders: result.context[:orders].size,
+          total_products: result.context[:products].size,
+          total_revenue: result.context[:orders].sum { |o| o[:total] }.round(2)
+        }
+      }
+    }
+
+    result.continue(output)
+  }, depends_on: [:validate_data, :aggregate_category_stats]
+end
+
+puts "\nStarting ETL pipeline..."
+puts "=" * 60
+puts
+
+start_time = Time.now
+result = etl_pipeline.call_parallel(SimpleFlow::Result.new({}))
+elapsed = Time.now - start_time
+
+puts "\n" + "=" * 60
+if result.continue?
+  puts "✅ ETL Pipeline completed successfully!"
+  puts "=" * 60
+
+  output = result.value
+
+  puts "\nMetadata:"
+  puts "  Processed at: #{output[:metadata][:processed_at]}"
+  puts "  Pipeline version: #{output[:metadata][:pipeline_version]}"
+  if output[:metadata][:warnings].any?
+    puts "  Warnings: #{output[:metadata][:warnings].join('; ')}"
+  end
+
+  puts "\nData Summary:"
+  puts "  Users processed: #{output[:analytics][:summary][:total_users]}"
+  puts "  Orders processed: #{output[:analytics][:summary][:total_orders]}"
+  puts "  Products processed: #{output[:analytics][:summary][:total_products]}"
+  puts "  Total revenue: $#{output[:analytics][:summary][:total_revenue]}"
+
+  puts "\nUser Statistics:"
+  output[:analytics][:user_stats].each do |stat|
+    puts "  #{stat[:name]} (#{stat[:email]})"
+    puts "    Orders: #{stat[:total_orders]}, Spent: $#{stat[:total_spent]}, Avg: $#{stat[:avg_order_value]}"
+  end
+
+  puts "\nCategory Statistics:"
+  output[:analytics][:category_stats].each do |category, stats|
+    puts "  #{category}: #{stats[:count]} products (#{stats[:products].join(', ')})"
+  end
+
+  puts "\nProcessing time: #{(elapsed * 1000).round(2)}ms"
+
+  # Show dependency graph execution
+  puts "\nExecution Flow:"
+  puts "  Phase 1 (Extract): users, orders, products (parallel)"
+  puts "  Phase 2 (Transform): transform_users, transform_orders, transform_products (parallel)"
+  puts "  Phase 3 (Aggregate): user_stats, category_stats (parallel after transforms)"
+  puts "  Phase 4 (Validate): data validation"
+  puts "  Phase 5 (Load): prepare output"
+
+  # Optionally save to file
+  if ARGV.include?("--save")
+    filename = "etl_output_#{Time.now.to_i}.json"
+    File.write(filename, JSON.pretty_generate(output))
+    puts "\n✅ Output saved to #{filename}"
+  end
+else
+  puts "❌ ETL Pipeline failed"
+  puts "=" * 60
+  puts "\nErrors:"
+  result.errors.each do |category, messages|
+    puts "  #{category}: #{messages.join(', ')}"
+  end
+end
+
+puts "\n" + "=" * 60
+puts "ETL example completed!"
+puts "Run with --save flag to save output to JSON file"
+puts "=" * 60

data/examples/08_graph_visualization.rb

@@ -0,0 +1,246 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/simple_flow'
+require 'timecop'
+Timecop.travel(Time.local(2001, 9, 11, 7, 0, 0))
+
+# Graph visualization examples
+
+puts "=" * 60
+puts "Dependency Graph Visualization"
+puts "=" * 60
+puts
+
+# Example 1: Simple graph
+puts "Example 1: Simple Dependency Graph"
+puts "-" * 60
+puts
+
+simple_graph = SimpleFlow::DependencyGraph.new(
+  step_a: [],
+  step_b: [:step_a],
+  step_c: [:step_a],
+  step_d: [:step_b, :step_c]
+)
+
+visualizer = SimpleFlow::DependencyGraphVisualizer.new(simple_graph)
+
+puts visualizer.to_ascii
+puts
+
+# Example 2: Complex real-world graph
+puts "\n" + "=" * 60
+puts "Example 2: E-commerce Order Processing Graph"
+puts "=" * 60
+puts
+
+ecommerce_graph = SimpleFlow::DependencyGraph.new(
+  validate_order: [],
+  check_inventory: [:validate_order],
+  calculate_shipping: [:validate_order],
+  calculate_totals: [:check_inventory, :calculate_shipping],
+  process_payment: [:calculate_totals],
+  reserve_inventory: [:process_payment],
+  create_shipment: [:reserve_inventory],
+  send_email: [:create_shipment],
+  send_sms: [:create_shipment],
+  finalize_order: [:send_email, :send_sms]
+)
+
+ecommerce_visualizer = SimpleFlow::DependencyGraphVisualizer.new(ecommerce_graph)
+
+puts ecommerce_visualizer.to_ascii
+puts
+
+# Example 3: Execution plan
+puts "\n" + "=" * 60
+puts "Example 3: Execution Plan"
+puts "=" * 60
+puts
+
+puts ecommerce_visualizer.to_execution_plan
+puts
+
+# Example 4: ETL Pipeline graph
+puts "\n" + "=" * 60
+puts "Example 4: ETL Pipeline Graph"
+puts "=" * 60
+puts
+
+etl_graph = SimpleFlow::DependencyGraph.new(
+  extract_users: [],
+  extract_orders: [],
+  extract_products: [],
+  transform_users: [:extract_users],
+  transform_orders: [:extract_orders],
+  transform_products: [:extract_products],
+  aggregate_user_stats: [:transform_users, :transform_orders],
+  aggregate_category_stats: [:transform_products],
+  validate_data: [:aggregate_user_stats],
+  prepare_output: [:validate_data, :aggregate_category_stats]
+)
+
+etl_visualizer = SimpleFlow::DependencyGraphVisualizer.new(etl_graph)
+
+puts etl_visualizer.to_execution_plan
+puts
+
+# Example 5: Export formats
+puts "\n" + "=" * 60
+puts "Example 5: Exporting to Different Formats"
+puts "=" * 60
+puts
+
+# Export to Graphviz DOT format
+dot_output = ecommerce_visualizer.to_dot(include_groups: true, orientation: 'TB')
+File.write('ecommerce_graph.dot', dot_output)
+puts "✓ Exported to Graphviz DOT format: ecommerce_graph.dot"
+puts "  To generate PNG: dot -Tpng ecommerce_graph.dot -o ecommerce_graph.png"
+puts "  To generate SVG: dot -Tsvg ecommerce_graph.dot -o ecommerce_graph.svg"
+puts
+
+# Export to Mermaid format
+mermaid_output = ecommerce_visualizer.to_mermaid
+File.write('ecommerce_graph.mmd', mermaid_output)
+puts "✓ Exported to Mermaid format: ecommerce_graph.mmd"
+puts "  View at: https://mermaid.live/"
+puts
+
+# Export to HTML
+html_output = ecommerce_visualizer.to_html(title: "E-commerce Order Processing Graph")
+File.write('ecommerce_graph.html', html_output)
+puts "✓ Exported to interactive HTML: ecommerce_graph.html"
+puts "  Open in browser to view interactive graph"
+puts
+
+# Show the DOT format
+puts "\nGraphviz DOT Format Preview:"
+puts "-" * 60
+puts dot_output.lines.take(20).join
+puts "... (truncated)"
+puts
+
+# Show the Mermaid format
+puts "\nMermaid Format Preview:"
+puts "-" * 60
+puts mermaid_output.lines.take(15).join
+puts "... (truncated)"
+puts
+
+# Example 6: Visualizing a Pipeline (Direct Method - RECOMMENDED)
+puts "\n" + "=" * 60
+puts "Example 6: Visualizing a Pipeline Directly"
+puts "=" * 60
+puts
+
+pipeline = SimpleFlow::Pipeline.new do
+  step :fetch_config, ->(result) {
+    result.with_context(:config, {}).continue(result.value)
+  }, depends_on: :none
+
+  step :load_data, ->(result) {
+    result.with_context(:data, []).continue(result.value)
+  }, depends_on: [:fetch_config]
+
+  step :validate_schema, ->(result) {
+    result.continue(result.value)
+  }, depends_on: [:load_data]
+
+  step :enrich_data, ->(result) {
+    result.continue(result.value)
+  }, depends_on: [:load_data]
+
+  step :save_results, ->(result) {
+    result.continue(result.value)
+  }, depends_on: [:validate_schema, :enrich_data]
+end
+
+# RECOMMENDED: Visualize directly from the pipeline
+puts "Pipeline Dependency Graph:"
+puts
+puts pipeline.visualize_ascii
+puts
+
+# Alternative (manual approach - not recommended):
+# pipeline_graph = SimpleFlow::DependencyGraph.new(pipeline.step_dependencies)
+# pipeline_visualizer = SimpleFlow::DependencyGraphVisualizer.new(pipeline_graph)
+# puts pipeline_visualizer.to_ascii
+
+# Example 7: Comparing different graph structures
+puts "\n" + "=" * 60
+puts "Example 7: Graph Structure Comparison"
+puts "=" * 60
+puts
+
+# Linear pipeline (no parallelism)
+linear_graph = SimpleFlow::DependencyGraph.new(
+  step1: [],
+  step2: [:step1],
+  step3: [:step2],
+  step4: [:step3],
+  step5: [:step4]
+)
+
+# Fan-out/fan-in (maximum parallelism)
+fanout_graph = SimpleFlow::DependencyGraph.new(
+  start: [],
+  task1: [:start],
+  task2: [:start],
+  task3: [:start],
+  task4: [:start],
+  end: [:task1, :task2, :task3, :task4]
+)
+
+puts "Linear Pipeline (Sequential):"
+puts SimpleFlow::DependencyGraphVisualizer.new(linear_graph).to_execution_plan
+puts
+
+puts "\nFan-out/Fan-in Pipeline (Parallel):"
+puts SimpleFlow::DependencyGraphVisualizer.new(fanout_graph).to_execution_plan
+puts
+
+# Example 8: Graph statistics
+puts "\n" + "=" * 60
+puts "Example 8: Graph Analytics"
+puts "=" * 60
+puts
+
+def analyze_graph(graph, name)
+  parallel_groups = graph.parallel_order
+  total_steps = graph.dependencies.size
+  max_parallel = parallel_groups.map(&:size).max
+
+  puts "#{name}:"
+  puts "  Total steps: #{total_steps}"
+  puts "  Execution phases: #{parallel_groups.size}"
+  puts "  Max parallel steps: #{max_parallel}"
+  puts "  Theoretical speedup: #{(total_steps.to_f / parallel_groups.size).round(2)}x"
+  puts "  Parallelization ratio: #{((max_parallel.to_f / total_steps) * 100).round(1)}%"
+  puts
+end
+
+analyze_graph(simple_graph, "Simple Graph")
+analyze_graph(ecommerce_graph, "E-commerce Graph")
+analyze_graph(etl_graph, "ETL Graph")
+analyze_graph(linear_graph, "Linear Graph")
+analyze_graph(fanout_graph, "Fan-out/Fan-in Graph")
+
+puts "=" * 60
+puts "Graph visualization examples completed!"
+puts
+puts "Generated files:"
+puts "  - ecommerce_graph.dot (Graphviz format)"
+puts "  - ecommerce_graph.mmd (Mermaid format)"
+puts "  - ecommerce_graph.html (Interactive HTML)"
+puts
+puts "To generate images with Graphviz:"
+puts "  $ dot -Tpng ecommerce_graph.dot -o ecommerce_graph.png"
+puts "  $ dot -Tsvg ecommerce_graph.dot -o ecommerce_graph.svg"
+puts "  $ dot -Tpdf ecommerce_graph.dot -o ecommerce_graph.pdf"
+puts
+puts "To view Mermaid diagram:"
+puts "  1. Visit https://mermaid.live/"
+puts "  2. Paste contents of ecommerce_graph.mmd"
+puts "  3. Or use Mermaid CLI: mmdc -i ecommerce_graph.mmd -o graph.png"
+puts "=" * 60