fractor 0.1.6 → 0.1.7

data/examples/specialized_workers/specialized_workers.rb

@@ -5,11 +5,14 @@ require_relative "../../lib/fractor"
 module SpecializedWorkers
   # First work type: Compute-intensive operations
   class ComputeWork < Fractor::Work
+    attr_reader :work_type
+
     def initialize(data, operation = :default, parameters = {})
       super({
         data: data,
         operation: operation,
         parameters: parameters,
+        work_type: :compute,  # Add work type identifier for Ractor compatibility
       })
     end
 
@@ -25,6 +28,10 @@ module SpecializedWorkers
       input[:parameters]
     end
 
+    def work_type
+      input[:work_type]
+    end
+
     def to_s
       "ComputeWork: operation=#{operation}, parameters=#{parameters}"
     end
@@ -32,6 +39,8 @@ module SpecializedWorkers
 
   # Second work type: Database operations
   class DatabaseWork < Fractor::Work
+    attr_reader :work_type
+
     def initialize(data = "", query_type = :select, table = "unknown",
 conditions = {})
       super({
@@ -39,6 +48,7 @@ conditions = {})
         query_type: query_type,
         table: table,
         conditions: conditions,
+        work_type: :database,  # Add work type identifier for Ractor compatibility
       })
     end
 
@@ -58,6 +68,10 @@ conditions = {})
       input[:conditions]
     end
 
+    def work_type
+      input[:work_type]
+    end
+
     def to_s
       "DatabaseWork: query_type=#{query_type}, table=#{table}, conditions=#{conditions}"
     end
@@ -65,14 +79,16 @@ conditions = {})
 
   # First worker type: Handles compute-intensive operations
   class ComputeWorker < Fractor::Worker
-    def initialize
+    def initialize(name: nil, **options)
+      super
       # Setup resources needed for computation
-      @compute_resources = { memory: 1024, cpu_cores: 4 }
+      # Use Ractor.make_shareable to make the hash shareable across Ractors
+      @compute_resources = Ractor.make_shareable({ memory: 1024, cpu_cores: 4 })
     end
 
     def process(work)
-      # Only handle ComputeWork
-      unless work.is_a?(ComputeWork)
+      # Only handle ComputeWork - check work_type for Ractor compatibility
+      unless work.respond_to?(:work_type) && work.work_type == :compute
         return Fractor::WorkResult.new(
           error: "ComputeWorker can only process ComputeWork, got: #{work.class}",
           work: work,
@@ -133,14 +149,16 @@ conditions = {})
 
   # Second worker type: Handles database operations
   class DatabaseWorker < Fractor::Worker
-    def initialize
+    def initialize(name: nil, **options)
+      super
       # Setup database connection and resources
-      @db_connection = { pool_size: 5, timeout: 30 }
+      # Use Ractor.make_shareable to make the hash shareable across Ractors
+      @db_connection = Ractor.make_shareable({ pool_size: 5, timeout: 30 })
     end
 
     def process(work)
-      # Only handle DatabaseWork
-      unless work.is_a?(DatabaseWork)
+      # Only handle DatabaseWork - check work_type for Ractor compatibility
+      unless work.respond_to?(:work_type) && work.work_type == :database
         return Fractor::WorkResult.new(
           error: "DatabaseWorker can only process DatabaseWork, got: #{work.class}",
           work: work,
@@ -255,14 +273,20 @@ conditions = {})
       compute_work_items = compute_tasks.map do |task|
         ComputeWork.new(task[:data], task[:operation], task[:parameters])
       end
+      puts "Created #{compute_work_items.size} compute work items"
+      puts "First compute work item: #{compute_work_items.first.inspect}" if compute_work_items.any?
       @compute_supervisor.add_work_items(compute_work_items)
+      puts "Added compute work items to supervisor"
 
       # Create and add database work items
       db_work_items = db_tasks.map do |task|
         DatabaseWork.new(task[:data], task[:query_type], task[:table],
                          task[:conditions])
       end
+      puts "Created #{db_work_items.size} database work items"
+      puts "First db work item: #{db_work_items.first.inspect}" if db_work_items.any?
       @db_supervisor.add_work_items(db_work_items)
+      puts "Added database work items to supervisor"
 
       # Run the supervisors directly - this is more reliable
       @compute_supervisor.run
@@ -273,7 +297,19 @@ conditions = {})
       db_results_agg = @db_supervisor.results
 
       puts "Received compute results: #{compute_results_agg.results.size} items"
+      puts "Received compute errors: #{compute_results_agg.errors.size} items"
+      if compute_results_agg.errors.any?
+        compute_results_agg.errors.each do |e|
+          puts "  Error: #{e.error}"
+        end
+      end
       puts "Received database results: #{db_results_agg.results.size} items"
+      puts "Received database errors: #{db_results_agg.errors.size} items"
+      if db_results_agg.errors.any?
+        db_results_agg.errors.each do |e|
+          puts "  Error: #{e.error}"
+        end
+      end
 
       # Format and store results
       @compute_results = format_compute_results(compute_results_agg)

data/examples/stream_processor/README.adoc

@@ -0,0 +1,206 @@
+= Real-Time Data Stream Processor Example
+:toc:
+:toclevels: 3
+
+Real-time event stream processor using Fractor::ContinuousServer for processing continuous data streams with windowing, metrics tracking, and graceful shutdown.
+
+== Purpose
+
+This example demonstrates:
+
+* Continuous event processing with Fractor::ContinuousServer
+* Sliding window aggregation (5-second windows)
+* Real-time metrics tracking
+* Backpressure handling
+* Graceful shutdown with SIGINT/SIGTERM
+* Performance monitoring
+
+== Features
+
+=== Continuous Processing
+
+Processes events as they arrive without stopping:
+
+* Non-blocking event ingestion
+* Parallel processing across workers
+* Result collection in background thread
+* Automatic workload distribution
+
+=== Sliding Window Aggregation
+
+Maintains a sliding time window of recent events:
+
+* Configurable window size (default: 5 seconds)
+* Automatic cleanup of old events
+* Window-based metrics calculation
+* Real-time window size tracking
+
+=== Performance Metrics
+
+Tracks and displays real-time metrics:
+
+* Total events processed
+* Events per second
+* Average processing time
+* Current window size
+
+=== Graceful Shutdown
+
+Handles termination signals properly:
+
+* SIGINT (Ctrl+C) for manual shutdown
+* SIGTERM for system shutdown
+* Preserves in-flight work
+* Final summary report
+
+== Usage
+
+=== Basic Usage
+
+Run with default settings (30 seconds, 10 events/second):
+
+[source,bash]
+----
+ruby stream_processor.rb
+----
+
+=== Custom Duration
+
+Process for 60 seconds:
+
+[source,bash]
+----
+ruby stream_processor.rb -d 60
+----
+
+=== Custom Event Rate
+
+Generate 50 events per second:
+
+[source,bash]
+----
+ruby stream_processor.rb -r 50
+----
+
+=== Custom Workers
+
+Use 8 worker ractors:
+
+[source,bash]
+----
+ruby stream_processor.rb -w 8
+----
+
+=== Custom Window Size
+
+Use 10-second window:
+
+[source,bash]
+----
+ruby stream_processor.rb --window 10
+----
+
+=== Combined Options
+
+[source,bash]
+----
+ruby stream_processor.rb -w 8 -d 120 -r 100 --window 10
+----
+
+== Examples
+
+=== Example 1: Default Settings
+
+[source,bash]
+----
+$ ruby stream_processor.rb
+
+Starting Stream Processor...
+Window size: 5 seconds
+Workers: 4
+Press Ctrl+C to stop gracefully
+
+Generating event stream for 30 seconds at 10 events/second...
+
+Events: 300 | Processed: 298 | Rate: 9.93 e/s | Window: 47 | Avg Time: 0.12 ms
+
+Generated 300 events
+
+Stopping Stream Processor...
+
+============================================================
+FINAL SUMMARY
+============================================================
+Total Events: 300
+Processed: 300
+Errors: 0
+Duration: 30.25 seconds
+Average Rate: 9.92 events/second
+============================================================
+----
+
+=== Example 2: High Throughput
+
+[source,bash]
+----
+$ ruby stream_processor.rb -w 8 -r 100 -d 10
+
+Starting Stream Processor...
+Window size: 5 seconds
+Workers: 8
+
+Events: 1000 | Processed: 998 | Rate: 99.8 e/s | Window: 492 | Avg Time: 0.08 ms
+
+Generated 1000 events
+
+FINAL SUMMARY
+============================================================
+Total Events: 1000
+Processed: 1000
+Duration: 10.05 seconds
+Average Rate: 99.50 events/second
+----
+
+=== Example 3: Graceful Shutdown
+
+Press Ctrl+C during processing:
+
+[source,bash]
+----
+$ ruby stream_processor.rb
+
+Events: 150 | Processed: 148 | Rate: 10.1 e/s | Window: 48 | Avg Time: 0.10 ms
+^C
+Stopping Stream Processor...
+
+FINAL SUMMARY
+============================================================
+Total Events: 150
+Processed: 150
+Duration: 14.87 seconds
+Average Rate: 10.09 events/second
+----
+
+== Architecture
+
+The processor consists of:
+
+* **Event**: Work item representing a stream event
+* **EventProcessorWorker**: Worker that processes events
+* **StreamProcessor**: Main processor managing the continuous server
+* **EventGenerator**: Test utility for generating event streams
+
+== Testing
+
+Run the test suite:
+
+[source,bash]
+----
+bundle exec rspec spec/examples/stream_processor_spec.rb
+----
+
+== See Also
+
+* link:../../README.adoc[Fractor Main Documentation]
+* link:../../docs/continuous-mode.adoc[Continuous Mode Guide]
+* link:../log_analyzer/README.adoc[Log Analyzer Example]

data/examples/stream_processor/stream_processor.rb

@@ -0,0 +1,284 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "../../lib/fractor"
+require "json"
+require "time"
+
+# Event data structure
+class Event < Fractor::Work
+  def initialize(type:, data:, timestamp: Time.now)
+    super({
+      type: type,
+      data: data,
+      timestamp: timestamp
+    })
+  end
+
+  def type
+    input[:type]
+  end
+
+  def data
+    input[:data]
+  end
+
+  def timestamp
+    input[:timestamp]
+  end
+
+  def to_s
+    "Event(#{type}, #{timestamp.iso8601})"
+  end
+end
+
+# Worker for processing events
+class EventProcessorWorker < Fractor::Worker
+  def process(work)
+    return nil unless work.is_a?(Event)
+
+    # Process event
+    {
+      type: work.type,
+      data: work.data,
+      timestamp: work.timestamp,
+      processed_at: Time.now,
+      processing_time: (Time.now - work.timestamp) * 1000 # ms
+    }
+  end
+end
+
+# Real-time stream processor (simplified for testing)
+class StreamProcessor
+  attr_reader :processed_count, :error_count, :window_size, :metrics
+
+  def initialize(window_size: 5, num_workers: 4)
+    @window_size = window_size
+    @num_workers = num_workers
+    @processed_count = 0
+    @error_count = 0
+    @events_in_window = []
+    @metrics = {
+      total_events: 0,
+      events_per_second: 0.0,
+      average_processing_time: 0.0,
+      current_window_count: 0
+    }
+    @start_time = Time.now
+    @mutex = Mutex.new
+    @supervisor = nil
+    @running = false
+  end
+
+  def start
+    puts "Starting Stream Processor..."
+    puts "Window size: #{@window_size} seconds"
+    puts "Workers: #{@num_workers}"
+    puts
+
+    @supervisor = Fractor::Supervisor.new(
+      worker_pools: [
+        { worker_class: EventProcessorWorker, num_workers: @num_workers }
+      ]
+    )
+
+    @running = true
+    @start_time = Time.now
+
+    self
+  end
+
+  def add_event(event)
+    return unless @supervisor && @running
+
+    @supervisor.add_work_item(event)
+
+    @mutex.synchronize do
+      @metrics[:total_events] += 1
+    end
+  end
+
+  def process_events
+    return unless @supervisor && @running
+
+    @supervisor.run
+
+    results_obj = @supervisor.results
+    all_results = results_obj.results + results_obj.errors
+
+    all_results.each do |work_result|
+      result = work_result.respond_to?(:result) ? work_result.result : work_result
+
+      next unless result.is_a?(Hash)
+
+      process_result(result)
+    end
+  end
+
+  def stop
+    puts "\nStopping Stream Processor..."
+    @running = false
+    print_final_summary
+  end
+
+  private
+
+  def process_result(result)
+    @mutex.synchronize do
+      @processed_count += 1
+
+      # Add to current window
+      @events_in_window << result
+
+      # Remove events outside window
+      cutoff_time = Time.now - @window_size
+      @events_in_window.reject! do |r|
+        r[:processed_at] < cutoff_time
+      end
+
+      # Update metrics
+      @metrics[:current_window_count] = @events_in_window.size
+
+      if @processed_count > 0
+        elapsed = Time.now - @start_time
+        @metrics[:events_per_second] = @processed_count / elapsed
+
+        total_time = @events_in_window.sum { |r| r[:processing_time] }
+        @metrics[:average_processing_time] =
+          @events_in_window.empty? ? 0.0 : total_time / @events_in_window.size
+      end
+    end
+  end
+
+  def print_metrics
+    @mutex.synchronize do
+      print "\r"
+      print "Events: #{@metrics[:total_events]} | "
+      print "Processed: #{@processed_count} | "
+      print "Rate: #{@metrics[:events_per_second].round(2)} e/s | "
+      print "Window: #{@metrics[:current_window_count]} | "
+      print "Avg Time: #{@metrics[:average_processing_time].round(2)} ms"
+      $stdout.flush
+    end
+  end
+
+  def print_final_summary
+    puts "\n"
+    puts "=" * 60
+    puts "FINAL SUMMARY"
+    puts "=" * 60
+    puts format("Total Events: %d", @metrics[:total_events])
+    puts format("Processed: %d", @processed_count)
+    puts format("Errors: %d", @error_count)
+    elapsed = Time.now - @start_time
+    puts format("Duration: %.2f seconds", elapsed)
+    rate = @processed_count > 0 ? @processed_count / elapsed : 0.0
+    puts format("Average Rate: %.2f events/second", rate)
+    puts "=" * 60
+  end
+end
+
+# Event generator for testing
+class EventGenerator
+  def self.generate_stream(processor, duration: 10, rate: 10)
+    puts "Generating event stream for #{duration} seconds at #{rate} events/second..."
+
+    start_time = Time.now
+    event_count = 0
+
+    while (Time.now - start_time) < duration
+      event = Event.new(
+        type: [:click, :view, :purchase, :signup].sample,
+        data: {
+          user_id: rand(1..1000),
+          value: rand(1.0..100.0).round(2)
+        }
+      )
+
+      processor.add_event(event)
+      event_count += 1
+
+      # Control rate
+      sleep(1.0 / rate)
+    end
+
+    puts "\nGenerated #{event_count} events"
+  end
+end
+
+# Run example if executed directly
+if __FILE__ == $PROGRAM_NAME
+  require "optparse"
+
+  options = {
+    workers: 4,
+    window_size: 5,
+    duration: 30,
+    rate: 10
+  }
+
+  OptionParser.new do |opts|
+    opts.banner = "Usage: stream_processor.rb [options]"
+
+    opts.on("-w", "--workers NUM", Integer, "Number of workers (default: 4)") do |n|
+      options[:workers] = n
+    end
+
+    opts.on("--window SIZE", Integer, "Window size in seconds (default: 5)") do |s|
+      options[:window_size] = s
+    end
+
+    opts.on("-d", "--duration SECONDS", Integer, "Test duration (default: 30)") do |d|
+      options[:duration] = d
+    end
+
+    opts.on("-r", "--rate NUM", Integer, "Events per second (default: 10)") do |r|
+      options[:rate] = r
+    end
+
+    opts.on("-h", "--help", "Show this message") do
+      puts opts
+      exit
+    end
+  end.parse!
+
+  # Create processor
+  processor = StreamProcessor.new(
+    window_size: options[:window_size],
+    num_workers: options[:workers]
+  )
+
+  processor.start
+
+  # Handle graceful shutdown
+  trap("INT") do
+    processor.stop
+    exit
+  end
+
+  # Generate test events in background
+  generator_thread = Thread.new do
+    EventGenerator.generate_stream(
+      processor,
+      duration: options[:duration],
+      rate: options[:rate]
+    )
+  end
+
+  # Process events periodically
+  metrics_thread = Thread.new do
+    while processor.instance_variable_get(:@running)
+      sleep(1)
+      processor.send(:print_metrics)
+    end
+  end
+
+  # Wait for generation to complete
+  generator_thread.join
+
+  # Process remaining events
+  processor.process_events
+
+  metrics_thread.kill
+  processor.stop
+end