fractor 0.1.0 → 0.1.1

data/lib/fractor/supervisor.rb

@@ -1,56 +1,91 @@
 # frozen_string_literal: true
 
-require 'thread' # Required for Queue
-
 module Fractor
   # Supervises multiple WrappedRactors, distributes work, and aggregates results.
   class Supervisor
-    attr_reader :work_queue, :workers, :results, :worker_class, :work_class
+    attr_reader :work_queue, :workers, :results, :worker_pools
 
     # Initializes the Supervisor.
-    # - worker_class: The class inheriting from Fractor::Worker (e.g., MyWorker).
-    # - work_class: The class inheriting from Fractor::Work (e.g., MyWork).
-    # - num_workers: The number of Ractors to spawn.
-    def initialize(worker_class:, work_class:, num_workers: 2)
-      unless worker_class < Fractor::Worker
-        raise ArgumentError, "#{worker_class} must inherit from Fractor::Worker"
-      end
-      unless work_class < Fractor::Work
-        raise ArgumentError, "#{work_class} must inherit from Fractor::Work"
+    # - worker_pools: An array of worker pool configurations, each containing:
+    #   - worker_class: The class inheriting from Fractor::Worker (e.g., MyWorker).
+    #   - num_workers: The number of Ractors to spawn for this worker class.
+    # - continuous_mode: Whether to run in continuous mode without expecting a fixed work count.
+    def initialize(worker_pools: [], continuous_mode: false)
+      @worker_pools = worker_pools.map do |pool_config|
+        worker_class = pool_config[:worker_class]
+        num_workers = pool_config[:num_workers] || 2
+
+        raise ArgumentError, "#{worker_class} must inherit from Fractor::Worker" unless worker_class < Fractor::Worker
+
+        {
+          worker_class: worker_class,
+          num_workers: num_workers,
+          workers: [] # Will hold the WrappedRactor instances
+        }
       end
 
-      @worker_class = worker_class
-      @work_class = work_class
       @work_queue = Queue.new
       @results = ResultAggregator.new
-      @num_workers = num_workers
-      @workers = []
+      @workers = [] # Flattened array of all workers across all pools
       @total_work_count = 0 # Track total items initially added
       @ractors_map = {} # Map Ractor object to WrappedRactor instance
+      @continuous_mode = continuous_mode
+      @running = false
+      @work_callbacks = []
+    end
+
+    # Adds a single work item to the queue.
+    # The item must be an instance of Fractor::Work or a subclass.
+    def add_work_item(work)
+      unless work.is_a?(Fractor::Work)
+        raise ArgumentError, "#{work.class} must be an instance of Fractor::Work"
+      end
+
+      @work_queue << work
+      @total_work_count += 1
+      return unless ENV["FRACTOR_DEBUG"]
+
+      puts "Work item added. Initial work count: #{@total_work_count}, Queue size: #{@work_queue.size}"
+    end
+
+    # Alias for better naming
+    alias add_work_item add_work_item
+
+    # Adds multiple work items to the queue.
+    # Each item must be an instance of Fractor::Work or a subclass.
+    def add_work_items(works)
+      works.each do |work|
+        add_work_item(work)
+      end
     end
 
-    # Adds work items to the queue.
-    # Items should be the raw input data, not Work objects yet.
-    def add_work(items)
-      items.each { |item| @work_queue << item }
-      @total_work_count += items.size
-      puts "Work added. Initial work count: #{@total_work_count}, Queue size: #{@work_queue.size}"
+    # Register a callback to provide new work items
+    # The callback should return nil or empty array when no new work is available
+    def register_work_source(&callback)
+      @work_callbacks << callback
     end
 
-    # Starts the worker Ractors.
+    # Starts the worker Ractors for all worker pools.
     def start_workers
-      @workers = (1..@num_workers).map do |i|
-        # Pass the client's worker class (e.g., MyWorker) to WrappedRactor
-        wrapped_ractor = WrappedRactor.new("worker #{i}", @worker_class)
-        wrapped_ractor.start # Start the underlying Ractor
-        # Map the actual Ractor object to the WrappedRactor instance
-        @ractors_map[wrapped_ractor.ractor] = wrapped_ractor if wrapped_ractor.ractor
-        wrapped_ractor
+      @worker_pools.each do |pool|
+        worker_class = pool[:worker_class]
+        num_workers = pool[:num_workers]
+
+        pool[:workers] = (1..num_workers).map do |i|
+          wrapped_ractor = WrappedRactor.new("worker #{worker_class}:#{i}", worker_class)
+          wrapped_ractor.start # Start the underlying Ractor
+          # Map the actual Ractor object to the WrappedRactor instance
+          @ractors_map[wrapped_ractor.ractor] = wrapped_ractor if wrapped_ractor.ractor
+          wrapped_ractor
+        end.compact
       end
-      # Filter out any workers that failed to start properly
-      @workers.compact!
+
+      # Flatten all workers for easier access
+      @workers = @worker_pools.flat_map { |pool| pool[:workers] }
       @ractors_map.compact! # Ensure map doesn't contain nil keys/values
-      puts "Workers started: #{@workers.size} active."
+      return unless ENV["FRACTOR_DEBUG"]
+
+      puts "Workers started: #{@workers.size} active across #{@worker_pools.size} pools."
     end
 
     # Sets up a signal handler for graceful shutdown (Ctrl+C).
@@ -61,12 +96,10 @@ module Fractor
         puts "\nCtrl+C received. Initiating immediate shutdown..."
         puts "Attempting to close worker Ractors..."
         workers_ref.each do |w|
-          begin
-            w.close # Use the close method of WrappedRactor
-            puts "Closed Ractor: #{w.name}"
-          rescue => e
-            puts "Error closing Ractor #{w.name}: #{e.message}"
-          end
+          w.close # Use the close method of WrappedRactor
+          puts "Closed Ractor: #{w.name}"
+        rescue StandardError => e
+          puts "Error closing Ractor #{w.name}: #{e.message}"
         end
         puts "Exiting now."
         exit(1) # Exit immediately
@@ -78,24 +111,48 @@ module Fractor
       setup_signal_handler
       start_workers
 
+      @running = true
       processed_count = 0
-      # Main loop: Process events until the number of results equals the initial work count.
-      while processed_count < @total_work_count
+
+      # Main loop: Process events until conditions are met for termination
+      while @running && (@continuous_mode || processed_count < @total_work_count)
         processed_count = @results.results.size + @results.errors.size
-        puts "Waiting for Ractor results. Processed: #{processed_count}/#{@total_work_count}, Queue size: #{@work_queue.size}"
+
+        if ENV["FRACTOR_DEBUG"]
+          if @continuous_mode
+            puts "Continuous mode: Waiting for Ractor results. Processed: #{processed_count}, Queue size: #{@work_queue.size}"
+          else
+            puts "Waiting for Ractor results. Processed: #{processed_count}/#{@total_work_count}, Queue size: #{@work_queue.size}"
+          end
+        end
 
         # Get active Ractor objects from the map keys
-        # Use keys from ractors_map for the active ractors
         active_ractors = @ractors_map.keys
 
+        # Check for new work from callbacks if in continuous mode and queue is empty
+        if @continuous_mode && @work_queue.empty? && !@work_callbacks.empty?
+          @work_callbacks.each do |callback|
+            new_work = callback.call
+            add_work_items(new_work) if new_work && !new_work.empty?
+          end
+        end
+
         # Break if no active workers and queue is empty, but work remains (indicates potential issue)
-        if active_ractors.empty? && @work_queue.empty? && processed_count < @total_work_count
-           puts "Warning: No active workers and queue is empty, but not all work is processed. Exiting loop."
-           break
+        if active_ractors.empty? && @work_queue.empty? && !@continuous_mode && processed_count < @total_work_count
+          if ENV["FRACTOR_DEBUG"]
+            puts "Warning: No active workers and queue is empty, but not all work is processed. Exiting loop."
+          end
+          break
         end
 
-        # Skip selection if no active ractors are available but loop should continue (e.g., waiting for final results)
-        next if active_ractors.empty?
+        # In continuous mode, just wait if no active ractors but keep running
+        if active_ractors.empty?
+          break unless @continuous_mode
+
+          sleep(0.1) # Small delay to avoid CPU spinning
+          next
+
+        end
 
         # Ractor.select blocks until a message is available from any active Ractor
         ready_ractor_obj, message = Ractor.select(*active_ractors)
@@ -103,47 +160,75 @@ module Fractor
         # Find the corresponding WrappedRactor instance
         wrapped_ractor = @ractors_map[ready_ractor_obj]
         unless wrapped_ractor
-          puts "Warning: Received message from unknown Ractor: #{ready_ractor_obj}. Ignoring."
+          if ENV["FRACTOR_DEBUG"]
+            puts "Warning: Received message from unknown Ractor: #{ready_ractor_obj}. Ignoring."
+          end
           next
         end
 
-        puts "Selected Ractor: #{wrapped_ractor.name}, Message Type: #{message[:type]}"
+        if ENV["FRACTOR_DEBUG"]
+          puts "Selected Ractor: #{wrapped_ractor.name}, Message Type: #{message[:type]}"
+        end
 
         # Process the received message
         case message[:type]
         when :initialize
-          puts "Ractor initialized: #{message[:processor]}"
+          if ENV["FRACTOR_DEBUG"]
+            puts "Ractor initialized: #{message[:processor]}"
+          end
           # Send work immediately upon initialization if available
           send_next_work_if_available(wrapped_ractor)
         when :result
           # The message[:result] should be a WorkResult object
           work_result = message[:result]
-          puts "Completed work: #{work_result.inspect} in Ractor: #{message[:processor]}"
+          if ENV["FRACTOR_DEBUG"]
+            puts "Completed work: #{work_result.inspect} in Ractor: #{message[:processor]}"
+          end
           @results.add_result(work_result)
-          puts "Result processed. Total processed: #{@results.results.size + @results.errors.size}/#{@total_work_count}"
-          puts "Aggregated Results: #{@results.inspect}"
+          if ENV["FRACTOR_DEBUG"]
+            puts "Result processed. Total processed: #{@results.results.size + @results.errors.size}"
+            puts "Aggregated Results: #{@results.inspect}" unless @continuous_mode
+          end
           # Send next piece of work
           send_next_work_if_available(wrapped_ractor)
         when :error
           # The message[:result] should be a WorkResult object containing the error
           error_result = message[:result]
-          puts "Error processing work #{error_result.work&.inspect} in Ractor: #{message[:processor]}: #{error_result.error}"
+          if ENV["FRACTOR_DEBUG"]
+            puts "Error processing work #{error_result.work&.inspect} in Ractor: #{message[:processor]}: #{error_result.error}"
+          end
           @results.add_result(error_result) # Add error to aggregator
-          puts "Error handled. Total processed: #{@results.results.size + @results.errors.size}/#{@total_work_count}"
-          puts "Aggregated Results (including errors): #{@results.inspect}"
+          if ENV["FRACTOR_DEBUG"]
+            puts "Error handled. Total processed: #{@results.results.size + @results.errors.size}"
+            puts "Aggregated Results (including errors): #{@results.inspect}" unless @continuous_mode
+          end
           # Send next piece of work even after an error
           send_next_work_if_available(wrapped_ractor)
         else
-          puts "Unknown message type received: #{message[:type]} from #{wrapped_ractor.name}"
+          if ENV["FRACTOR_DEBUG"]
+            puts "Unknown message type received: #{message[:type]} from #{wrapped_ractor.name}"
+          end
         end
         # Update processed count for the loop condition
         processed_count = @results.results.size + @results.errors.size
       end
 
-      puts "Main loop finished."
+      if ENV["FRACTOR_DEBUG"]
+        puts "Main loop finished."
+      end
+      return if @continuous_mode
+
+      return unless ENV["FRACTOR_DEBUG"]
+
       puts "Final Aggregated Results: #{@results.inspect}"
     end
 
+    # Stop the supervisor (for continuous mode)
+    def stop
+      @running = false
+      puts "Stopping supervisor..."
+    end
+
     private
 
     # Helper method to send the next available work item to a specific Ractor.
@@ -151,21 +236,33 @@ module Fractor
       # Ensure the wrapped_ractor instance is valid and its underlying ractor is not closed
       if wrapped_ractor && !wrapped_ractor.closed?
         if !@work_queue.empty?
-          raw_input = @work_queue.pop # Get raw input data
-          # Create an instance of the client's Work class (e.g., MyWork)
-          work_item = @work_class.new(raw_input)
-          puts "Sending next work #{work_item.inspect} to Ractor: #{wrapped_ractor.name}"
+          work_item = @work_queue.pop # Now directly a Work object
+
+          if ENV["FRACTOR_DEBUG"]
+            puts "Sending next work #{work_item.inspect} to Ractor: #{wrapped_ractor.name}"
+          end
           wrapped_ractor.send(work_item) # Send the Work object
-          puts "Work sent to #{wrapped_ractor.name}."
+          if ENV["FRACTOR_DEBUG"]
+            puts "Work sent to #{wrapped_ractor.name}."
+          end
         else
-          puts "Work queue empty. Not sending new work to Ractor #{wrapped_ractor.name}."
-          # Consider closing the Ractor if the queue is empty and no more work is expected.
-          # wrapped_ractor.close
-          # @ractors_map.delete(wrapped_ractor.ractor)
-          # puts "Closed idle Ractor: #{wrapped_ractor.name}"
+          if ENV["FRACTOR_DEBUG"]
+            puts "Work queue empty. Not sending new work to Ractor #{wrapped_ractor.name}."
+          end
+          # In continuous mode, don't close workers as more work may come
+          unless @continuous_mode
+            # Consider closing the Ractor if the queue is empty and no more work is expected.
+            # wrapped_ractor.close
+            # @ractors_map.delete(wrapped_ractor.ractor)
+            # if ENV["FRACTOR_DEBUG"]
+            #   puts "Closed idle Ractor: #{wrapped_ractor.name}"
+            # end
+          end
         end
       else
-        puts "Attempted to send work to an invalid or closed Ractor: #{wrapped_ractor&.name || 'unknown'}."
+        if ENV["FRACTOR_DEBUG"]
+          puts "Attempted to send work to an invalid or closed Ractor: #{wrapped_ractor&.name || "unknown"}."
+        end
         # Remove from map if found but closed
         @ractors_map.delete(wrapped_ractor.ractor) if wrapped_ractor && @ractors_map.key?(wrapped_ractor.ractor)
       end

data/lib/fractor/version.rb

@@ -2,5 +2,5 @@
 
 module Fractor
   # Fractor version
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/fractor.rb

@@ -1,15 +1,13 @@
 # frozen_string_literal: true
 
-require 'thread' # Required for Queue
-
 # Require all component files
-require_relative 'fractor/version'
-require_relative 'fractor/worker'
-require_relative 'fractor/work'
-require_relative 'fractor/work_result'
-require_relative 'fractor/result_aggregator'
-require_relative 'fractor/wrapped_ractor'
-require_relative 'fractor/supervisor'
+require_relative "fractor/version"
+require_relative "fractor/worker"
+require_relative "fractor/work"
+require_relative "fractor/work_result"
+require_relative "fractor/result_aggregator"
+require_relative "fractor/wrapped_ractor"
+require_relative "fractor/supervisor"
 
 # Fractor: Function-driven Ractors framework
 module Fractor

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fractor
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Ronald Tse
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-05-05 00:00:00.000000000 Z
+date: 2025-05-07 00:00:00.000000000 Z
 dependencies: []
 description: Fractor is a lightweight Ruby framework designed to simplify the process
   of distributing computational work across multiple Ractors.
@@ -20,11 +20,23 @@ extra_rdoc_files: []
 files:
 - ".rspec"
 - ".rubocop.yml"
+- ".rubocop_todo.yml"
 - CODE_OF_CONDUCT.md
 - README.adoc
 - Rakefile
-- examples/hierarchical_hasher.rb
-- examples/producer_subscriber.rb
+- examples/hierarchical_hasher/README.adoc
+- examples/hierarchical_hasher/hierarchical_hasher.rb
+- examples/multi_work_type/README.adoc
+- examples/multi_work_type/multi_work_type.rb
+- examples/pipeline_processing/README.adoc
+- examples/pipeline_processing/pipeline_processing.rb
+- examples/producer_subscriber/README.adoc
+- examples/producer_subscriber/producer_subscriber.rb
+- examples/scatter_gather/README.adoc
+- examples/scatter_gather/scatter_gather.rb
+- examples/simple/sample.rb
+- examples/specialized_workers/README.adoc
+- examples/specialized_workers/specialized_workers.rb
 - lib/fractor.rb
 - lib/fractor/result_aggregator.rb
 - lib/fractor/supervisor.rb
@@ -33,7 +45,6 @@ files:
 - lib/fractor/work_result.rb
 - lib/fractor/worker.rb
 - lib/fractor/wrapped_ractor.rb
-- sample.rb
 - sig/fractor.rbs
 - tests/sample.rb.bak
 - tests/sample_working.rb.bak

data/examples/hierarchical_hasher.rb

@@ -1,158 +0,0 @@
-# frozen_string_literal: true
-
-require 'digest'
-require_relative '../lib/fractor'
-
-module HierarchicalHasher
-  class ChunkWork < Fractor::Work
-    work_type :chunk_hash
-    attr_reader :start, :length, :data
-
-    def initialize(start, length, data)
-      super(nil, { start: start, length: length, data: data })
-      @start = start
-      @length = length
-      @data = data
-      @retry_count = 0
-      @max_retries = 3
-    end
-
-    def should_retry?
-      @retry_count < @max_retries
-    end
-
-    def failed
-      @retry_count += 1
-    end
-  end
-
-  class ChunkResult < Fractor::WorkResult
-    attr_reader :start, :length, :hash_result
-
-    def initialize(work, hash_result)
-      super(work, hash_result)
-      @start = work.start
-      @length = work.length
-      @hash_result = hash_result
-    end
-  end
-
-  class HashWorker < Fractor::Worker
-    work_type_accepted :chunk_hash
-
-    def process_work(work)
-      # Simulate some processing time
-      sleep(rand(0.01..0.05))
-
-      # Calculate SHA-3 hash for the chunk
-      hash = Digest::SHA3.hexdigest(work.data)
-
-      # Return the result
-      ChunkResult.new(work, hash)
-    end
-  end
-
-  class HashResultAssembler < Fractor::ResultAssembler
-    def finalize
-      return nil if @results.empty?
-
-      # Sort results by start position
-      sorted_results = @results.sort_by { |result| result.start }
-
-      # Concatenate all hashes with newlines
-      combined_hash_string = sorted_results.map(&:hash_result).join("\n")
-
-      # Calculate final SHA-3 hash
-      Digest::SHA3.hexdigest(combined_hash_string)
-    end
-  end
-
-  class HashSupervisor < Fractor::Supervisor
-    def initialize(file_path, chunk_size = 1024, worker_count = 4)
-      super()
-      @file_path = file_path
-      @chunk_size = chunk_size
-      @worker_count = worker_count
-      @assembler = HashResultAssembler.new
-
-      # Create a queue for chunk work
-      chunk_queue = Fractor::Queue.new(work_types: [:chunk_hash])
-      add_queue(chunk_queue)
-
-      # Create a worker pool
-      worker_pool = Fractor::Pool.new(size: @worker_count)
-      @worker_count.times do
-        worker_pool.add_worker(HashWorker.new)
-      end
-      add_pool(worker_pool)
-
-      # Load the file and create work chunks
-      load_file_chunks
-    end
-
-    def load_file_chunks
-      File.open(@file_path, 'rb') do |file|
-        start_pos = 0
-
-        while chunk = file.read(@chunk_size)
-          work = ChunkWork.new(start_pos, chunk.length, chunk)
-          @queues.first.push(work)
-          start_pos += chunk.length
-        end
-      end
-    end
-
-    def process_results
-      until @result_queue.empty? && @queues.all?(&:empty?) && @idle_workers == @active_workers.size
-        result_data = next_result
-        next if result_data.nil?
-
-        type, *data = result_data
-
-        case type
-        when :result
-          result = data.first
-          @assembler.add_result(result)
-        when :error
-          work, error = data
-          @assembler.add_failed_work(work, error)
-        end
-
-        # Small sleep to prevent CPU spinning
-        sleep 0.001
-      end
-
-      # Return the final hash
-      @assembler.finalize
-    end
-  end
-end
-
-# Example usage
-if __FILE__ == $PROGRAM_NAME
-  if ARGV.empty?
-    puts "Usage: ruby hierarchical_hasher.rb <file_path> [worker_count]"
-    exit 1
-  end
-
-  file_path = ARGV[0]
-  worker_count = (ARGV[1] || 4).to_i
-
-  unless File.exist?(file_path)
-    puts "Error: File '#{file_path}' not found"
-    exit 1
-  end
-
-  puts "Starting hierarchical hasher with #{worker_count} workers..."
-  puts "Processing file: #{file_path}"
-
-  start_time = Time.now
-  supervisor = HierarchicalHasher::HashSupervisor.new(file_path, 1024, worker_count)
-  final_hash = supervisor.start
-  end_time = Time.now
-
-  puts "Final SHA-3 hash: #{final_hash}"
-  puts "Processing completed in #{end_time - start_time} seconds"
-
-  supervisor.shutdown
-end