fractor 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e137bd644ad72869b5ac03f1afcfad823c8880bd31d148fca23afd990bb7647f
-  data.tar.gz: b5c4e6a60e809f4da86be51a7b606dbd91b357d6c20ed726b170f5ca44c5d54c
+  metadata.gz: b9d15481939349c5d4ad4f3b09368d0221b690cf06a737007b5f247a20cda6e6
+  data.tar.gz: 66aca66a7c4b1ac1559a77fa97d20917bd52b3a9cf3c4da04906f2a6295ded75
 SHA512:
-  metadata.gz: 67f40bc03a9b7bb2ca6c3c3c6fa98a4f692ef49c35608090fdafd06566d689d23085dead346e8560317d03179062f16405b0fd0e66ac5335ef941ee46d41396c
-  data.tar.gz: 01c3622f2ef6e5d0bf55b76c4d7ec6beecd76f094110cc4ea22dd0916ea765aec387d55ea563d29aeaa4955c7f97d6f7756b1186b82b22ca2bc8fae47bbeaa9f
+  metadata.gz: 205fbcb518ea078314f5964e3e60a61212c95e3f2959bb3420e6a060bb17f8bbcef0fbd7a74e27ec5304718ead2c8defd41591f4707757f7ce25bcb22374d0c0
+  data.tar.gz: c66a1af867247746dfd5d63bd30348ae0cf18bb13186b549ccd1380eb3829dd4aa662f4dc359a9479d7666f418cb0e586e6d0b0caf0b74c45f17f29282048326

data/.rubocop.yml

@@ -1,3 +1,5 @@
+inherit_from: .rubocop_todo.yml
+
 AllCops:
   TargetRubyVersion: 3.0
 

data/.rubocop_todo.yml

@@ -0,0 +1,82 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2025-05-06 11:12:48 UTC using RuboCop version 1.75.5.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 3
+# Configuration parameters: AllowedMethods.
+# AllowedMethods: enums
+Lint/ConstantDefinitionInBlock:
+  Exclude:
+    - 'spec/fractor/integration_spec.rb'
+    - 'spec/fractor/work_spec.rb'
+
+# Offense count: 1
+Lint/HashCompareByIdentity:
+  Exclude:
+    - 'examples/producer_subscriber/producer_subscriber.rb'
+
+# Offense count: 2
+# Configuration parameters: AllowedParentClasses.
+Lint/MissingSuper:
+  Exclude:
+    - 'examples/specialized_workers/specialized_workers.rb'
+
+# Offense count: 3
+Lint/RescueException:
+  Exclude:
+    - 'lib/fractor/wrapped_ractor.rb'
+
+# Offense count: 15
+# Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes.
+Metrics/AbcSize:
+  Max: 83
+
+# Offense count: 8
+# Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
+# AllowedMethods: refine
+Metrics/BlockLength:
+  Max: 78
+
+# Offense count: 2
+# Configuration parameters: CountComments, CountAsOne.
+Metrics/ClassLength:
+  Max: 155
+
+# Offense count: 3
+# Configuration parameters: AllowedMethods, AllowedPatterns.
+Metrics/CyclomaticComplexity:
+  Max: 25
+
+# Offense count: 32
+# Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
+Metrics/MethodLength:
+  Max: 60
+
+# Offense count: 2
+# Configuration parameters: AllowedMethods, AllowedPatterns.
+Metrics/PerceivedComplexity:
+  Max: 25
+
+# Offense count: 1
+Security/Eval:
+  Exclude:
+    - 'examples/multi_work_type/multi_work_type.rb'
+
+# Offense count: 1
+# Configuration parameters: AllowedConstants.
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'examples/hierarchical_hasher/hierarchical_hasher.rb'
+
+# Offense count: 8
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns, SplitStrings.
+# URISchemes: http, https
+Layout/LineLength:
+  Max: 160

data/README.adoc

@@ -181,7 +181,7 @@ Handles graceful shutdown on `SIGINT` (Ctrl+C).
 
 
 
-== Quick start guide
+== Quick start
 
 === General
 
@@ -198,16 +198,18 @@ encapsulates the input data needed for processing.
 require 'fractor'
 
 class MyWork < Fractor::Work
-  # The base class already provides input storage and basic functionality
-  # You can optionally override to_s for better debugging
+  # Store all properties in the input hash
+  def initialize(value)
+    super({ value: value })
+  end
 
-  def initialize(input)
-    super # This stores input in @input
-    # Add any additional initialization or replace @input with your own logic
+  # Accessor method for the stored value
+  def value
+    input[:value]
   end
 
   def to_s
-    "MyWork: #{@input}"
+    "MyWork: #{value}"
   end
 end
 ----
@@ -257,28 +259,43 @@ returns an error result.
 The Supervisor class orchestrates the entire framework, managing worker Ractors,
 distributing work, and collecting results.
 
-It initializes a pool of Ractors, each running an instance of the Worker
+It initializes pools of Ractors, each running an instance of a Worker
 class. The Supervisor handles the communication between the main thread and
 the Ractors, including sending work items and receiving results.
 
 The Supervisor also manages the work queue and the ResultAggregator, which
 collects and organizes all results from the workers.
 
-To set up the Supervisor, you need to specify the Worker and Work classes you
-created earlier. You can also specify the number of parallel Ractors to use.
-The default is 2, but you can increase this for more parallelism.
+To set up the Supervisor, you specify worker pools, each containing a Worker class
+and the number of workers to create. You can create multiple worker pools with
+different worker types to handle different kinds of work. Each worker pool can
+process any type of Work object that inherits from Fractor::Work.
 
 [source,ruby]
 ----
 # Create the supervisor
 supervisor = Fractor::Supervisor.new(
-  worker_class: MyWorker,
-  work_class: MyWork,
-  num_workers: 4  # Number of parallel Ractors
+  worker_pools: [
+    { worker_class: MyWorker, num_workers: 4 }  # One pool with 4 workers
+  ]
 )
 
-# Add work items (raw data)
-supervisor.add_work([1, 2, 3, 4, 5].map { |i| MyWork.new(i) })
+# Add individual work items (instances of Work subclasses)
+supervisor.add_work_item(MyWork.new(1))
+
+# Add multiple work items
+supervisor.add_work_items([
+  MyWork.new(2),
+  MyWork.new(3),
+  MyWork.new(4),
+  MyWork.new(5)
+])
+
+# You can add different types of Work objects to the same supervisor
+supervisor.add_work_items([
+  MyWork.new(6),
+  OtherWork.new("data")
+])
 
 # Run the processing
 supervisor.run
@@ -292,7 +309,7 @@ That's it! With these three simple steps, you have a working parallel processing
 system using Fractor.
 
 
-== Detailed guides
+== Usage
 
 === Work class
 
@@ -345,9 +362,11 @@ end
 
 [TIP]
 ====
+====
 * Keep Work objects lightweight and serializable since they will be passed
   between Ractors
 * Implement a meaningful `to_s` method for better debugging
+====
 * Consider adding validation in the initializer to catch issues early
 ====
 
@@ -415,7 +434,7 @@ def process(work)
 end
 ----
 
-=== Unexpected errors caught by rescue
+===== Unexpected errors caught by rescue
 
 These are unexpected exceptions that may occur during processing. You should
 catch these and convert them into error results.
@@ -434,10 +453,12 @@ end
 ----
 
 [TIP]
+====
 * Keep the `process` method focused on a single responsibility
 * Use meaningful error messages that help diagnose issues
 * Consider adding logging within the `process` method for debugging
 * Ensure all paths return a valid `WorkResult` object
+====
 
 === WorkResult class
 
@@ -572,6 +593,7 @@ The WrappedRactor handles error propagation in two ways:
   yielded back
 . Unexpected errors in the Ractor itself are caught and logged
 
+
 === Supervisor class
 
 ==== Purpose and responsibilities
@@ -586,9 +608,14 @@ When creating a Supervisor, you can configure:
 [source,ruby]
 ----
 supervisor = Fractor::Supervisor.new(
-  worker_class: MyWorker,     # Required: Your Worker subclass
-  work_class: MyWork,         # Required: Your Work subclass
-  num_workers: 4              # Optional: Number of Ractors (default: 2)
+  worker_pools: [
+    # Pool 1 - for general data processing
+    { worker_class: MyWorker, num_workers: 4 },
+
+    # Pool 2 - for specialized image processing
+    { worker_class: ImageWorker, num_workers: 2 }
+  ],
+  continuous_mode: false      # Optional: Run in continuous mode (default: false)
 )
 ----
 
@@ -599,18 +626,27 @@ You can add work items individually or in batches:
 [source,ruby]
 ----
 # Add a single item
-supervisor.add_work([42])
+supervisor.add_work_item(MyWork.new(42))
 
 # Add multiple items
-supervisor.add_work([1, 2, 3, 4, 5])
+supervisor.add_work_items([
+  MyWork.new(1),
+  MyWork.new(2),
+  MyWork.new(3),
+  MyWork.new(4),
+  MyWork.new(5)
+])
 
-# Add complex items
-supervisor.add_work([
-  {id: 1, data: "foo"},
-  {id: 2, data: "bar"}
+# Add items of different work types
+supervisor.add_work_items([
+  TextWork.new("Process this text"),
+  ImageWork.new({ width: 800, height: 600 })
 ])
 ----
 
+The Supervisor can handle any Work object that inherits from Fractor::Work.
+Workers must check the type of Work they receive and process it accordingly.
+
 ==== Running and monitoring
 
 To start processing:
@@ -628,6 +664,7 @@ The Supervisor automatically handles:
 * Collecting results and errors
 * Graceful shutdown on completion or interruption (Ctrl+C)
 
+
 ==== Accessing results
 
 After processing completes:
@@ -652,33 +689,42 @@ aggregator.errors.each do |error_result|
 end
 ----
 
-==== Advanced usage patterns
+== Advanced usage patterns
 
-===== Custom work distribution
+=== Custom work distribution
 
 For more complex scenarios, you might want to prioritize certain work items:
 
 [source,ruby]
 ----
+# Create Work objects for high priority items
+high_priority_works = high_priority_items.map { |item| MyWork.new(item) }
+
 # Add high-priority items first
-supervisor.add_work(high_priority_items)
+supervisor.add_work_items(high_priority_works)
 
 # Run with just enough workers for high-priority items
 supervisor.run
 
+# Create Work objects for lower priority items
+low_priority_works = low_priority_items.map { |item| MyWork.new(item) }
+
 # Add and process lower-priority items
-supervisor.add_work(low_priority_items)
+supervisor.add_work_items(low_priority_works)
 supervisor.run
 ----
 
-===== Handling large datasets
+=== Handling large datasets
 
 For very large datasets, consider processing in batches:
 
 [source,ruby]
 ----
 large_dataset.each_slice(1000) do |batch|
-  supervisor.add_work(batch)
+  # Convert batch items to Work objects
+  work_batch = batch.map { |item| MyWork.new(item) }
+
+  supervisor.add_work_items(work_batch)
   supervisor.run
 
   # Process this batch's results before continuing
@@ -686,11 +732,13 @@ large_dataset.each_slice(1000) do |batch|
 end
 ----
 
-== Running the example
+
+== Running a basic example
 
 . Install the gem as described in the Installation section.
 
-. Create a new Ruby file (e.g., `my_fractor_example.rb`) with your implementation:
+. Create a new Ruby file (e.g., `my_fractor_example.rb`) with your
+implementation:
 
 [source,ruby]
 ----
@@ -717,15 +765,18 @@ class MyWorker < Fractor::Worker
   end
 end
 
-# Create supervisor
+# Create supervisor with a worker pool
 supervisor = Fractor::Supervisor.new(
-  worker_class: MyWorker,
-  work_class: MyWork,
-  num_workers: 2
+  worker_pools: [
+    { worker_class: MyWorker, num_workers: 2 }
+  ]
 )
 
-# Add work items (1..10)
-supervisor.add_work((1..10).to_a)
+# Create Work objects
+work_items = (1..10).map { |i| MyWork.new(i) }
+
+# Add work items
+supervisor.add_work_items(work_items)
 
 # Run processing
 supervisor.run
@@ -747,6 +798,195 @@ the final aggregated results, including any errors encountered. Press `Ctrl+C`
 during execution to test the graceful shutdown.
 
 
+== Continuous mode
+
+=== General
+
+Fractor provides a powerful feature called "continuous mode" that allows
+supervisors to run indefinitely, processing work items as they arrive without
+stopping after the initial work queue is empty.
+
+=== Features
+
+* *Non-stopping Execution*: Supervisors run indefinitely until explicitly stopped
+* *On-demand Work*: Workers only process work when it's available
+* *Resource Efficiency*: Workers idle when no work is available, without consuming excessive resources
+* *Dynamic Work Addition*: New work can be added at any time through the work source callback
+* *Graceful Shutdown*: Resources are properly cleaned up when the supervisor is stopped
+
+Continuous mode is particularly useful for:
+
+* *Chat servers*: Processing incoming messages as they arrive
+* *Background job processors*: Handling tasks from a job queue
+* *Real-time data processing*: Analyzing data streams as they come in
+* *Web servers*: Responding to incoming requests in parallel
+* *Monitoring systems*: Continuously checking system statuses
+
+See the Chat Server example in the examples directory for a complete implementation of continuous mode.
+
+
+=== Using continuous mode
+
+==== Step 1. Create a supervisor with the `continuous_mode: true` option
+
+[source,ruby]
+----
+supervisor = Fractor::Supervisor.new(
+  worker_pools: [
+    { worker_class: MyWorker, num_workers: 2 }
+  ],
+  continuous_mode: true  # Enable continuous mode
+)
+----
+
+==== Step 2. Register a work source callback that provides new work on demand
+
+[source,ruby]
+----
+supervisor.register_work_source do
+  # Return nil or empty array if no work is available
+  # Return a work item or array of work items when available
+  items = get_next_work_items
+  if items && !items.empty?
+    # Convert to Work objects if needed
+    items.map { |item| MyWork.new(item) }
+  else
+    nil
+  end
+end
+----
+
+==== Step 4. Run the supervisor in a non-blocking way
+
+Typically in a background thread.
+
+[source,ruby]
+----
+supervisor_thread = Thread.new { supervisor.run }
+----
+
+==== Step 4. Explicitly call `stop` on the supervisor to stop processing
+
+[source,ruby]
+----
+supervisor.stop
+supervisor_thread.join  # Wait for the supervisor thread to finish
+----
+
+
+
+== Example applications
+
+=== General
+
+The Fractor gem comes with several example applications that demonstrate various
+patterns and use cases. Each example can be found in the `examples` directory of
+the gem repository. Detailed descriptions for these are provided below.
+
+=== Simple example
+
+The Simple Example (link:examples/simple/[examples/simple/]) demonstrates the
+basic usage of the Fractor framework. It shows how to create a simple Work
+class, a Worker class, and a Supervisor to manage the processing of work items
+in parallel. This example serves as a starting point for understanding how to
+use Fractor.
+
+Key features:
+
+* Basic Work and Worker class implementation
+* Simple Supervisor setup
+* Parallel processing of work items
+* Error handling and result aggregation
+* Graceful shutdown on completion
+
+=== Hierarchical hasher
+
+The Hierarchical Hasher example
+(link:examples/hierarchical_hasher/[examples/hierarchical_hasher/]) demonstrates
+how to use the Fractor framework to process a file in parallel by breaking it
+into chunks, hashing each chunk independently, and then combining the results
+into a final hash. This approach is useful for processing large files
+efficiently.
+
+Key features:
+
+* Parallel data chunking for large files
+* Independent processing of data segments
+* Aggregation of results to form a final output
+
+=== Multi-work type
+
+The Multi-Work Type example
+(link:examples/multi_work_type/[examples/multi_work_type/]) demonstrates how a
+single Fractor supervisor and worker can handle multiple types of work items
+(e.g., `TextWork` and `ImageWork`). The worker intelligently adapts its
+processing strategy based on the class of the incoming work item.
+
+Key features:
+
+* Support for multiple `Fractor::Work` subclasses
+* Polymorphic worker processing based on work type
+* Unified workflow for diverse tasks
+
+=== Pipeline processing
+
+The Pipeline Processing example
+(link:examples/pipeline_processing/[examples/pipeline_processing/]) implements a
+multi-stage processing pipeline where data flows sequentially through a series
+of transformations. The output of one stage becomes the input for the next, and
+different stages can operate concurrently on different data items.
+
+Key features:
+
+* Sequential data flow through multiple processing stages
+* Concurrent execution of different pipeline stages
+* Data transformation at each step of the pipeline
+
+=== Producer/subscriber
+
+The Producer/Subscriber example
+(link:examples/producer_subscriber/[examples/producer_subscriber/]) showcases a
+multi-stage document processing system where initial work (processing a
+document) can generate additional sub-work items (processing sections of the
+document). This creates a hierarchical processing pattern.
+
+Key features:
+
+* Implementation of producer-consumer patterns
+* Dynamic generation of sub-work based on initial processing
+* Construction of hierarchical result structures
+
+=== Scatter/gather
+
+The Scatter/Gather example
+(link:examples/scatter_gather/[examples/scatter_gather/]) illustrates how a
+large task or dataset is broken down (scattered) into smaller, independent
+subtasks. These subtasks are processed in parallel by multiple workers, and
+their results are then collected (gathered) and combined to produce the final
+output.
+
+Key features:
+
+* Distribution of a large task into smaller, parallelizable subtasks
+* Concurrent processing of subtasks
+* Aggregation of partial results into a final result
+
+=== Specialized workers
+
+The Specialized Workers example
+(link:examples/specialized_workers/[examples/specialized_workers/]) demonstrates
+creating distinct worker types, each tailored to handle specific kinds of tasks
+(e.g., `ComputeWorker` for CPU-intensive operations and `DatabaseWorker` for
+I/O-bound database interactions). This allows for optimized resource utilization
+and domain-specific logic.
+
+Key features:
+
+* Creation of worker classes for specific processing domains
+* Routing of work items to appropriately specialized workers
+* Optimization of resources and logic per task type
+
+
 
 == Copyright and license
 

data/examples/hierarchical_hasher/README.adoc

@@ -0,0 +1,75 @@
+= Hierarchical Hasher Example
+:toc: macro
+:toc-title: Table of Contents
+:toclevels: 3
+
+toc::[]
+
+== Overview
+
+The Hierarchical Hasher example demonstrates how to use the Fractor framework to process a file in parallel by breaking it into chunks, hashing each chunk independently, and then combining the results into a final hash.
+
+This example is particularly useful for:
+
+* Processing large files efficiently
+* Demonstrating parallel data chunking patterns
+* Showcasing result aggregation techniques
+
+== Implementation Details
+
+The example consists of the following key components:
+
+=== ChunkWork
+
+A subclass of `Fractor::Work` that represents a chunk of a file to be hashed. Each `ChunkWork` instance contains:
+
+* The chunk data
+* The starting position within the file
+* The length of the chunk
+
+=== HashWorker
+
+A subclass of `Fractor::Worker` that processes `ChunkWork` instances by:
+
+1. Calculating a SHA-256 hash for the chunk
+2. Returning a work result containing the hash, start position, and length
+
+=== FileHasher
+
+The main orchestration class that:
+
+1. Breaks a file into chunks of a specified size
+2. Creates a `Fractor::Supervisor` with the `HashWorker` and `ChunkWork` classes
+3. Processes all chunks in parallel
+4. Aggregates the results to create a final hash by combining all chunk hashes
+
+== Usage
+
+[source,ruby]
+----
+# Basic usage
+ruby hierarchical_hasher.rb <file_path> [worker_count]
+
+# Examples
+ruby hierarchical_hasher.rb sample.txt         # Use default 4 workers
+ruby hierarchical_hasher.rb large_file.dat 8   # Use 8 workers
+----
+
+== How It Works
+
+1. The file is divided into 1KB chunks (configurable)
+2. Each chunk is assigned to a worker for processing
+3. Workers calculate SHA-256 hashes for their assigned chunks
+4. Results are collected and sorted by their original position in the file
+5. The individual chunk hashes are concatenated with newlines
+6. A final SHA-256 hash is calculated on the combined hash string
+
+== Performance Considerations
+
+* The chunk size can be adjusted to optimize performance for different file types
+* The number of workers can be increased for better parallelization on multi-core systems
+* Very small files may not benefit from parallelization due to the overhead
+
+== Ractor Compatibility Note
+
+This example uses SHA-256 instead of SHA3 because the SHA3 implementation in some Ruby versions is not Ractor-compatible.