rapidflow 0.1.0 → 0.2.0

data/test/rapidflow/batch_test.rb

@@ -1,18 +1,19 @@
+# frozen_string_literal: true
 require "test_helper"
 
-module Rapidflow
+module RapidFlow
   class BatchTest < Minitest::Test
     def test_basic_functionality_with_arg_tasks
-      belt = Batch.new(
+      batch = Batch.new(
         { fn: ->(data) { data.upcase }, workers: 4 },
         { fn: ->(data) { data + "!" }, workers: 4 }
       )
-      belt.start
+      batch.start
 
-      belt.push("hello")
-      belt.push("world")
+      batch.push("hello")
+      batch.push("world")
 
-      results = belt.results
+      results = batch.results
 
       assert_equal 2, results.length
       assert_equal ["HELLO!", nil], results[0]
@@ -20,7 +21,7 @@ module Rapidflow
     end
 
     def test_basic_functionality_with_build
-      belt = Batch.build do
+      batch = Batch.build do
         # first stage to up case string
         stage ->(data) { data.upcase }
 
@@ -28,38 +29,23 @@ module Rapidflow
         stage ->(data) { data + "!" }
       end
 
-      belt.push("hello")
-      belt.push("world")
+      batch.push("hello")
+      batch.push("world")
 
-      results = belt.results
+      results = batch.results
 
       assert_equal 2, results.length
       assert_equal ["HELLO!", nil], results[0]
       assert_equal ["WORLD!", nil], results[1]
     end
 
-    def test_no_stages_with_build
-      assert_raises(Batch::ConfigError, "Unable to start the belt without any stages") do
-        Batch.build do
-          # no stages
-        end
-      end
-    end
-
-    def test_no_stages_belt_start
-      assert_raises(Batch::ConfigError, "Unable to start the belt without any stages") do
-        belt = Batch.new
-        belt.start
-      end
-    end
-
     def test_concurrent_execution_is_faster_than_sequential
       # Each lambda sleeps for 0.5 seconds
       # With 4 items and 2 stages:
       # - Sequential would take: 4 items * 0.5s * 2 stages = 4 seconds
       # - Concurrent (4 workers per stage) should take: max(0.5s, 0.5s) = ~0.5-1s
 
-      belt = Batch.build do
+      batch = Batch.build do
         stage ->(data) {
           sleep(0.5)
           data
@@ -72,8 +58,8 @@ module Rapidflow
 
       start_time = Time.now
 
-      4.times { |i| belt.push(i) }
-      results = belt.results
+      4.times { |i| batch.push(i) }
+      results = batch.results
 
       elapsed = Time.now - start_time
 
@@ -89,7 +75,7 @@ module Rapidflow
       stage1_executing = []
       stage2_executing = []
 
-      belt = Batch.build do
+      batch = Batch.build do
         stage ->(data) {
           execution_tracker.synchronize { stage1_executing << data }
           sleep(0.3)
@@ -105,7 +91,7 @@ module Rapidflow
       end
 
       # Push multiple items quickly
-      10.times { |i| belt.push(i) }
+      10.times { |i| batch.push(i) }
 
       # Give threads time to start processing
       sleep(0.1)
@@ -117,7 +103,7 @@ module Rapidflow
                "Expected concurrent execution, but only #{stage1_executing.length} items processing"
       end
 
-      results = belt.results
+      results = batch.results
       assert_equal 10, results.length
     end
 
@@ -125,7 +111,7 @@ module Rapidflow
       # Track execution order to verify pipeline behavior
       execution_log = Queue.new
 
-      belt = Batch.build do
+      batch = Batch.build do
         stage ->(data) {
           execution_log.push("stage1_start_#{data}")
           sleep(0.2)
@@ -140,11 +126,11 @@ module Rapidflow
         }
       end
 
-      belt.push("A")
+      batch.push("A")
       sleep(0.1) # Let A start processing
-      belt.push("B")
+      batch.push("B")
 
-      belt.results
+      batch.results
 
       # Convert log to array
       log = []
@@ -162,49 +148,23 @@ module Rapidflow
       assert stage1_start_b, "B should have started in stage1"
     end
 
-    def test_error_handling_captures_exceptions
-      belt = Batch.build do
-        stage ->(data) {
-          raise "Error in stage 1" if data == "bad"
-          data
-        }
-        stage ->(data) { data.upcase }
-      end
-
-      belt.push("good")
-      belt.push("bad")
-
-      results = belt.results
-
-      assert_equal 2, results.length
-
-      # Good result should complete both stages
-      assert_equal "GOOD", results[0][0]
-      assert_nil results[0][1]
-
-      # Bad result should have error from stage 1 and not be processed by stage 2
-      assert_equal "bad", results[1][0] # Original data preserved
-      assert_instance_of RuntimeError, results[1][1]
-      assert_equal "Error in stage 1", results[1][1].message
-    end
-
     def test_cannot_push_after_results_called
-      belt = Batch.build do
+      batch = Batch.build do
         stage ->(data) { data }
       end
 
-      belt.push("item1")
-      belt.results
+      batch.push("item1")
+      batch.results
 
-      assert_raises(Batch::RunError, "Cannot push to a locked belt when results are requested") do
-        belt.push("item2")
-      end
+      error = assert_raises(RapidFlow::RunError) { batch.push("item2") }
+
+      assert_equal "Cannot push to a locked batch when results are requested", error.message
     end
 
     def test_results_waits_for_all_processing_to_complete
       completion_times = Queue.new
 
-      belt = Batch.build do
+      batch = Batch.build do
         stage ->(data) {
           sleep(0.5)
           data
@@ -215,11 +175,11 @@ module Rapidflow
         }
       end
 
-      belt.push("item1")
-      belt.push("item2")
+      batch.push("item1")
+      batch.push("item2")
 
       Time.now
-      results = belt.results
+      results = batch.results
       results_end = Time.now
 
       # All items should have completed before results returns
@@ -237,7 +197,7 @@ module Rapidflow
     def test_high_throughput_with_many_items
       item_count = 100
 
-      belt = Batch.build do
+      batch = Batch.build do
         stage ->(data) {
           sleep(0.01)
           data * 2
@@ -249,8 +209,8 @@ module Rapidflow
       end
 
       start_time = Time.now
-      item_count.times { |i| belt.push(i) }
-      results = belt.results
+      item_count.times { |i| batch.push(i) }
+      results = batch.results
       elapsed = Time.now - start_time
 
       assert_equal item_count, results.length
@@ -266,7 +226,7 @@ module Rapidflow
     end
 
     def test_three_stage_pipeline
-      belt = Batch.build do
+      batch = Batch.build do
         stage ->(data) {
           sleep(0.1)
           data.upcase
@@ -281,10 +241,10 @@ module Rapidflow
         }
       end
 
-      belt.push("hello")
-      belt.push("world")
+      batch.push("hello")
+      batch.push("world")
 
-      results = belt.results
+      results = batch.results
 
       assert_equal 2, results.length
       assert_equal ["HELLO!HELLO!", nil], results[0]
@@ -293,7 +253,7 @@ module Rapidflow
 
     def test_results_preserve_input_order
       # Even though items complete at different times, results should match push order
-      belt = Batch.build do
+      batch = Batch.build do
         stage ->(data) {
           # Make later items finish faster
           sleep_time = (data[:id] == 0) ? 0.5 : 0.1
@@ -304,9 +264,9 @@ module Rapidflow
 
       # Push items in order 0, 1, 2, 3
       # But item 0 will take longer to complete
-      4.times { |i| belt.push({ id: i }) }
+      4.times { |i| batch.push({ id: i }) }
 
-      results = belt.results
+      results = batch.results
 
       # Results should still be in order 0, 1, 2, 3
       assert_equal 4, results.length
@@ -317,14 +277,14 @@ module Rapidflow
     end
 
     def test_single_stage_pipeline
-      belt = Batch.build do
+      batch = Batch.build do
         stage ->(data) { data * 2 }
       end
 
-      belt.push(5)
-      belt.push(10)
+      batch.push(5)
+      batch.push(10)
 
-      results = belt.results
+      results = batch.results
 
       assert_equal 2, results.length
       assert_equal [10, nil], results[0]
@@ -332,78 +292,13 @@ module Rapidflow
     end
 
     def test_empty_pipeline
-      belt = Batch.build { stage ->(_data) { } }
+      batch = Batch.build { stage ->(_data) { } }
 
-      results = belt.results
+      results = batch.results
 
       assert_equal 0, results.length
     end
 
-    def test_error_in_middle_stage
-      belt = Batch.build do
-        stage ->(data) { data.upcase }
-        stage ->(data) {
-          raise "Error in stage 2" if data == "BAD"
-          data
-        }
-        stage ->(data) { data + "!" }
-      end
-
-      belt.push("good")
-      belt.push("bad")
-      belt.push("also_good")
-
-      results = belt.results
-
-      assert_equal 3, results.length
-      assert_equal ["GOOD!", nil], results[0]
-      assert_equal ["BAD", results[1][1]], [results[1][0], results[1][1]]
-      assert_equal "Error in stage 2", results[1][1].message
-      assert_equal ["ALSO_GOOD!", nil], results[2]
-    end
-
-    def test_error_in_last_stage
-      belt = Batch.build do
-        stage ->(data) { data.upcase }
-        stage ->(data) {
-          raise "Error in final stage" if data == "BAD"
-          data
-        }
-      end
-
-      belt.push("good")
-      belt.push("bad")
-
-      results = belt.results
-
-      assert_equal 2, results.length
-      assert_equal ["GOOD", nil], results[0]
-      assert_equal ["BAD", results[1][1]], [results[1][0], results[1][1]]
-      assert_equal "Error in final stage", results[1][1].message
-    end
-
-    def test_multiple_errors_in_sequence
-      belt = Batch.build do
-        stage ->(data) {
-          raise "Error at #{data}" if data.start_with?("bad")
-          data
-        }
-      end
-
-      belt.push("good1")
-      belt.push("bad1")
-      belt.push("bad2")
-      belt.push("good2")
-
-      results = belt.results
-
-      assert_equal 4, results.length
-      assert_equal ["good1", nil], results[0]
-      assert_instance_of RuntimeError, results[1][1]
-      assert_instance_of RuntimeError, results[2][1]
-      assert_equal ["good2", nil], results[3]
-    end
-
     def test_different_worker_counts
       # Test with 1 worker per stage (sequential at each stage)
       j1 = Batch.build do
@@ -431,15 +326,15 @@ module Rapidflow
     end
 
     def test_complex_data_types
-      belt = Batch.build do
+      batch = Batch.build do
         stage ->(data) { { original: data, processed: true } }
         stage ->(data) { data.merge(stage2: Time.now.to_i) }
       end
 
-      belt.push({ id: 1, name: "test" })
-      belt.push([1, 2, 3])
+      batch.push({ id: 1, name: "test" })
+      batch.push([1, 2, 3])
 
-      results = belt.results
+      results = batch.results
 
       assert_equal 2, results.length
       assert results[0][0].is_a?(Hash)
@@ -449,15 +344,15 @@ module Rapidflow
     end
 
     def test_nil_values
-      belt = Batch.build do
+      batch = Batch.build do
         stage ->(data) { data.nil? ? "was_nil" : data }
         stage ->(data) { data.upcase }
       end
 
-      belt.push(nil)
-      belt.push("hello")
+      batch.push(nil)
+      batch.push("hello")
 
-      results = belt.results
+      results = batch.results
 
       assert_equal 2, results.length
       assert_equal ["WAS_NIL", nil], results[0]
@@ -467,15 +362,15 @@ module Rapidflow
     def test_large_dataset_stress_test
       item_count = 500
 
-      belt = Batch.build do
+      batch = Batch.build do
         stage ->(data) { data * 2 }, workers: 8
         stage ->(data) { data + 1 }, workers: 8
         stage ->(data) { data.to_s }, workers: 8
       end
 
-      item_count.times { |i| belt.push(i) }
+      item_count.times { |i| batch.push(i) }
 
-      results = belt.results
+      results = batch.results
 
       assert_equal item_count, results.length
 
@@ -488,7 +383,7 @@ module Rapidflow
 
     def test_varying_processing_times
       # Simulate real-world scenario with varying processing times
-      belt = Batch.build do
+      batch = Batch.build do
         stage ->(data) {
           sleep(rand * 0.1) # Random 0-100ms
           data.upcase
@@ -500,9 +395,9 @@ module Rapidflow
       end
 
       words = %w[apple banana cherry date elderberry fig grape]
-      words.each { |word| belt.push(word) }
+      words.each { |word| batch.push(word) }
 
-      results = belt.results
+      results = batch.results
 
       assert_equal words.length, results.length
       words.each_with_index do |word, i|
@@ -511,61 +406,15 @@ module Rapidflow
       end
     end
 
-    def test_exception_types_preserved
-      belt = Batch.build do
-        stage ->(data) {
-          case data
-          when "argument_error"
-            raise ArgumentError, "Bad argument"
-          when "runtime_error"
-            raise "Runtime problem"
-          when "custom_error"
-            raise StandardError, "Custom error"
-          else
-            data
-          end
-        }
-      end
-
-      belt.push("good")
-      belt.push("argument_error")
-      belt.push("runtime_error")
-      belt.push("custom_error")
-
-      results = belt.results
-
-      assert_equal 4, results.length
-      assert_equal ["good", nil], results[0]
-      assert_instance_of ArgumentError, results[1][1]
-      assert_instance_of RuntimeError, results[2][1]
-      assert_instance_of StandardError, results[3][1]
-    end
-
-    def test_all_items_fail
-      belt = Batch.build do
-        stage ->(data) { raise "Always fails" }
-      end
-
-      5.times { |i| belt.push(i) }
-
-      results = belt.results
-
-      assert_equal 5, results.length
-      results.each do |result, error|
-        assert_instance_of RuntimeError, error
-        assert_equal "Always fails", error.message
-      end
-    end
-
     def test_push_many_items_quickly
-      belt = Batch.build do
+      batch = Batch.build do
         stage ->(data) { data }
       end
 
       # Push 1000 items as fast as possible
-      1000.times { |i| belt.push(i) }
+      1000.times { |i| batch.push(i) }
 
-      results = belt.results
+      results = batch.results
 
       assert_equal 1000, results.length
       # Verify order is maintained
@@ -575,22 +424,22 @@ module Rapidflow
     end
 
     def test_idempotent_results_calls_not_allowed
-      belt = Batch.build do
+      batch = Batch.build do
         stage ->(data) { data }
       end
 
-      belt.push(1)
-      belt.results
+      batch.push(1)
+      batch.results
 
       # Can't call results again or push again
-      assert_raises(RuntimeError) { belt.push(2) }
+      assert_raises(RuntimeError) { batch.push(2) }
     end
 
     def test_thread_safety_of_shared_state
       shared_counter = { count: 0 }
       mutex = Mutex.new
 
-      belt = Batch.build do
+      batch = Batch.build do
         stage ->(data) {
           # Safely increment shared counter
           mutex.synchronize { shared_counter[:count] += 1 }
@@ -598,8 +447,8 @@ module Rapidflow
         }, workers: 10
       end
 
-      100.times { |i| belt.push(i) }
-      results = belt.results
+      100.times { |i| batch.push(i) }
+      results = batch.results
 
       assert_equal 100, results.length
       assert_equal 100, shared_counter[:count]

data/test/rapidflow/counter_test.rb

@@ -1,6 +1,6 @@
 require "test_helper"
 
-module Rapidflow
+module RapidFlow
   class CounterTest < Minitest::Test
     def test_sequential_indices
       counter = Counter.new

data/test/rapidflow/pipeline_test.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+module RapidFlow
+  class PipelineTest < Minitest::Test
+    def test_empty_pipeline
+      pipeline = Pipeline.new(0, 1)
+
+      pipeline.wait_for_completion
+      pipeline.shutdown
+
+      assert pipeline.results_empty?
+    end
+
+    def test_pipeline_with_single_stage
+      pipeline = Pipeline.new(1, 1)
+
+      pipeline.enqueue(0, "test_item")
+
+      # Simulate stage processing
+      item = pipeline.dequeue(0)
+      pipeline.enqueue(1, item.upcase)
+      pipeline.decrement_active_workers
+
+      result = pipeline.dequeue_result
+
+      assert_equal "TEST_ITEM", result
+
+      pipeline.shutdown
+    end
+
+    def test_pipeline_queues_created_correctly
+      pipeline = Pipeline.new(3, 2)
+
+      # Pipeline with 3 stages should have 4 queues (one per stage + results queue)
+      (0..3).each do |i|
+        pipeline.enqueue(i, "item_#{i}")
+
+        result = pipeline.dequeue(i)
+        assert_equal "item_#{i}", result
+      end
+
+      pipeline.shutdown
+    end
+
+    def test_active_workers_tracking
+      pipeline = Pipeline.new(1, 1)
+
+      pipeline.enqueue(0, "item1")
+      pipeline.enqueue(0, "item2")
+
+      # Simulate processing
+      pipeline.dequeue(0)
+      pipeline.decrement_active_workers
+
+      pipeline.dequeue(0)
+      pipeline.decrement_active_workers
+
+      pipeline.wait_for_completion
+
+      pipeline.shutdown
+
+      assert pipeline.results_empty?
+    end
+  end
+end