lazier_data 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c57f03c689415868d20b1c1be957564aa4e3b2517aae7dd9d8b51ca24f655f9
-  data.tar.gz: b620c052b871cf8d68d945baea968d834856791e564052f6b26276ab9fd867b7
+  metadata.gz: 72acf70d853c5d5e20ca99092cb4f032c5fcec0178846710d378a80f701cfef8
+  data.tar.gz: 3b23b30f7cd8ad4bc3dad4bbeaa5ac2b00f00ee5779fc7501fbaef477a0a3ff7
 SHA512:
-  metadata.gz: 030f3f706f1b6dacaa8778d7619a8f25b015138799cb90921d3ce5471ae2f2315b6e0382656c0f1198c4c304bd554495846d078323e445c458e60c424ee75fce
-  data.tar.gz: 91bdc0898cc4e93958f0fb40d9fb5b1700b12b8fc1a7eb06ec9a6d3d95990d84611124295bb11b660d4740ee2c9d826c1c7352a130681c423397a9a27062b1e0
+  metadata.gz: 147f8439216de43c030b5ff87cfa20f121d94e1ea04a8167ad97a551f0c9bd218ff18f98b3f50f335e8f1a37b68665fa4ac09c10ec0d705287d0039ae5222a62
+  data.tar.gz: 20abfe4f10cd0209c291e78fdadfc97377a6425f93848f7eb1668710fc1b48bf30015722f97c0b64d347691944bf7402c367c16787c6277cc5ea5a3e207e1ed8

data/lib/lazier_data/item_store.rb

@@ -16,6 +16,10 @@ class LazierData
       end
     end
 
+    def [](key)
+      @store[key][ITEMS_KEY]
+    end
+
     private
 
     def new_layer

data/lib/lazier_data/processor/child_each.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class LazierData
+  class Processor
+    class ChildEach
+      NOTHING = :_lazier_data_nothing
+
+      attr_reader :upstream, :downstream, :batch_size, :input_path, :block
+
+      def initialize(upstream, downstream, input_path, &block)
+        @upstream = upstream
+        @downstream = downstream
+        @batch_size = nil
+        @input_path = input_path
+        @block = block
+      end
+
+      def call
+        upstream.each do |root_item, item_store|
+          output_yielders = build_output_yielders(item_store)
+          item_store.dig(*input_path).each do |item|
+            block.call(item, *output_yielders)
+          end
+          downstream << [root_item, item_store]
+        end
+      end
+
+      private
+
+      def output_path_parts
+        @output_path_parts ||= block.parameters[1..].map(&:last)
+      end
+
+      def build_output_yielders(item_store)
+        output_path_parts.map do |output_path_part|
+          ::Enumerator::Yielder.new do |item|
+            storage_path = input_path + [output_path_part]
+            item_store.dig(*storage_path) << item
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lazier_data/processor/child_each_slice.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+class LazierData
+  class Processor
+    class ChildEachSlice
+      NOTHING = :_lazier_data_nothing
+
+      attr_reader :upstream, :downstream, :batch_size, :input_path, :block
+
+      def initialize(upstream, downstream, batch_size, input_path, &block)
+        @upstream = upstream
+        @downstream = downstream
+        @batch_size = batch_size
+        @input_path = input_path
+        @block = block
+      end
+
+      def call
+        slicer.each_slice(batch_size) do |raw_yielded|
+          items = raw_yielded.map(&:first)
+          yielders = raw_yielded.last[3]
+          @block.call(items, *yielders)
+          raw_yielded.each do |_, root_item, item_store, _|
+            next if root_item == NOTHING
+
+            downstream << [root_item, item_store]
+          end
+        end
+      end
+
+      private
+
+      def slicer
+        Enumerator.new do |slicer|
+          upstream.each do |root_item, item_store|
+            yield_items(slicer, root_item, item_store)
+          end
+        end
+      end
+
+      def yield_items(slicer, root_item, item_store)
+        output_yielders = build_output_yielders(item_store)
+        items = item_store.dig(*input_path)
+        if items.count.zero?
+          downstream << [root_item, item_store]
+        elsif items.count == 1
+          slicer << [items.first, root_item, item_store, output_yielders]
+        elsif items.count > 1
+          yield_multiple(slicer, items, root_item, item_store, output_yielders)
+        end
+      end
+
+      def yield_multiple(slicer, items, root_item, item_store, output_yielders)
+        items[0..-2].each do |item|
+          slicer << [item, NOTHING, item_store, output_yielders]
+        end
+
+        slicer << [items.last, root_item, item_store, output_yielders]
+      end
+
+      def output_path_parts
+        @output_path_parts ||= block.parameters[1..].map(&:last)
+      end
+
+      def build_output_yielders(item_store)
+        output_path_parts.map do |output_path_part|
+          ::Enumerator::Yielder.new do |item|
+            storage_path = input_path + [output_path_part]
+            item_store.dig(*storage_path) << item
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lazier_data/processor/root_each.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class LazierData
+  class Processor
+    class RootEach
+      def initialize(upstream, downstream, &block)
+        @upstream = upstream
+        @downstream = downstream
+        @block = block
+      end
+
+      def call
+        upstream.each do |root_item, item_store|
+          output_yielders = build_output_yielders(item_store)
+          block.call(root_item, *output_yielders)
+          downstream << [root_item, item_store]
+        end
+      end
+
+      private
+
+      attr_reader :upstream, :downstream, :block
+
+      def output_path_parts
+        @output_path_parts ||= block.parameters[1..].map(&:last)
+      end
+
+      def build_output_yielders(item_store)
+        output_path_parts.map do |output_path_part|
+          ::Enumerator::Yielder.new do |item|
+            item_store[output_path_part] << item
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lazier_data/processor/root_each_slice.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+class LazierData
+  class Processor
+    class RootEachSlice
+      def initialize(upstream, downstream, batch_size, &block)
+        @upstream = upstream
+        @downstream = downstream
+        @batch_size = batch_size
+        @block = block
+      end
+
+      def call
+        slicer.each_slice(batch_size) do |raw_yielded|
+          root_items = raw_yielded.map(&:first)
+          yielders = raw_yielded.last[2]
+          @block.call(root_items, *yielders)
+          raw_yielded.each do |root_item, item_store, _|
+            downstream << [root_item, item_store]
+          end
+        end
+      end
+
+      private
+
+      attr_reader :upstream, :downstream, :batch_size, :block
+
+      def slicer
+        Enumerator.new do |slicer|
+          upstream.each do |root_item, item_store|
+            output_yielders = build_output_yielders(item_store)
+            slicer << [root_item, item_store, output_yielders]
+          end
+        end
+      end
+
+      # probably go in module
+      def output_path_parts
+        @output_path_parts ||= block.parameters[1..].map(&:last)
+      end
+
+      def build_output_yielders(item_store)
+        output_path_parts.map do |output_path_part|
+          ::Enumerator::Yielder.new do |item|
+            item_store[output_path_part] << item
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lazier_data/processor.rb

@@ -1,120 +1,56 @@
 # frozen_string_literal: true
 
+require 'lazier_data/item_store'
+
 class LazierData
   class Processor
     NOTHING = :_lazier_data_nothing
+    class << self
+      def root(inputs)
+        Enumerator.new do |y|
+          inputs.each do |item|
+            y << [item, ItemStore.new]
+          end
+        end
+      end
+    end
 
-    attr_reader :upstream, :downstream, :batch_size, :input_path, :output_path_parts, :block
-
-    def initialize(upstream, downstream, batch_size, input_path, output_path_parts, &block)
+    def initialize(upstream, batch_size, path, &block)
       @upstream = upstream
       @downstream = downstream
       @batch_size = batch_size
-      @input_path = input_path
-      @output_path_parts = output_path_parts
+      @path = path
       @block = block
     end
 
     def call
-      if batch_size.nil?
-        process_each
-      else
-        process_each_slice
-      end
-    end
-
-    private
-
-    def dug # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/PerceivedComplexity
-      Enumerator.new do |dug|
-        upstream.each do |root_item, item_store|
-          output_yielders = build_output_yielders(item_store)
-
-          if input_path.empty?
-            log_and_yield(dug, [root_item, root_item, item_store, output_yielders], :root)
-          else
-            items = item_store.dig(*input_path)
-            if items.count.zero?
-              log_and_yield(downstream, [root_item, item_store], :no_dug)
-            elsif items.count == 1
-              log_and_yield(dug, [items.first, root_item, item_store, output_yielders], :only)
-            elsif items.count > 1
-              items[0..-2].each.with_index(1) do |item, item_number|
-                log_and_yield(dug, [item, NOTHING, item_store, output_yielders], :stored, item_number, items.count)
-              end
-
-              log_and_yield(dug, [items.last, root_item, item_store, output_yielders], :last, items.count, items.count)
-            else
-              raise 'wat'
-            end
-          end
+      Enumerator.new do |downstream|
+        if batch_size.nil?
+          build_each_processor(downstream)
+        else
+          build_each_slice_processor(downstream)
         end
       end
     end
 
-    def process_each
-      dug.each do |item, root_item, item_store, output_yielders|
-        log_and_call(block, [item, *output_yielders], :item)
-        log_and_yield(downstream, [root_item, item_store], :after_item)
-      end
-    end
-
-    def process_each_slice
-      dug.each_slice(batch_size) do |raw_yielded|
-        item_slice = raw_yielded.map(&:first)
-        log_and_call(block, [item_slice, *raw_yielded.last[3]], :slice)
-        raw_yielded.each do |_, root_item, item_store, _|
-          next if root_item == NOTHING
+    private
 
-          log_and_yield(downstream, [root_item, item_store], :after_slice)
-        end
-      end
-    end
+    attr_reader :upstream, :downstream, :batch_size, :path, :block
 
-    def build_output_yielders(item_store)
-      output_path_parts.map do |output_path_part|
-        ::Enumerator::Yielder.new do |item|
-          storage_path = input_path + [output_path_part]
-          logger.debug { "storing item at #{storage_path}: #{item.inspect}" }
-          item_store.dig(*storage_path) << item
-        end
+    def build_each_processor(downstream)
+      if path.empty?
+        Processor::RootEach.new(upstream, downstream, &block).call
+      else
+        Processor::ChildEach.new(upstream, downstream, path, &block).call
       end
     end
 
-    def log_and_yield(yielder, to_yield, msg_type, item_number = nil, found_count = nil)
-      logger.debug { build_log_message(msg_type, to_yield, item_number, found_count) }
-      yielder << to_yield
-    end
-
-    def log_and_call(callee, to_yield, msg_type)
-      logger.debug { build_log_message(msg_type, to_yield) }
-      callee.call(*to_yield)
-    end
-
-    def build_log_message(msg_type, to_yield, item_number = nil, found_count = nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity
-      case msg_type
-      when :root
-        "yielding dug root item: #{to_yield[0..1].inspect}"
-      when :no_dug
-        "yielding downstream (no dug items from #{input_path.inspect}): #{to_yield[0..0].inspect}"
-      when :only
-        "yielding dug stored item (1 of 1) from #{input_path.inspect}: #{to_yield[0..1].inspect}"
-      when :stored, :last_stored
-        "yielding dug stored item (#{item_number} of #{found_count}) " \
-        "from #{input_path.inspect}: #{to_yield[0..1].inspect}"
-      when :item
-        "yielding item to #{block.source_location}: #{to_yield[0..0].inspect}"
-      when :after_item
-        "yielding downstream (after item): #{to_yield[0..0].inspect}"
-      when :slice
-        "yielding slice to #{block.source_location}: #{to_yield[0..0].inspect}"
-      when :after_slice
-        "yielding downstream (after slice): #{to_yield[0..0].inspect}"
+    def build_each_slice_processor(downstream)
+      if path.empty?
+        Processor::RootEachSlice.new(upstream, downstream, batch_size, &block).call
+      else
+        Processor::ChildEachSlice.new(upstream, downstream, batch_size, path, &block).call
       end
     end
-
-    def logger
-      LazierData.logger
-    end
   end
 end

data/lib/lazier_data/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class LazierData
-  VERSION = '0.1.1'
+  VERSION = '0.1.2'
 end

data/lib/lazier_data.rb

@@ -3,32 +3,14 @@
 require 'lazier_data/child'
 require 'lazier_data/item_store'
 require 'lazier_data/processor'
+require 'lazier_data/processor/root_each'
+require 'lazier_data/processor/root_each_slice'
+require 'lazier_data/processor/child_each'
+require 'lazier_data/processor/child_each_slice'
 
 class LazierData
-  class << self
-    attr_writer :logger
-
-    def logger
-      @logger ||= null_logger
-    end
-
-    private
-
-    def null_logger
-      Class.new do
-        class << self
-          def method_missing(*, **, &); end
-
-          def respond_to_missing?
-            true
-          end
-        end
-      end
-    end
-  end
-
   def initialize(inputs)
-    @initial_processor_builder = proc { root_processor(inputs) }
+    @initial_processor_builder = proc { Processor.root(inputs) }
     @processor_builders = []
     @children = {}
   end
@@ -41,9 +23,7 @@ class LazierData
     end
 
     parent.add do |upstream|
-      Enumerator.new do |downstream|
-        Processor.new(upstream, downstream, batch_size, my_path, output_path_parts, &block).call
-      end
+      Processor.new(upstream, batch_size, my_path, &block).call
     end
   end
 
@@ -63,12 +43,7 @@ class LazierData
     @children.fetch(path_part)
   end
 
-  def logger
-    LazierData.logger
-  end
-
   def go
-    logger.info { 'initiating processing' }
     upstream = @initial_processor_builder.call
     processors = @processor_builders.map do |processor_builder|
       upstream = processor_builder.call(upstream)
@@ -77,7 +52,6 @@ class LazierData
   end
 
   def go_stepwise
-    logger.info { 'initiating stepwise processing' }
     stepwise_results = []
     results = @initial_processor_builder.call.to_a
     stepwise_results << results
@@ -100,14 +74,6 @@ class LazierData
 
   private
 
-  def root_processor(inputs)
-    Enumerator.new do |y|
-      inputs.each do |item|
-        y << [item, ItemStore.new]
-      end
-    end
-  end
-
   def parent
     self
   end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: lazier_data
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Tyler Hartland
 bindir: exe
 cert_chain: []
-date: 2025-04-11 00:00:00.000000000 Z
+date: 2025-04-12 00:00:00.000000000 Z
 dependencies: []
 description: 'Allows setting up data processing that works intuitively, but behind
   the secenes processes lazily.
@@ -31,6 +31,10 @@ files:
 - lib/lazier_data/child.rb
 - lib/lazier_data/item_store.rb
 - lib/lazier_data/processor.rb
+- lib/lazier_data/processor/child_each.rb
+- lib/lazier_data/processor/child_each_slice.rb
+- lib/lazier_data/processor/root_each.rb
+- lib/lazier_data/processor/root_each_slice.rb
 - lib/lazier_data/version.rb
 homepage: https://github.com/th7/lazier_data
 licenses: