lazier_data 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c23efda68aba293371fd76e5a8ce33b7b9826b18fb580f863a2aa0dd4167214f
-  data.tar.gz: 3cdc7f4cc2f6bfd502f73f1ff51722a0e5085f9b372967878e35e6fa4b612a34
+  metadata.gz: 72acf70d853c5d5e20ca99092cb4f032c5fcec0178846710d378a80f701cfef8
+  data.tar.gz: 3b23b30f7cd8ad4bc3dad4bbeaa5ac2b00f00ee5779fc7501fbaef477a0a3ff7
 SHA512:
-  metadata.gz: '098d51524372436ba513efc67ced6f6f7ed7f9adfefc261b67adda298383e77fac9aab9f5acdc10c811fc1c24012dbb0702c852b0e3f8b16299d215337d5314c'
-  data.tar.gz: 2436fbd078e94e17269b081f3a4c8c9dcf16e981326880796c7670ed143a3e7bf50985d3eafac7169e026482ee5247c0198619c62d06f6f71a1b2990a36ad5a5
+  metadata.gz: 147f8439216de43c030b5ff87cfa20f121d94e1ea04a8167ad97a551f0c9bd218ff18f98b3f50f335e8f1a37b68665fa4ac09c10ec0d705287d0039ae5222a62
+  data.tar.gz: 20abfe4f10cd0209c291e78fdadfc97377a6425f93848f7eb1668710fc1b48bf30015722f97c0b64d347691944bf7402c367c16787c6277cc5ea5a3e207e1ed8

data/README.md

@@ -1,28 +1,107 @@
-# BetterBatch
+# LazierData
 
-TODO: Delete this and the text below, and describe your gem
+The incredible productivity of massive laziness.
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/better_batch`. To experiment with that code, run `bin/console` for an interactive prompt.
+LazierData allows splitting, filtering, transforming, batching, and basically everything else you might want to do with data. LazierData processing _looks like_ multiple full iterations of the input data, but under the hood it will process incrementally.
 
-## Installation
-
-TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
+LazierData guarantees each row of input data will be processed by steps in the order the steps are defined. This means not only does LazierData processing _look like_ each step is happening in order, but you can generally think about your data that way.
 
-Install the gem and add to the application's Gemfile by executing:
+## Installation
 
-```bash
-bundle add UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+In your Gemfile:
+```ruby
+source 'https://rubygems.org'
+gem 'lazier_data'
 ```
+Then:
+`bundle`
 
-If bundler is not being used to manage dependencies, install the gem by executing:
+## Usage
 
-```bash
-gem install UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+```ruby
+# inputs are anything that can be iterated with .each
+lazier = LazierData.new(inputs)
+
+# transform
+lazier.enum do |input, clean_inputs|
+  clean_inputs << input.except(:unwanted_field)
+end
+
+# LazierData uses the parameter name from your previous block
+# to allow further processing
+# lets filter our clean inputs
+lazier[:clean_inputs].enum do |input, filtered_inputs|
+  filtered_inputs << input unless input[:skip_me]
+end
+
+# let's say our inputs have multiple types
+# that need to be handled differently
+lazier[:clean_inputs][:filtered_inputs].enum do |input, type_a, type_b|
+  case input[:type]
+  when :a
+  	type_a << input
+  when :b
+  	type_b << input
+  end
+end
+
+# LazierData puts the results of your previous steps
+# in the corresponding places according to your block params
+type_a = lazier[:clean_inputs][:filtered_inputs][:type_a]
+type_b = lazier[:clean_inputs][:filtered_inputs][:type_b]
+
+# we can now process each type separately
+type_a.each_slice(1000) do |batch_a|
+  ModelA.upsert_all(
+    batch_a,
+    unique_by: %i[unique_field1 unique_field2]
+  )
+end
+type_b.each_slice(1000) do |batch_b|
+  ModelB.upsert_all(
+    batch_b,
+    unique_by: %i[unique_field3 unique_field4]
+  )
+end
+
+# we can also pull sub items off the inputs
+# note that in this example we're taking sub_items from all inputs
+# instead of lazier[:clean_inputs][:filtered_inputs].enum do ...
+# while it may look like we're reprocessing the entire input list
+# under the hood LazierData will pass each item through each step
+lazier.enum do |input, sub_items|
+  # the below is shorthand for:
+  # input[:sub_items].each { |sub_item| sub_items << sub_item }
+  input[:sub_items].each(&sub_items)
+end
+
+# you can mutate items directly
+lazier[:sub_items].each do |sub_item|
+  sub_item[:other_data] = fetch_other_data
+  sub_item[:mutated] = true
+end
+
+# items are guaranteed to have passed through previous steps
+# before reaching later steps
+lazier[:sub_items].each_slice(1000) do |sub_items|
+  if sub_items.any? { |sub_item| !sub_item[:mutated] }
+  	# this will never happen :)
+  	raise 'LazierData failed me!'
+  end
+
+  SubItem.upsert_all(
+  	sub_items,
+  	unique_by: %i[unique_sub_item_field]
+  )
+end
+
+# at this point in the code
+# none of the above has actually happened
+# so, finally, we actually go
+lazier.go
 ```
 
-## Usage
-
-TODO: Write usage instructions here
+See [lazier_data_spec.rb](spec/lazier_data_spec.rb) for proven examples.
 
 ## Development
 

data/lib/lazier_data/item_store.rb

@@ -16,6 +16,10 @@ class LazierData
       end
     end
 
+    def [](key)
+      @store[key][ITEMS_KEY]
+    end
+
     private
 
     def new_layer

data/lib/lazier_data/processor/child_each.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class LazierData
+  class Processor
+    class ChildEach
+      NOTHING = :_lazier_data_nothing
+
+      attr_reader :upstream, :downstream, :batch_size, :input_path, :block
+
+      def initialize(upstream, downstream, input_path, &block)
+        @upstream = upstream
+        @downstream = downstream
+        @batch_size = nil
+        @input_path = input_path
+        @block = block
+      end
+
+      def call
+        upstream.each do |root_item, item_store|
+          output_yielders = build_output_yielders(item_store)
+          item_store.dig(*input_path).each do |item|
+            block.call(item, *output_yielders)
+          end
+          downstream << [root_item, item_store]
+        end
+      end
+
+      private
+
+      def output_path_parts
+        @output_path_parts ||= block.parameters[1..].map(&:last)
+      end
+
+      def build_output_yielders(item_store)
+        output_path_parts.map do |output_path_part|
+          ::Enumerator::Yielder.new do |item|
+            storage_path = input_path + [output_path_part]
+            item_store.dig(*storage_path) << item
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lazier_data/processor/child_each_slice.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+class LazierData
+  class Processor
+    class ChildEachSlice
+      NOTHING = :_lazier_data_nothing
+
+      attr_reader :upstream, :downstream, :batch_size, :input_path, :block
+
+      def initialize(upstream, downstream, batch_size, input_path, &block)
+        @upstream = upstream
+        @downstream = downstream
+        @batch_size = batch_size
+        @input_path = input_path
+        @block = block
+      end
+
+      def call
+        slicer.each_slice(batch_size) do |raw_yielded|
+          items = raw_yielded.map(&:first)
+          yielders = raw_yielded.last[3]
+          @block.call(items, *yielders)
+          raw_yielded.each do |_, root_item, item_store, _|
+            next if root_item == NOTHING
+
+            downstream << [root_item, item_store]
+          end
+        end
+      end
+
+      private
+
+      def slicer
+        Enumerator.new do |slicer|
+          upstream.each do |root_item, item_store|
+            yield_items(slicer, root_item, item_store)
+          end
+        end
+      end
+
+      def yield_items(slicer, root_item, item_store)
+        output_yielders = build_output_yielders(item_store)
+        items = item_store.dig(*input_path)
+        if items.count.zero?
+          downstream << [root_item, item_store]
+        elsif items.count == 1
+          slicer << [items.first, root_item, item_store, output_yielders]
+        elsif items.count > 1
+          yield_multiple(slicer, items, root_item, item_store, output_yielders)
+        end
+      end
+
+      def yield_multiple(slicer, items, root_item, item_store, output_yielders)
+        items[0..-2].each do |item|
+          slicer << [item, NOTHING, item_store, output_yielders]
+        end
+
+        slicer << [items.last, root_item, item_store, output_yielders]
+      end
+
+      def output_path_parts
+        @output_path_parts ||= block.parameters[1..].map(&:last)
+      end
+
+      def build_output_yielders(item_store)
+        output_path_parts.map do |output_path_part|
+          ::Enumerator::Yielder.new do |item|
+            storage_path = input_path + [output_path_part]
+            item_store.dig(*storage_path) << item
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lazier_data/processor/root_each.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class LazierData
+  class Processor
+    class RootEach
+      def initialize(upstream, downstream, &block)
+        @upstream = upstream
+        @downstream = downstream
+        @block = block
+      end
+
+      def call
+        upstream.each do |root_item, item_store|
+          output_yielders = build_output_yielders(item_store)
+          block.call(root_item, *output_yielders)
+          downstream << [root_item, item_store]
+        end
+      end
+
+      private
+
+      attr_reader :upstream, :downstream, :block
+
+      def output_path_parts
+        @output_path_parts ||= block.parameters[1..].map(&:last)
+      end
+
+      def build_output_yielders(item_store)
+        output_path_parts.map do |output_path_part|
+          ::Enumerator::Yielder.new do |item|
+            item_store[output_path_part] << item
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lazier_data/processor/root_each_slice.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+class LazierData
+  class Processor
+    class RootEachSlice
+      def initialize(upstream, downstream, batch_size, &block)
+        @upstream = upstream
+        @downstream = downstream
+        @batch_size = batch_size
+        @block = block
+      end
+
+      def call
+        slicer.each_slice(batch_size) do |raw_yielded|
+          root_items = raw_yielded.map(&:first)
+          yielders = raw_yielded.last[2]
+          @block.call(root_items, *yielders)
+          raw_yielded.each do |root_item, item_store, _|
+            downstream << [root_item, item_store]
+          end
+        end
+      end
+
+      private
+
+      attr_reader :upstream, :downstream, :batch_size, :block
+
+      def slicer
+        Enumerator.new do |slicer|
+          upstream.each do |root_item, item_store|
+            output_yielders = build_output_yielders(item_store)
+            slicer << [root_item, item_store, output_yielders]
+          end
+        end
+      end
+
+      # probably go in module
+      def output_path_parts
+        @output_path_parts ||= block.parameters[1..].map(&:last)
+      end
+
+      def build_output_yielders(item_store)
+        output_path_parts.map do |output_path_part|
+          ::Enumerator::Yielder.new do |item|
+            item_store[output_path_part] << item
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lazier_data/processor.rb

@@ -1,120 +1,56 @@
 # frozen_string_literal: true
 
+require 'lazier_data/item_store'
+
 class LazierData
   class Processor
     NOTHING = :_lazier_data_nothing
+    class << self
+      def root(inputs)
+        Enumerator.new do |y|
+          inputs.each do |item|
+            y << [item, ItemStore.new]
+          end
+        end
+      end
+    end
 
-    attr_reader :upstream, :downstream, :batch_size, :input_path, :output_path_parts, :block
-
-    def initialize(upstream, downstream, batch_size, input_path, output_path_parts, &block)
+    def initialize(upstream, batch_size, path, &block)
       @upstream = upstream
       @downstream = downstream
       @batch_size = batch_size
-      @input_path = input_path
-      @output_path_parts = output_path_parts
+      @path = path
       @block = block
     end
 
     def call
-      if batch_size.nil?
-        process_each
-      else
-        process_each_slice
-      end
-    end
-
-    private
-
-    def dug # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/PerceivedComplexity
-      Enumerator.new do |dug|
-        upstream.each do |root_item, item_store|
-          output_yielders = build_output_yielders(item_store)
-
-          if input_path.empty?
-            log_and_yield(dug, [root_item, root_item, item_store, output_yielders], :root)
-          else
-            items = item_store.dig(*input_path)
-            if items.count.zero?
-              log_and_yield(downstream, [root_item, item_store], :no_dug)
-            elsif items.count == 1
-              log_and_yield(dug, [items.first, root_item, item_store, output_yielders], :only)
-            elsif items.count > 1
-              items[0..-2].each.with_index(1) do |item, item_number|
-                log_and_yield(dug, [item, NOTHING, item_store, output_yielders], :stored, item_number, items.count)
-              end
-
-              log_and_yield(dug, [items.last, root_item, item_store, output_yielders], :last, items.count, items.count)
-            else
-              raise 'wat'
-            end
-          end
+      Enumerator.new do |downstream|
+        if batch_size.nil?
+          build_each_processor(downstream)
+        else
+          build_each_slice_processor(downstream)
         end
       end
     end
 
-    def process_each
-      dug.each do |item, root_item, item_store, output_yielders|
-        log_and_call(block, [item, *output_yielders], :item)
-        log_and_yield(downstream, [root_item, item_store], :after_item)
-      end
-    end
-
-    def process_each_slice
-      dug.each_slice(batch_size) do |raw_yielded|
-        item_slice = raw_yielded.map(&:first)
-        log_and_call(block, [item_slice, *raw_yielded.last[3]], :slice)
-        raw_yielded.each do |_, root_item, item_store, _|
-          next if root_item == NOTHING
+    private
 
-          log_and_yield(downstream, [root_item, item_store], :after_slice)
-        end
-      end
-    end
+    attr_reader :upstream, :downstream, :batch_size, :path, :block
 
-    def build_output_yielders(item_store)
-      output_path_parts.map do |output_path_part|
-        ::Enumerator::Yielder.new do |item|
-          storage_path = input_path + [output_path_part]
-          logger.debug { "storing item at #{storage_path}: #{item.inspect}" }
-          item_store.dig(*storage_path) << item
-        end
+    def build_each_processor(downstream)
+      if path.empty?
+        Processor::RootEach.new(upstream, downstream, &block).call
+      else
+        Processor::ChildEach.new(upstream, downstream, path, &block).call
       end
     end
 
-    def log_and_yield(yielder, to_yield, msg_type, item_number = nil, found_count = nil)
-      logger.debug { build_log_message(msg_type, to_yield, item_number, found_count) }
-      yielder << to_yield
-    end
-
-    def log_and_call(callee, to_yield, msg_type)
-      logger.debug { build_log_message(msg_type, to_yield) }
-      callee.call(*to_yield)
-    end
-
-    def build_log_message(msg_type, to_yield, item_number = nil, found_count = nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity
-      case msg_type
-      when :root
-        "yielding dug root item: #{to_yield[0..1].inspect}"
-      when :no_dug
-        "yielding downstream (no dug items from #{input_path.inspect}): #{to_yield[0..0].inspect}"
-      when :only
-        "yielding dug stored item (1 of 1) from #{input_path.inspect}: #{to_yield[0..1].inspect}"
-      when :stored, :last_stored
-        "yielding dug stored item (#{item_number} of #{found_count}) " \
-        "from #{input_path.inspect}: #{to_yield[0..1].inspect}"
-      when :item
-        "yielding item to #{block.source_location}: #{to_yield[0..0].inspect}"
-      when :after_item
-        "yielding downstream (after item): #{to_yield[0..0].inspect}"
-      when :slice
-        "yielding slice to #{block.source_location}: #{to_yield[0..0].inspect}"
-      when :after_slice
-        "yielding downstream (after slice): #{to_yield[0..0].inspect}"
+    def build_each_slice_processor(downstream)
+      if path.empty?
+        Processor::RootEachSlice.new(upstream, downstream, batch_size, &block).call
+      else
+        Processor::ChildEachSlice.new(upstream, downstream, batch_size, path, &block).call
       end
     end
-
-    def logger
-      LazierData.logger
-    end
   end
 end

data/lib/lazier_data/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class LazierData
-  VERSION = '0.1.0'
+  VERSION = '0.1.2'
 end

data/lib/lazier_data.rb

@@ -3,32 +3,14 @@
 require 'lazier_data/child'
 require 'lazier_data/item_store'
 require 'lazier_data/processor'
+require 'lazier_data/processor/root_each'
+require 'lazier_data/processor/root_each_slice'
+require 'lazier_data/processor/child_each'
+require 'lazier_data/processor/child_each_slice'
 
 class LazierData
-  class << self
-    attr_writer :logger
-
-    def logger
-      @logger ||= null_logger
-    end
-
-    private
-
-    def null_logger
-      Class.new do
-        class << self
-          def method_missing(*, **, &); end
-
-          def respond_to_missing?
-            true
-          end
-        end
-      end
-    end
-  end
-
   def initialize(inputs)
-    @initial_processor_builder = proc { root_processor(inputs) }
+    @initial_processor_builder = proc { Processor.root(inputs) }
     @processor_builders = []
     @children = {}
   end
@@ -41,9 +23,7 @@ class LazierData
     end
 
     parent.add do |upstream|
-      Enumerator.new do |downstream|
-        Processor.new(upstream, downstream, batch_size, my_path, output_path_parts, &block).call
-      end
+      Processor.new(upstream, batch_size, my_path, &block).call
     end
   end
 
@@ -63,12 +43,7 @@ class LazierData
     @children.fetch(path_part)
   end
 
-  def logger
-    LazierData.logger
-  end
-
   def go
-    logger.info { 'initiating processing' }
     upstream = @initial_processor_builder.call
     processors = @processor_builders.map do |processor_builder|
       upstream = processor_builder.call(upstream)
@@ -77,7 +52,6 @@ class LazierData
   end
 
   def go_stepwise
-    logger.info { 'initiating stepwise processing' }
     stepwise_results = []
     results = @initial_processor_builder.call.to_a
     stepwise_results << results
@@ -100,14 +74,6 @@ class LazierData
 
   private
 
-  def root_processor(inputs)
-    Enumerator.new do |y|
-      inputs.each do |item|
-        y << [item, ItemStore.new]
-      end
-    end
-  end
-
   def parent
     self
   end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: lazier_data
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Tyler Hartland
 bindir: exe
 cert_chain: []
-date: 2025-04-10 00:00:00.000000000 Z
+date: 2025-04-12 00:00:00.000000000 Z
 dependencies: []
 description: 'Allows setting up data processing that works intuitively, but behind
   the secenes processes lazily.
@@ -31,14 +31,18 @@ files:
 - lib/lazier_data/child.rb
 - lib/lazier_data/item_store.rb
 - lib/lazier_data/processor.rb
+- lib/lazier_data/processor/child_each.rb
+- lib/lazier_data/processor/child_each_slice.rb
+- lib/lazier_data/processor/root_each.rb
+- lib/lazier_data/processor/root_each_slice.rb
 - lib/lazier_data/version.rb
-homepage: https://github.com/th7/lazier
+homepage: https://github.com/th7/lazier_data
 licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  homepage_uri: https://github.com/th7/lazier
-  source_code_uri: https://github.com/th7/lazier
+  homepage_uri: https://github.com/th7/lazier_data
+  source_code_uri: https://github.com/th7/lazier_data
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths: