sequel-packer 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f7666c0db1cac9f34075e40a398a5963b33e6756d38507953dc14c59d5712d2
-  data.tar.gz: 10bf3ff431ba9fab4ff6bdfec84e51f1243cf3da1b5eff3ea66d6b7b45f94fbc
+  metadata.gz: 811c7b7319fa09e9885726f132a84e52d8944746288be9331c7d9ac83408537f
+  data.tar.gz: 4a11f786524e3576470b744068fcd7735271743a79bc6874755f0e7e2eadf09a
 SHA512:
-  metadata.gz: 67d08fbd8aef15f7005ae59b411f87a237d83b53aeb26e7e92efff2a628e6ae29bf14724cfedea03f34b5f7cf61e4c8d28be6cc4e01027535d04519fa681f7e1
-  data.tar.gz: c96e1f683319af1a20d6cb775f6b0ac8a479297947b6a26008082e4a8aa8e35a677de7d3a8f66a7a74fc57079fb8ecab1b678201a214f5768fc57eef61b4b150
+  metadata.gz: 608fc72d310784fe7b8edbe0fcc196b3207f4ed652e33fa4d3625864eb6ffaebc9a4b5291046bad92f620b85b7bd6ec1831e445f1934f8dd1b32d31d82e38643
+  data.tar.gz: 296826320aac8126a6c6f53ac3f4cac8828f87ae092428a9f346d8fea1faedcbc9c039f2e7e2ecb40f009d7e9568e75cc8ea2b93ab089ab9873e3eaa411d5ecb

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+### 0.4.0 (2020-05-17)
+
+* **_BREAKING CHANGE:_** `#pack_models` and `#pack_model` have been changed to
+  private methods. In their place `Sequel::Packer#pack` has been changed to
+  accept a dataset, an array of models or a single model, while still ensuring
+  eager loading takes place.
+* Add `self.precompute(&block)` for performing bulk computations outside of
+  Packer paradigm.
+
 ### 0.3.0 (2020-05-14)
 
 * Add `self.set_association_packer(association, packer_class, *traits)` and

data/README.md

@@ -493,16 +493,61 @@ class PostPacker < Sequel::Packer
 end
 ```
 
+### `self.precompute(&block)`
+
+Occasionally packing a model may require a computation that doesn't fit in with
+the rest of the Packer paradigm. This may be a Sequel query that is particularly
+difficult to express as an association, or even a call to an external service.
+If such a computation can be performed in bulk, then the `precompute` method can
+be used as an entry point for that operation.
+
+The `precompute` method will execute a given block and pass it all of the models
+that will be packed using that packer. This block will be executed a single
+time, even when called by a deeply nested packer.
+
+The `precompute` block is `instance_exec`ed in the context of the packer
+instance, the result of any computation can be saved in a simple instance
+variable (`@precomputed_result`) and later referenced inside the blocks that are
+passed to `field` methods.
+
+As an example, suppose a video uploading platform performs additional video
+processing on every uploaded video and exposes the status of that processing as
+a separate service over the network, rather than directly with the upload
+metadata in the database. `precompute` could be used as follows:
+
+```ruby
+class VideoUploadPacker < Sequel::Packer
+  model VideoUpload
+
+  precompute do |video_uploads|
+    @processing_statuses = ResolutionService
+      .get_status_bulk(ids: video_uploads.map(&:id))
+  end
+
+  field :id
+  field :filename
+  field :processing_status do |video_upload|
+    @processing_statuses[video_upload.id]
+  end
+end
+```
+
 
 ### `initialize(*traits)`
 
 When creating an instance of a Packer class, pass in any traits desired to
 specify what additional data should be packed, if any.
 
-### `pack(dataset)`
+### `pack(dataset_or_models_or_model)`
+
+After creating a new instance of a Packer class, call `packer.pack` to tranform
+your data into packed Ruby hashes.
 
-After creating a new instance of a Packer class, call `packer.pack(dataset)` to
-materialize a dataset and convert it to an array of packed Ruby hashes.
+`pack` can accept a dataset, an array of models, or a single model. Even when
+passing models that have already been materialized, the Packer will make sure
+to eagerly load any nested associations needed for packing. When passed a
+dataset or an array of models, `pack` will return an array of hashes, and when
+passed just a single model, it will return a single hash.
 
 ## Development
 

data/lib/sequel/packer/eager_hash.rb

@@ -1,5 +1,3 @@
-require 'sequel'
-
 module Sequel
   class Packer
     module EagerHash

data/lib/sequel/packer/eager_loading.rb

@@ -0,0 +1,102 @@
+module Sequel
+  class Packer
+    module EagerLoading
+      # This methods allows eager loading associations _after_ a record, or
+      # multiple records, have been fetched from the database. It is useful when
+      # you know you will be accessing model associations, but models have
+      # already been materialized.
+      #
+      # This method accepts a normalized_eager_hash, as specified by
+      # Sequel::Packer::EagerHash.
+      #
+      # This method will handle procs used to filter association datasets, but
+      # if an association has already been loaded for every model, the dataset
+      # will not be refetched and the proc will not be applied.
+      #
+      # This method borrows a lot from the #eager_load Sequel::Dataset method
+      # defined in Sequels lib/sequel/model/associations.rb.
+      def self.eager_load(model_class, model_or_models, normalized_eager_hash)
+        models = model_or_models.is_a?(Array) ?
+          model_or_models :
+          [model_or_models]
+
+        # Cache to avoid building id maps multiple times.
+        key_hash = {}
+
+        normalized_eager_hash.each do |association, nested_associations|
+          eager_block = nil
+
+          if EagerHash.is_proc_hash?(nested_associations)
+            eager_block, nested_associations = nested_associations.entries[0]
+          end
+
+          reflection = model_class.association_reflections[association]
+
+          # If all of the models have already loaded the association, we'll just
+          # recursively call ::eager_load to load nested associations.
+          if models.all? {|m| m.associations.key?(association)}
+            if nested_associations
+              associated_records = if reflection.returns_array?
+                models.flat_map {|m| m.send(association)}.uniq
+              else
+                models.map {|m| m.send(association)}.compact
+              end
+
+              eager_load(
+                reflection.associated_class,
+                associated_records,
+                nested_associations,
+              )
+            end
+          else
+            key = reflection.eager_loader_key
+            id_map = nil
+
+            if key && !key_hash[key]
+              id_map = Hash.new {|h, k| h[k] = []}
+
+              models.each do |model|
+                case key
+                when Symbol
+                  model_id = model.get_column_value(key)
+                  id_map[model_id] << model if model_id
+                when Array
+                  model_id = key.map {|col| model.get_column_value(col)}
+                  id_map[model_id] << model if model_id.all?
+                else
+                  raise(
+                    Sequel::Error,
+                    "unhandled eager_loader_key #{key.inspect} for " +
+                      "association #{association}",
+                  )
+                end
+              end
+            end
+
+            loader = reflection[:eager_loader]
+
+            loader.call(
+              key_hash: key_hash,
+              rows: models,
+              associations: nested_associations,
+              self: self,
+              eager_block: eager_block,
+              id_map: id_map,
+            )
+
+            if reflection[:after_load]
+              models.each do |model|
+                model.send(
+                  :run_association_callbacks,
+                  reflection,
+                  :after_load,
+                  model.associations[association],
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sequel/packer/version.rb

@@ -1,5 +1,5 @@
 module Sequel
   class Packer
-    VERSION = "0.3.0"
+    VERSION = "0.4.0"
   end
 end

data/lib/sequel/packer.rb

@@ -1,3 +1,5 @@
+require 'sequel'
+
 module Sequel
   class Packer
     # For invalid arguments provided to the field class method.
@@ -13,6 +15,7 @@ module Sequel
       subclass.instance_variable_set(:@class_traits, {})
       subclass.instance_variable_set(:@class_packers, {})
       subclass.instance_variable_set(:@class_eager_hash, nil)
+      subclass.instance_variable_set(:@class_precomputations, [])
     end
 
     def self.model(klass)
@@ -96,6 +99,13 @@ module Sequel
       )
     end
 
+    def self.precompute(&block)
+      if !block
+        raise ArgumentError, 'Sequel::Packer.precompute must be passed a block'
+      end
+      @class_precomputations << block
+    end
+
     def initialize(*traits)
       @subpackers = nil
 
@@ -104,10 +114,12 @@ module Sequel
         @instance_fields = class_fields
         @instance_packers = class_packers
         @instance_eager_hash = class_eager_hash
+        @instance_precomputations = class_precomputations
       else
         @instance_fields = class_fields.dup
         @instance_packers = class_packers.dup
         @instance_eager_hash = EagerHash.deep_dup(class_eager_hash)
+        @instance_precomputations = class_precomputations.dup
       end
 
       # Evaluate trait blocks, which might add new fields to @instance_fields,
@@ -136,10 +148,70 @@ module Sequel
       end
     end
 
-    def pack(dataset)
-      dataset = dataset.eager(@instance_eager_hash) if @instance_eager_hash
-      models = dataset.all
-      pack_models(models)
+    def pack(to_be_packed)
+      case to_be_packed
+      when Sequel::Dataset
+        if @instance_eager_hash
+          to_be_packed = to_be_packed.eager(@instance_eager_hash)
+        end
+        models = to_be_packed.all
+
+        run_precomputations(models)
+        pack_models(models)
+      when Sequel::Model
+        if @instance_eager_hash
+          EagerLoading.eager_load(
+            class_model,
+            [to_be_packed],
+            @instance_eager_hash
+          )
+        end
+
+        run_precomputations([to_be_packed])
+        pack_model(to_be_packed)
+      when Array
+        if @instance_eager_hash
+          EagerLoading.eager_load(
+            class_model,
+            to_be_packed,
+            @instance_eager_hash
+          )
+        end
+
+        run_precomputations(to_be_packed)
+        pack_models(to_be_packed)
+      when NilClass
+        nil
+      end
+    end
+
+    private
+
+    def run_precomputations(models)
+      @instance_packers.each do |association, _|
+        subpacker = @subpackers[association]
+        next if !subpacker.send(:has_precomputations?)
+
+        reflection = class_model.association_reflection(association)
+
+        if reflection.returns_array?
+          all_associated_records = models.flat_map {|m| m.send(association)}.uniq
+        else
+          all_associated_records = models.map {|m| m.send(association)}.compact
+        end
+
+        subpacker.send(:run_precomputations, all_associated_records)
+      end
+
+      @instance_precomputations.each do |block|
+        instance_exec(models, &block)
+      end
+    end
+
+    def has_precomputations?
+      return true if @instance_precomputations.any?
+      return false if !@subpackers
+      @subpackers.values.any? {|sp| sp.send(:has_precomputations?)}
     end
 
     def pack_model(model)
@@ -174,20 +246,17 @@ module Sequel
       packer = @subpackers[association]
 
       if associated_models.is_a?(Array)
-        packer.pack_models(associated_models)
+        packer.send(:pack_models, associated_models)
       else
-        packer.pack_model(associated_models)
+        packer.send(:pack_model, associated_models)
       end
     end
 
-    private
-
     def field(field_name=nil, packer_class=nil, *traits, &block)
       klass = self.class
-      model = klass.instance_variable_get(:@model)
 
       Validation.check_field_arguments(
-        model, field_name, packer_class, traits, &block)
+        class_model, field_name, packer_class, traits, &block)
       field_type =
         klass.send(:determine_field_type, field_name, packer_class, block)
 
@@ -203,17 +272,12 @@ module Sequel
     end
 
     def set_association_packer(association, packer_class, *traits)
-      model = self.class.instance_variable_get(:@model)
       Validation.check_association_packer(
-        model, association, packer_class, traits)
+        class_model, association, packer_class, traits)
 
       @instance_packers[association] = [packer_class, traits]
     end
 
-    def eager_hash
-      @instance_eager_hash
-    end
-
     def eager(*associations)
       @instance_eager_hash = EagerHash.merge!(
         @instance_eager_hash,
@@ -221,6 +285,21 @@ module Sequel
       )
     end
 
+    def precompute(&block)
+      if !block
+        raise ArgumentError, 'Sequel::Packer.precompute must be passed a block'
+      end
+      @instance_precomputations << block
+    end
+
+    def eager_hash
+      @instance_eager_hash
+    end
+
+    def class_model
+      self.class.instance_variable_get(:@model)
+    end
+
     def class_fields
       self.class.instance_variable_get(:@class_fields)
     end
@@ -236,9 +315,14 @@ module Sequel
     def class_traits
       self.class.instance_variable_get(:@class_traits)
     end
+
+    def class_precomputations
+      self.class.instance_variable_get(:@class_precomputations)
+    end
   end
 end
 
 require 'sequel/packer/eager_hash'
+require 'sequel/packer/eager_loading'
 require 'sequel/packer/validation'
 require "sequel/packer/version"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sequel-packer
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Paul Julius Martinez
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-14 00:00:00.000000000 Z
+date: 2020-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -128,6 +128,7 @@ files:
 - bin/setup
 - lib/sequel/packer.rb
 - lib/sequel/packer/eager_hash.rb
+- lib/sequel/packer/eager_loading.rb
 - lib/sequel/packer/validation.rb
 - lib/sequel/packer/version.rb
 - sequel-packer.gemspec