sequel-packer 0.3.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f7666c0db1cac9f34075e40a398a5963b33e6756d38507953dc14c59d5712d2
-  data.tar.gz: 10bf3ff431ba9fab4ff6bdfec84e51f1243cf3da1b5eff3ea66d6b7b45f94fbc
+  metadata.gz: 8782014d5d4fd8a6da590674ebc1d8d7a248f13ab80a1c571a0036f5a158af4f
+  data.tar.gz: d58397494def47b14aba23ce8f3dd2006936116e1273016f5bafb8e22c20809a
 SHA512:
-  metadata.gz: 67d08fbd8aef15f7005ae59b411f87a237d83b53aeb26e7e92efff2a628e6ae29bf14724cfedea03f34b5f7cf61e4c8d28be6cc4e01027535d04519fa681f7e1
-  data.tar.gz: c96e1f683319af1a20d6cb775f6b0ac8a479297947b6a26008082e4a8aa8e35a677de7d3a8f66a7a74fc57079fb8ecab1b678201a214f5768fc57eef61b4b150
+  metadata.gz: a30ae3faee56c47a4da2efcddb0a09cb5cbf892451968d010fddd2c3b937b59caadeb3128c78c4ca94d8cfaf0d2741755459fb1729c4f6fbb9b97f2787289201
+  data.tar.gz: 109e270732bec0bb7311f5d5f5a7e5f9c6f5f91e4a685bbdec5e20f28a38ef92093fd64d56067e6e64a3a718c2076df700b82e401f2004898aee846dc6cd879b

data/CHANGELOG.md

@@ -1,3 +1,30 @@
+### 1.0.1 (2021-08-02)
+
+* Update internal method call to remove "Using the last argument as
+  keyword parameters is deprecated" warning.
+
+### 1.0.0 (2020-05-18)
+
+* Version 1.0.0 release! No changes since 0.5.0 except some small changes to the
+  README.
+
+### 0.5.0 (2020-05-17)
+
+* Add `**context` argument to `#pack` method, exposed as `@context` in blocks
+  passed to `field` and `trait`.
+* Add `::with_context(&block)`, for accessing `@context` to use in additional
+  DSL calls, or modify data fetching.
+* Update README some re-organization and table of contents.
+
+### 0.4.0 (2020-05-17)
+
+* **_BREAKING CHANGE:_** `#pack_models` and `#pack_model` have been changed to
+  private methods. In their place `Sequel::Packer#pack` has been changed to
+  accept a dataset, an array of models or a single model, while still ensuring
+  eager loading takes place.
+* Add `self.precompute(&block)` for performing bulk computations outside of
+  Packer paradigm.
+
 ### 0.3.0 (2020-05-14)
 
 * Add `self.set_association_packer(association, packer_class, *traits)` and

data/README.md

@@ -1,24 +1,138 @@
 # Sequel::Packer
 
-`Sequel::Packer` is a Ruby JSON serialization library to be used with the Sequel
-ORM offering the following features:
+`Sequel::Packer` is a Ruby serialization library to be used with the [Sequel
+ORM](https://github.com/jeremyevans/sequel) with the following qualities:
 
-* *Declarative:* Define the shape of your serialized data with a simple,
+* **Declarative:** Define the shape of your serialized data with a simple,
   straightforward DSL.
-* *Flexible:* Certain contexts require different data. Packers provide an easy
+* **Flexible:** Certain contexts require different data. Packers provide an easy
   way to opt-in to serializing certain data only when you need it. The library
   also provides convenient escape hatches when you need to do something not
   explicitly supported by the API.
-* *Reusable:* The Packer library naturally composes well with itself. Nested
+* **Reusable:** The Packer library naturally composes well with itself. Nested
   data can be serialized in the same way no matter what endpoint it's fetched
   from.
-* *Efficient:* When not using Sequel's
+* **Efficient:** When not using Sequel's
   [`TacticalEagerLoading`](https://sequel.jeremyevans.net/rdoc-plugins/classes/Sequel/Plugins/TacticalEagerLoading.html)
   plugin, the Packer library will intelligently determine which associations
   and nested associations it needs to eager load in order to avoid any N+1 query
   issues.
 
-## Installation
+## Example
+
+`Sequel::Packer` uses your existing `Sequel::Model` declarations and leverages
+the use of associations to efficiently serialize data.
+
+```ruby
+class User < Sequel::Model(:users)
+  one_to_many :posts
+end
+class Post < Sequel::Model(:posts); end
+```
+
+Packer definitions use a simple domain-specific language (DSL) to declare which
+fields to serialize:
+
+```ruby
+class PostPacker < Sequel::Packer
+  model Post
+
+  field :id
+  field :title
+
+  trait :truncated_content do
+    field :truncated_content do |post|
+      post.content[0..Post::PREVIEW_LENGTH]
+    end
+  end
+end
+
+class UserPacker < Sequel::Packer
+  model User
+
+  field :id
+  field :name
+
+  trait :posts do
+    field :posts, PostPacker, :truncated_content
+  end
+end
+```
+
+Once defined, Packers are easy to use; just call `.pack` and pass in a Sequel
+dataset, an array of models, or a single model, and get back Ruby hashes.
+From there you can simply call `to_json` on the result!
+
+```ruby
+UserPacker.pack(User.dataset)
+=> [
+  {id: 1, name: 'Paul'},
+  {id: 2, name: 'Julius'},
+  ...
+]
+
+UserPacker.pack(User[1], :posts)
+=> {
+  id: 1,
+  name: 'Paul',
+  posts: [
+    {
+      id: 15,
+      title: 'Announcing Sequel::Packer!',
+      truncated_content: 'Sequel::Packer is a new gem...',
+    },
+    {
+      id: 21,
+      title: 'Postgres Internals',
+      truncated_content: 'I never quite understood autovacuum...',
+    },
+    ...
+  ],
+}
+```
+
+## Contents
+
+- [Example](#example)
+- [Getting Started](#getting-started)
+  - [Installation](#installation)
+  - [Example Schema](#example-schema)
+  - [Basic Fields](#basic-fields)
+  - [Packing Associations by Nesting Packers](#packing-associations-by-nesting-packers)
+  - [Traits](#traits)
+- [API Reference](#api-reference)
+  - [Using a Packer](#using-a-packer)
+  - [Defining a Packer](#defining-a-packer)
+    - [`self.model(sequel_model_class)`](#selfmodelsequel_model_class)
+    - [`self.field(column_name)` (or `self.field(method_name)`)](#selffieldcolumn_name-or-selffieldmethod_name)
+    - [`self.field(key, &block)`](#selffieldkey-block)
+    - [`self.field(association, subpacker, *traits)`](#selffieldassociation-subpacker-traits)
+    - [`self.field(&block)`](#selffieldblock)
+    - [`self.trait(trait_name, &block)`](#selftraittrait_name-block)
+    - [`self.eager(*associations)`](#selfeagerassociations)
+    - [`self.set_association_packer(association, subpacker, *traits)`](#selfset_association_packerassociation-subpacker-traits)
+    - [`self.pack_association(association, models)`](#selfpack_associationassociation-models)
+    - [`self.precompute(&block)`](#selfprecomputeblock)
+  - [Context](#context)
+    - [`self.with_context(&block)`](#selfwith_contextblock)
+- [Potential Future Functionality](#potential-future-functionality)
+  - [Automatically Generated Type Declarations](#automatically-generated-type-declarations)
+  - [Lifecycle Hooks](#lifecycle-hooks)
+  - [Less Data Fetching](#less-data-fetching)
+  - [Other Enhancements](#other-enhancements)
+- [Contributing](#contributing)
+  - [Development](#development)
+  - [Releases](#releases)
+- [Attribution](#attribution)
+- [License](#license)
+
+## Getting Started
+
+This section will explain the basic use of `Sequel::Packer`. Check out the [API
+Reference](#api-reference) for an exhaustive coverage of the API and more
+detailed documentation.
+
+### Installation
 
 Add this line to your application's Gemfile:
 
@@ -34,9 +148,9 @@ Or install it yourself as:
 
     $ gem install sequel-packer
 
-## Usage
+### Example Schema
 
-Suppose we have the following basic database schema:
+Most of the following examples will use the following database schema:
 
 ```ruby
 DB.create_table(:users) do
@@ -72,7 +186,7 @@ end
 ### Basic Fields
 
 Suppose an endpoint wants to fetch all the ten most recent comments by a user.
-After validating the user id, we end up with the Sequel dataset represting the
+After validating the user id, we end up with the Sequel dataset representing the
 data we want to return:
 
 ```ruby
@@ -97,7 +211,7 @@ end
 This can then be used as follows:
 
 ```ruby
-CommentPacker.new.pack(recent_comments)
+CommentPacker.pack(recent_comments)
 => [
   {id: 536, content: "Great post, man!"},
   {id: 436, content: "lol"},
@@ -105,10 +219,10 @@ CommentPacker.new.pack(recent_comments)
 ]
 ```
 
-### Packing associations by nesting Packers
+### Packing Associations by Nesting Packers
 
 Now, suppose that we want to fetch a post and all of its comments. We can do
-this by defining another packer for Post that uses the CommentPacker:
+this by defining another packer for `Post` that uses the `CommentPacker`:
 
 ```ruby
 class PostPacker < Sequel::Packer
@@ -121,14 +235,15 @@ class PostPacker < Sequel::Packer
 end
 ```
 
-Since `post.comments` is an array of Sequel::Models and not a primitive value,
-we must tell the Packer how to serialize them using another packer. This is
-what the second argument in `field :comments, CommentPacker` is doing.
+Since `post.comments` is an array of `Sequel::Models` and not a primitive value,
+we must tell the Packer how to serialize them using another packer. The second
+argument in `field :comments, CommentPacker` tells the `PostPacker` to use the
+pack those comments using the `CommentPacker`.
 
 We can then use this as follows:
 
 ```ruby
-PostPacker.new.pack(Post.order(:id.desc).limit(1))
+PostPacker.pack(Post[validated_id])
 => [
   {
     id: 682,
@@ -158,8 +273,8 @@ end
 ```
 
 We could now define a new packer, `CommentWithAuthorPacker`, and use that in the
-PostPacker instead, but then we'd have to redeclare all the other fields we want
-on a packed Comment:
+`PostPacker` instead, but then we'd have to redeclare all the other fields we
+want on a packed `Comment`:
 
 ```ruby
 class CommentWithAuthorPacker < Sequel::Packer
@@ -184,7 +299,7 @@ end
 Declaring these fields in two places could cause them to get out of sync
 as more fields are added. Instead, we will use a _trait_. A _trait_ is a
 way to define a set of fields that we only want to pack sometimes. Instead
-of defining a totally new packer, we can extend the CommentPacker as follows:
+of defining a totally new packer, we can extend the `CommentPacker` as follows:
 
 ```ruby
 class CommentPacker < Sequel::Packer
@@ -199,18 +314,18 @@ class CommentPacker < Sequel::Packer
 end
 ```
 
-To use a trait, simply pass it in when creating the packer instance:
+To use a trait, simply pass it in when calling `pack`:
 
 ```ruby
 # Without the trait
-CommentPacker.new.pack(Comment.dataset)
+CommentPacker.pack(Comment.dataset)
 => [
   {id: 536, content: "Great post, man!"},
   ...
 ]
 
 # With the trait
-CommentPacker.new(:author).pack(Comment.dataset)
+CommentPacker.pack(Comment.dataset, :author)
 => [
   {
     id: 536,
@@ -249,13 +364,64 @@ Custom packers are written by creating subclasses of `Sequel::Packer`. This
 class defines a DSL for declaring how a Sequel Model will be converted into a
 plain Ruby hash.
 
-### `self.model(sequel_model_class)`
+### Using a Packer
+
+Using a Packer is dead simple. There's a single class method:
+
+```ruby
+self.pack(data, *traits, **context)
+```
+
+`data` can be in the form of a Sequel dataset, an array of Sequel models, or
+a single Sequel model. No matter which form the data is passed in, the Packer
+class will ensure nested data is efficiently loaded.
+
+To pack additional fields defined in a trait, pass the name of the trait as an
+additional argument, e.g., `UserPacker.pack(users, :recent_posts)` to include
+recent posts with each user.
+
+Finally, additional context can be provided to the Packer by passing additional
+keyword arguments to `pack`. This context is handled opaquely by the Packer, but
+it can be accessed in the blocks passed to `field` declarations. Common uses of
+`context` include passing in the current user making a request, or passing in
+additional precomputed data.
+
+The implementation of `pack` is very simple. It creates an instance of a Packer,
+by passing in the traits and the context, then calls `pack` on that instance,
+and passes in the data:
+
+```ruby
+def self.pack(data, *traits, **context)
+  return nil if !data # small easy optimization to avoid unnecessary work
+  new(*traits, **context).pack(data)
+end
+```
+
+It simply combines a constructor and single exposed instance method:
+
+#### `initialize(*traits, **context)`
+
+#### `pack(data)`
+
+One instantiated, the same Packer could be used to pack data multiple times.
+This is unlikely to be needed, but the functionality is there.
+
+### Defining a Packer
+
+#### `self.model(sequel_model_class)`
 
 The beginning of each Packer class must begin with `model MySequelModel`, which
 specifies which Sequel Model this Packer class will serialize. This is mostly
-to catch certain errors at load time, rather than at run time.
+to catch certain errors at load time, rather than at run time:
+
+```ruby
+class UserPacker < Sequel::Packer
+  model User
+  ...
+end
+```
 
-### `self.field(column_name)` (or `self.field(method_name)`)
+#### `self.field(column_name)` (or `self.field(method_name)`)
 
 Defining the shape of the outputted data is done using the `field` method, which
 exists in four different variants. This first variant is the simplest. It simply
@@ -283,7 +449,7 @@ Then when `User.create(first_name: "Paul", last_name: "Martinez")` gets packed
 with `field :full_name` specified, the outputted hash will contain
 `full_name: "Paul Martinez"`.
 
-### `self.field(key, &block)`
+#### `self.field(key, &block)`
 
 A block can be passed to `field` to perform arbitrary computation and store the
 result under the specified `key`. The block will be passed the model as a single
@@ -309,7 +475,7 @@ class MyPacker < Sequel::Packer
 end
 ```
 
-### `self.field(association, packer_class, *traits)`
+#### `self.field(association, subpacker, *traits)`
 
 A Sequel association (defined in the model file using `one_to_many`, or
 `many_to_one`, etc.), can be packed using another Packer class, possibly with
@@ -317,15 +483,16 @@ multiple traits specified. A similar output could be generated by doing:
 
 ```ruby
 field :association do |model|
-  packer_class.new(*traits).pack(model.association_dataset)
+  subpacker.pack(model.association_dataset, *traits)
 end
 ```
 
-Though this version of course would result in many more queries to the database,
-which are not required when using the shorthand form, and also requires creating
-a new instance of `packer_class` for every packed model.
+This form is very inefficient though, because it would result in a new subpacker
+getting instantiated for every packed model. Additionally, unless the subpacker
+is declared up-front, the Packer won't know to eager load that association,
+potentially resulting in many unnecessary database queries.
 
-### `self.field(&block)`
+#### `self.field(&block)`
 
 Passing a block but no `key` to `field` allows for arbitrary manipulation of the
 packed hash. The block will be passed the model and the partially packed hash.
@@ -338,7 +505,7 @@ field do |model, hash|
 end
 ```
 
-### `self.trait(trait_name, &block)`
+#### `self.trait(trait_name, &block)`
 
 Define optional serialization behavior by defining additional fields within a
 `trait` block. Traits can be opted into when initializing a packer by passing
@@ -347,15 +514,19 @@ the name of the trait as an argument:
 ```ruby
 class MyPacker < Sequel::Packer
   model MyObj
+  field :id
+
   trait :my_trait do
-    field :my_optional_field
+    field :trait_field
   end
 end
 
-# packed objects don't have my_optional_field
-MyPacker.new.pack(dataset)
-# packed objects do have my_optional_field
-MyPacker.new(:my_trait).pack(dataset)
+# packed objects don't have trait_field
+MyPacker.pack(dataset)
+=> [{id: 1}, {id: 2}, ...]
+# packed objects do have trait_field
+MyPacker.pack(dataset, :my_trait)
+=> [{id: 1, trait_field: 'foo'}, {id: 2, trait_field: 'bar'}, ...]
 ```
 
 Traits can also be used when packing associations by passing the name of the
@@ -368,7 +539,7 @@ class MyOtherPacker < Sequel::Packer
 end
 ```
 
-### `self.eager(*associations)`
+#### `self.eager(*associations)`
 
 When packing an association, a Packer will automatically ensure that association
 is eager loaded, but there may be cases when an association will be accessed
@@ -392,7 +563,7 @@ class UserPacker < Sequel::Packer
   end
 end
 
-UserPacker.new.pack(User.dataset)
+UserPacker.pack(User.dataset)
 => [
   {id: 123, num_posts: 7},
   {id: 456, num_posts: 3},
@@ -400,7 +571,7 @@ UserPacker.new.pack(User.dataset)
 ]
 ```
 
-This helps prevent N+1 query problems when not using Sequel's
+Using `eager` can help prevent N+1 query problems when not using Sequel's
 [`TacticalEagerLoading`](https://sequel.jeremyevans.net/rdoc-plugins/classes/Sequel/Plugins/TacticalEagerLoading.html)
 plugin.
 
@@ -421,22 +592,28 @@ class UserPacker < Sequel::Packer
 end
 ```
 
-Keep in mind that this limits the association that gets used by ALL fields, so
-if another field actually needs access to all the users posts, it might not make
-sense to use `eager`.
+**IMPORTANT NOTE:** Eager procs are not guaranteed to be executed when passing
+in models, rather than a dataset, to `pack`. Specifically, if the models already
+have fetched the association, the Packer won't refetch it. Because of this, it's
+good practice to use `set_association_packer` and `pack_association` (see next
+section) in a `field` block and duplicate the filtering action.
+
+Also keep in mind that this limits the association that gets used by ALL fields,
+so if another field actually needs access to all the users posts, it might not
+make sense to use `eager`.
 
-Also, it's important to note that if `eager` is called multiple times, with
-multiple procs, each proc will get applied to the dataset, likely resulting in
-overly restrictive filtering.
+Additionally, it's important to note that if `eager` is called multiple times,
+with multiple procs, each proc will get applied to the dataset, likely resulting
+in overly restrictive filtering.
 
-### `self.set_association_packer(association, packer_class, *traits)`
+#### `self.set_association_packer(association, subpacker, *traits)`
 
 See `self.pack_association(association, models)` below.
 
-### `self.pack_association(association, models)`
+#### `self.pack_association(association, models)`
 
 The simplest way to pack an association is to use
-`self.field(association, packer_class, *traits)`, but sometimes this doesn't do
+`self.field(association, subpacker, *traits)`, but sometimes this doesn't do
 exactly what we want. We may want to pack the association under a different key
 than the name of the association. Or we may only want to pack some of the
 associated models (and it may be difficult or impossible to express which subset
@@ -475,7 +652,7 @@ but if it is passed a single model, it will return just that packed model.
 
 Examples:
 
-#### Use a different field name than the name of the association
+##### Use a different field name than the name of the association
 ```ruby
 set_association_packer :ugly_internal_names, InternalPacker
 field :nice_external_names do |model|
@@ -483,7 +660,7 @@ field :nice_external_names do |model|
 end
 ```
 
-#### Pack a single instance of a `one_to_many` association
+##### Pack a single instance of a `one_to_many` association
 ```ruby
 class PostPacker < Sequel::Packer
   set_association_packer :comments, CommentPacker
@@ -493,34 +670,213 @@ class PostPacker < Sequel::Packer
 end
 ```
 
+#### `self.precompute(&block)`
 
-### `initialize(*traits)`
+Occasionally packing a model may require a computation that doesn't fit in with
+the rest of the Packer paradigm. This may be a Sequel query that is particularly
+difficult to express as an association, or even a call to an external service.
+If such a computation can be performed in bulk, then the `precompute` method can
+be used as an entry point for that operation.
 
-When creating an instance of a Packer class, pass in any traits desired to
-specify what additional data should be packed, if any.
+The `precompute` method will execute a given block and pass it all of the models
+that will be packed using that packer. This block will be executed a single
+time, even when called by a deeply nested packer.
 
-### `pack(dataset)`
+The `precompute` block is `instance_exec`ed in the context of the packer
+instance, the result of any computation can be saved in a simple instance
+variable (`@precomputed_result`) and later referenced inside the blocks that are
+passed to `field` methods.
 
-After creating a new instance of a Packer class, call `packer.pack(dataset)` to
-materialize a dataset and convert it to an array of packed Ruby hashes.
+As an example, suppose a video uploading platform performs additional video
+processing on every uploaded video and exposes the status of that processing as
+a separate service over the network, rather than directly with the upload
+metadata in the database. `precompute` could be used as follows:
 
-## Development
+```ruby
+class VideoUploadPacker < Sequel::Packer
+  model VideoUpload
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run
-`rake test` to run the tests. You can also run `bin/console` for an interactive
-prompt that will allow you to experiment.
+  precompute do |video_uploads|
+    @processing_statuses = ResolutionService
+      .get_status_bulk(ids: video_uploads.map(&:id))
+  end
 
-To install this gem onto your local machine, run `bundle exec rake install`. To
-release a new version, update the version number in `version.rb`, and then run
-`bundle exec rake release`, which will create a git tag for the version, push
-git commits and tags, and push the `.gem` file to
-[rubygems.org](https://rubygems.org).
+  field :id
+  field :filename
+  field :processing_status do |video_upload|
+    @processing_statuses[video_upload.id]
+  end
+end
+```
+
+#### Instance method versions
+
+In addition to the class method versions of `field`, `eager`,
+`set_association_packer`, and `precompute`, there are also regular instance method
+versions which take the exact same arguments. When writing a `trait` block, the
+block is evaulated in the context of a new Packer instance and actually calls the
+instance method versions instead.
+
+### Context
+
+In addition to the data to be packed, and a set of traits, the `pack` method
+also accepts arbitrary keyword arguments. This is referred to as `context` is
+handled opaquely by the Packer. The data passed in here is saved as the
+`@context` instance variable, which is then accessible from within the blocks
+passed to `field`, `trait`, and `precompute`, for whatever purpose. It is also
+automatically passed to any nested subpackers.
+
+The most common usage for context would be to pass in the current user making
+a request. It could then be used to pack permission levels about records, for
+example.
+
+```ruby
+class PostPacker < Sequel::Packer
+  model Post
+
+  eager :permissions
+  field :access_level do |post|
+    user_permission = post.permissions.find do |perm|
+      perm.user_id == @context[:user].id
+    end
+
+    user_permission.access_level
+  end
+end
+```
+
+You might notice something inefficient about the above code. Even though we only
+want to look at the user's permission record, we fetch ALL of the permission
+records for each Post. Ideally we would filter the `permissions` association
+dataset when we call `eager`, but we don't have access to `@context` at that
+point. This leads to the final DSL method available when writing a Packer:
+
+#### `self.with_context(&block)`
+
+You can pass a block to `with_context` that will be executed as soon as a Packer
+instance is constructed. The block can access `@context` and can also call the
+standard Packer DSL methods, `field`, `eager`, etc.
+
+The above example could then be made more efficient as follows:
+
+```ruby
+class PostPacker < Sequel::Packer
+  model Post
+
+- eager :permissions
++ with_context do
++   eager permissions: (proc {|ds| ds.where(user_id: @context[:user].id)})
++ end
+end
+```
+
+A very tricky usage of `with_context` (and not recommended...) would be to
+control the traits used on subpackers:
+
+```ruby
+class UserPacker < Sequel::Packer
+  model User
+
+  with_context do
+    field :comments, CommentPacker, *@context[:comment_traits]
+  end
+end
+
+UserPacker.pack(User.dataset, comment_traits: [])
+=> [{comments: [{id: 7}, ...]}]
+UserPacker.pack(User.dataset, comment_traits: [:author])
+=> [{comments: [{id: 7, author: {id: 1, ...}}, ...]}]
+UserPacker.pack(User.dataset, comment_traits: [:num_likes])
+=> [{comments: [{id: 7, likes: 53}, ...]}]
+```
+
+## Potential Future Functionality
+
+The 1.0.0 version of the Packer library is flexible to support many use cases.
+That said, of course there are ways to improve it! There are three main
+improvements I can imagine adding:
+
+### Automatically Generated Type Declarations
+
+It would be fairly easy to add generate type definitions by adding arguments to
+`field`. Packers could produce [TypeScript
+interface](https://www.typescriptlang.org/docs/handbook/interfaces.html)
+declarations, and adding a simple build step to a CI pipeline could enforce type
+safety across the frontend and backend. Or they could produce
+[OpenAPI](http://spec.openapis.org/oas/v3.0.3) specifications, which could then
+be used to automatically generate clients using something like
+[Swagger](https://swagger.io/).
+
+### Lifecycle Hooks
+
+It should be fairly easy to extend the Packer library using standard Ruby
+features like subclassing, mixins, via `include`, or even monkey-patching. It
+may be beneficial to have explicit hooks for common operations however, like
+`before_fetch`, or `around_pack`. It's more likely that these hooks are needed
+for logging and tracing capabilities, than for actual functionality, so I'd
+like to see some real-world usage before committing to a specific style of
+integration.
+
+### Less Data Fetching
+
+Sequel by default fetches every column in a table, but a Packer knows (roughly)
+what data is going to be used so it could only select the columns neede for
+actual serialization, and limit how much data is actually fetched from the
+database. I haven't done any benchmarking on this, so I'm not sure how much of
+a benefit could be gained by this, but it would be interesting!
+
+This could work roughly as follows:
+
+* Start by fetching all columns that appear in simple `field(:column_name)`
+  declarations
+* Add any columns need to fetch nested associations, or to re-asscociate fetched
+  records with their "parent" models, using the `left_key` and `right_key`
+  fields of the `AssociationReflections`
+* Add a `column(*columns)` DSL method to explicitly fetch additional columns
+
+### Other Enhancements
+
+Here are some other potential enhancements, though these are less fleshed out.
+
+* Support not including a key in a hash if the associated value is nil, to
+  reduce size of outputted data.
+* Support different casing of the outputted hashes, i.e., `snake_case` vs.
+  `camelCase`.
+* Explicitly support different output formats, rather than just plain Ruby
+  hashes, such as [Protocol
+  Buffers](https://developers.google.com/protocol-buffers) or [Cap'n
+  Proto](https://capnproto.org/).
+* When using nested `precompute` blocks, the Packer has to flatten the
+  associations of a model, which may be expensive, but has not been benchmarked.
+  These flattened arrays already exist internally in Sequel when the eager
+  loading occurs, but those aren't exposed. The code in Sequel could be
+  re-implemented as part of the library to avoid re-constructing those arrays.
 
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at
 https://github.com/PaulJuliusMartinez/sequel-packer.
 
+### Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run
+`rake test` to run the tests. You can also run `bin/console` for an interactive
+prompt that will allow you to experiment.
+
+### Releases
+
+To release a new version, update the version number in
+`lib/sequel/packer/version.rb`, update the `CHANGELOG.md` with new changes, then
+run `rake release`, which which will create a git tag for the version, push git
+commits and tags, and push the `.gem` file to
+[rubygems.org](https://rubygems.org).
+
+## Attribution
+
+[Karthik Viswanathan](https://github.com/karthikv) designed the original API
+of the Packer library while at [Affinity](https://www.affinity.co/). This
+library is a ground up rewrite which defines a very similar API, but shares no
+code with the original implementation.
 
 ## License
 

data/lib/sequel/packer.rb

@@ -1,3 +1,5 @@
+require 'sequel'
+
 module Sequel
   class Packer
     # For invalid arguments provided to the field class method.
@@ -7,14 +9,33 @@ module Sequel
     class AssociationDoesNotExistError < StandardError; end
     class InvalidAssociationPackerError < StandardError; end
     class UnknownTraitError < StandardError; end
+    class UnnecessaryWithContextError < StandardError; end
+    class NoAssociationSubpackerDefinedError < StandardError; end
 
+    # Think of this method as the "initialize" method for a Packer class.
+    # Every Packer class keeps track of the fields, traits, and other various
+    # operations defined using the DSL internally.
     def self.inherited(subclass)
-      subclass.instance_variable_set(:@class_fields, [])
-      subclass.instance_variable_set(:@class_traits, {})
-      subclass.instance_variable_set(:@class_packers, {})
-      subclass.instance_variable_set(:@class_eager_hash, nil)
+      subclass.instance_variable_set(:@model, @model)
+      subclass.instance_variable_set(:@class_fields, @class_fields&.dup || [])
+      subclass.instance_variable_set(:@class_traits, @class_traits&.dup || {})
+      subclass.instance_variable_set(:@class_packers, @class_packers&.dup || {})
+      subclass.instance_variable_set(
+        :@class_eager_hash,
+        EagerHash.deep_dup(@class_eager_hash),
+      )
+      subclass.instance_variable_set(
+        :@class_precomputations,
+        @class_precomputations&.dup || [],
+      )
+      subclass.instance_variable_set(
+        :@class_with_contexts,
+        @class_with_contexts&.dup || [],
+      )
     end
 
+    # Declare the type of Sequel::Model this Packer will be used for. Used to
+    # validate associations at declaration time.
     def self.model(klass)
       if !(klass < Sequel::Model)
         fail(
@@ -32,18 +53,36 @@ module Sequel
     METHOD_FIELD = :method_field
     # field(:foo, &block)
     BLOCK_FIELD = :block_field
-    # field(:association, packer_class)
+    # field(:association, subpacker)
     ASSOCIATION_FIELD = :association_field
     # field(&block)
     ARBITRARY_MODIFICATION_FIELD = :arbitrary_modification_field
 
-    def self.field(field_name=nil, packer_class=nil, *traits, &block)
+    # Declare a field to be packed in the output hash. This method can be called
+    # in multiple ways:
+    #
+    # field(:field_name)
+    # - Calls the method :field_name on a model and stores the result under the
+    #   key :field_name in the packed hash.
+    #
+    # field(:field_name, &block)
+    # - Yields the model to the block and stores the result under the key
+    #   :field_name in the packed hash.
+    #
+    # field(:association, subpacker, *traits)
+    # - Packs model.association using the designated subpacker with the
+    #   specified traits.
+    #
+    # field(&block)
+    # - Yields the model and the partially packed hash to the block, allowing
+    #   for arbitrary modification of the output hash.
+    def self.field(field_name=nil, subpacker=nil, *traits, &block)
       Validation.check_field_arguments(
-        @model, field_name, packer_class, traits, &block)
-      field_type = determine_field_type(field_name, packer_class, block)
+        @model, field_name, subpacker, traits, &block)
+      field_type = determine_field_type(field_name, subpacker, block)
 
       if field_type == ASSOCIATION_FIELD
-        set_association_packer(field_name, packer_class, *traits)
+        set_association_packer(field_name, subpacker, *traits)
       end
 
       @class_fields << {
@@ -53,15 +92,10 @@ module Sequel
       }
     end
 
-    def self.set_association_packer(association, packer_class, *traits)
-      Validation.check_association_packer(
-        @model, association, packer_class, traits)
-      @class_packers[association] = [packer_class, traits]
-    end
-
+    # Helper for determing a field type from the arguments to field.
     private_class_method def self.determine_field_type(
       field_name,
-      packer_class,
+      subpacker,
       block
     )
       if block
@@ -71,7 +105,7 @@ module Sequel
           ARBITRARY_MODIFICATION_FIELD
         end
       else
-        if packer_class
+        if subpacker
           ASSOCIATION_FIELD
         else
           METHOD_FIELD
@@ -79,6 +113,17 @@ module Sequel
       end
     end
 
+    # Register that nested models related to the packed model by association
+    # should be packed using the given subpacker with the specified traits.
+    def self.set_association_packer(association, subpacker, *traits)
+      Validation.check_association_packer(
+        @model, association, subpacker, traits)
+      @class_packers[association] = [subpacker, traits]
+    end
+
+    # Define a trait, a set of optional fields that can be packed in certain
+    # situations. The block can call main Packer DSL methods: field,
+    # set_association_packer, eager, or precompute.
     def self.trait(name, &block)
       if @class_traits.key?(name)
         raise ArgumentError, "Trait :#{name} already defined"
@@ -89,6 +134,14 @@ module Sequel
       @class_traits[name] = block
     end
 
+    # Specify additional eager loading that should take place when fetching data
+    # to be packed. Commonly used to add filters to association datasets via
+    # eager procs.
+    #
+    # Users should not assume when using eager procs that the proc actually gets
+    # executed. If models with their associations already loaded are passed to
+    # pack then the proc will never get processed. Any filtering logic should be
+    # duplicated within a field block.
     def self.eager(*associations)
       @class_eager_hash = EagerHash.merge!(
         @class_eager_hash,
@@ -96,23 +149,68 @@ module Sequel
       )
     end
 
-    def initialize(*traits)
-      @subpackers = nil
+    # Declare an arbitrary operation to be performed one all the data has been
+    # fetched. The block will be executed once and be passed all of the models
+    # that will be packed by this Packer, even if this Packer is nested as a
+    # subpacker of other packers. The block can save the result of the
+    # computation in an instance variable which can then be accessed in the
+    # blocks passed to field.
+    def self.precompute(&block)
+      if !block
+        raise ArgumentError, 'Sequel::Packer.precompute must be passed a block'
+      end
+      @class_precomputations << block
+    end
+
+    # Declare a block to be called after a Packer has been initialized with
+    # context. The block can call the common Packer DSL methods. It is most
+    # commonly used to pass eager procs that depend on the Packer context to
+    # eager.
+    def self.with_context(&block)
+      if !block
+        raise ArgumentError, 'Sequel::Packer.with_context must be passed a block'
+      end
+      @class_with_contexts << block
+    end
 
-      # If there aren't any traits, we can just re-use the class variables.
-      if traits.empty?
-        @instance_fields = class_fields
-        @instance_packers = class_packers
-        @instance_eager_hash = class_eager_hash
-      else
-        @instance_fields = class_fields.dup
-        @instance_packers = class_packers.dup
-        @instance_eager_hash = EagerHash.deep_dup(class_eager_hash)
+    # Pack the given data with the specified traits and additional context.
+    # Context is automatically passed down to any subpackers.
+    #
+    # Data can be provided as a Sequel::Dataset, an array of Sequel::Models, a
+    # single Sequel::Model, or nil. Even when passing models that have already
+    # been materialized, eager loading will be used to efficiently fetch
+    # associations.
+    #
+    # Returns an array of packed hashes, or a single packed hash if a single
+    # model was passed in. Returns nil if nil was passed in.
+    def self.pack(data, *traits, **context)
+      return nil if !data
+      new(*traits, **context).pack(data)
+    end
+
+    # Initialize a Packer instance with the given traits and additional context.
+    # This Packer can then pack multiple datasets or models via the pack method.
+    def initialize(*traits, **context)
+      @context = context
+
+      @subpackers = {}
+
+      # Technically we only need to duplicate these fields if we modify any of
+      # them, but manually implementing some sort of copy-on-write functionality
+      # is messy and error prone.
+      @instance_fields = class_fields.dup
+      @instance_packers = class_packers.dup
+      @instance_eager_hash = EagerHash.deep_dup(class_eager_hash)
+      @instance_precomputations = class_precomputations.dup
+
+      class_with_contexts.each do |with_context_block|
+        self.instance_exec(&with_context_block)
       end
 
       # Evaluate trait blocks, which might add new fields to @instance_fields,
-      # new packers to @instance_packers, and/or new associations to
-      # @instance_eager_hash.
+      # new packers to @instance_packers, new associations to
+      # @instance_eager_hash, and/or new precomputations to
+      # @instance_precomputations.
       traits.each do |trait|
         trait_block = class_traits[trait]
         if !trait_block
@@ -123,10 +221,9 @@ module Sequel
       end
 
       # Create all the subpackers, and merge in their eager hashes.
-      @instance_packers.each do |association, (packer_class, traits)|
-        association_packer = packer_class.new(*traits)
+      @instance_packers.each do |association, (subpacker, traits)|
+        association_packer = subpacker.new(*traits, **@context)
 
-        @subpackers ||= {}
         @subpackers[association] = association_packer
 
         @instance_eager_hash = EagerHash.merge!(
@@ -136,12 +233,77 @@ module Sequel
       end
     end
 
-    def pack(dataset)
-      dataset = dataset.eager(@instance_eager_hash) if @instance_eager_hash
-      models = dataset.all
-      pack_models(models)
+    # Pack the given data with the traits and additional context specified when
+    # the Packer instance was created.
+    #
+    # Data can be provided as a Sequel::Dataset, an array of Sequel::Models, a
+    # single Sequel::Model, or nil. Even when passing models that have already
+    # been materialized, eager loading will be used to efficiently fetch
+    # associations.
+    #
+    # Returns an array of packed hashes, or a single packed hash if a single
+    # model was passed in. Returns nil if nil was passed in.
+    def pack(data)
+      case data
+      when Sequel::Dataset
+        data = data.eager(@instance_eager_hash) if @instance_eager_hash
+        models = data.all
+
+        run_precomputations(models)
+        pack_models(models)
+      when Sequel::Model
+        if @instance_eager_hash
+          EagerLoading.eager_load(class_model, [data], @instance_eager_hash)
+        end
+
+        run_precomputations([data])
+        pack_model(data)
+      when Array
+        if @instance_eager_hash
+          EagerLoading.eager_load(class_model, data, @instance_eager_hash)
+        end
+
+        run_precomputations(data)
+        pack_models(data)
+      when NilClass
+        nil
+      end
     end
 
+    private
+
+    # Run any blocks declared using precompute on the given models, as well as
+    # any precompute blocks declared by subpackers.
+    def run_precomputations(models)
+      @instance_packers.each do |association, _|
+        subpacker = @subpackers[association]
+        next if !subpacker.send(:has_precomputations?)
+
+        reflection = class_model.association_reflection(association)
+
+        if reflection.returns_array?
+          all_associated_records = models.flat_map {|m| m.send(association)}.uniq
+        else
+          all_associated_records = models.map {|m| m.send(association)}.compact
+        end
+
+        subpacker.send(:run_precomputations, all_associated_records)
+      end
+
+      @instance_precomputations.each do |block|
+        instance_exec(models, &block)
+      end
+    end
+
+    # Check if a Packer has any precompute blocks declared, to avoid the
+    # overhead of flattening the child associations.
+    def has_precomputations?
+      return true if @instance_precomputations.any?
+      return false if !@subpackers
+      @subpackers.values.any? {|sp| sp.send(:has_precomputations?)}
+    end
+
+    # Pack a single model by processing all of the Packer's declared fields.
     def pack_model(model)
       h = {}
 
@@ -164,35 +326,45 @@ module Sequel
       h
     end
 
+    # Pack an array of models by processing all of the Packer's declared fields.
     def pack_models(models)
       models.map {|m| pack_model(m)}
     end
 
+    # Pack models from an association using the designated subpacker.
     def pack_association(association, associated_models)
       return nil if !associated_models
 
       packer = @subpackers[association]
 
+      if !packer
+        raise(
+          NoAssociationSubpackerDefinedError,
+          "pack_association called for the #{class_model}.#{association} " +
+            'association, but no Packer has been set for that association.',
+        )
+      end
+
       if associated_models.is_a?(Array)
-        packer.pack_models(associated_models)
+        packer.send(:pack_models, associated_models)
       else
-        packer.pack_model(associated_models)
+        packer.send(:pack_model, associated_models)
       end
     end
 
-    private
-
-    def field(field_name=nil, packer_class=nil, *traits, &block)
+    # See the definition of self.field. This method accepts the exact same
+    # arguments. When fields are declared within trait blocks, this method is
+    # called rather than the class method.
+    def field(field_name=nil, subpacker=nil, *traits, &block)
       klass = self.class
-      model = klass.instance_variable_get(:@model)
 
       Validation.check_field_arguments(
-        model, field_name, packer_class, traits, &block)
+        class_model, field_name, subpacker, traits, &block)
       field_type =
-        klass.send(:determine_field_type, field_name, packer_class, block)
+        klass.send(:determine_field_type, field_name, subpacker, block)
 
       if field_type == ASSOCIATION_FIELD
-        set_association_packer(field_name, packer_class, *traits)
+        set_association_packer(field_name, subpacker, *traits)
       end
 
       @instance_fields << {
@@ -202,18 +374,19 @@ module Sequel
       }
     end
 
-    def set_association_packer(association, packer_class, *traits)
-      model = self.class.instance_variable_get(:@model)
+    # See the definition of self.set_association_packer. This method accepts the
+    # exact same arguments. When used within a trait block, this method is
+    # called rather than the class method.
+    def set_association_packer(association, subpacker, *traits)
       Validation.check_association_packer(
-        model, association, packer_class, traits)
-
-      @instance_packers[association] = [packer_class, traits]
-    end
+        class_model, association, subpacker, traits)
 
-    def eager_hash
-      @instance_eager_hash
+      @instance_packers[association] = [subpacker, traits]
     end
 
+    # See the definition of self.eager. This method accepts the exact same
+    # arguments. When used within a trait block, this method is called rather
+    # than the class method.
     def eager(*associations)
       @instance_eager_hash = EagerHash.merge!(
         @instance_eager_hash,
@@ -221,6 +394,39 @@ module Sequel
       )
     end
 
+    # See the definition of self.precompute. This method accepts the exact same
+    # arguments. When used within a trait block, this method is called rather
+    # than the class method.
+    def precompute(&block)
+      if !block
+        raise ArgumentError, 'Sequel::Packer.precompute must be passed a block'
+      end
+      @instance_precomputations << block
+    end
+
+    # See the definition of self.with_context. This method accepts the exact
+    # same arguments. When used within a trait block, this method is called
+    # rather than the class method.
+    def with_context(&block)
+      raise(
+        UnnecessaryWithContextError,
+        'There is no need to call with_context from within a trait block; ' +
+          '@context can be accessed directly.',
+      )
+    end
+
+    # Access the internal eager hash.
+    def eager_hash
+      @instance_eager_hash
+    end
+
+    # The following methods expose the class instance variables containing the
+    # core definition of the Packer.
+
+    def class_model
+      self.class.instance_variable_get(:@model)
+    end
+
     def class_fields
       self.class.instance_variable_get(:@class_fields)
     end
@@ -236,9 +442,18 @@ module Sequel
     def class_traits
       self.class.instance_variable_get(:@class_traits)
     end
+
+    def class_precomputations
+      self.class.instance_variable_get(:@class_precomputations)
+    end
+
+    def class_with_contexts
+      self.class.instance_variable_get(:@class_with_contexts)
+    end
   end
 end
 
 require 'sequel/packer/eager_hash'
+require 'sequel/packer/eager_loading'
 require 'sequel/packer/validation'
 require "sequel/packer/version"

data/lib/sequel/packer/eager_hash.rb

@@ -1,5 +1,3 @@
-require 'sequel'
-
 module Sequel
   class Packer
     module EagerHash

data/lib/sequel/packer/eager_loading.rb

@@ -0,0 +1,102 @@
+module Sequel
+  class Packer
+    module EagerLoading
+      # This methods allows eager loading associations _after_ a record, or
+      # multiple records, have been fetched from the database. It is useful when
+      # you know you will be accessing model associations, but models have
+      # already been materialized.
+      #
+      # This method accepts a normalized_eager_hash, as specified by
+      # Sequel::Packer::EagerHash.
+      #
+      # This method will handle procs used to filter association datasets, but
+      # if an association has already been loaded for every model, the dataset
+      # will not be refetched and the proc will not be applied.
+      #
+      # This method borrows a lot from the #eager_load Sequel::Dataset method
+      # defined in Sequels lib/sequel/model/associations.rb.
+      def self.eager_load(model_class, model_or_models, normalized_eager_hash)
+        models = model_or_models.is_a?(Array) ?
+          model_or_models :
+          [model_or_models]
+
+        # Cache to avoid building id maps multiple times.
+        key_hash = {}
+
+        normalized_eager_hash.each do |association, nested_associations|
+          eager_block = nil
+
+          if EagerHash.is_proc_hash?(nested_associations)
+            eager_block, nested_associations = nested_associations.entries[0]
+          end
+
+          reflection = model_class.association_reflections[association]
+
+          # If all of the models have already loaded the association, we'll just
+          # recursively call ::eager_load to load nested associations.
+          if models.all? {|m| m.associations.key?(association)}
+            if nested_associations
+              associated_records = if reflection.returns_array?
+                models.flat_map {|m| m.send(association)}.uniq
+              else
+                models.map {|m| m.send(association)}.compact
+              end
+
+              eager_load(
+                reflection.associated_class,
+                associated_records,
+                nested_associations,
+              )
+            end
+          else
+            key = reflection.eager_loader_key
+            id_map = nil
+
+            if key && !key_hash[key]
+              id_map = Hash.new {|h, k| h[k] = []}
+
+              models.each do |model|
+                case key
+                when Symbol
+                  model_id = model.get_column_value(key)
+                  id_map[model_id] << model if model_id
+                when Array
+                  model_id = key.map {|col| model.get_column_value(col)}
+                  id_map[model_id] << model if model_id.all?
+                else
+                  raise(
+                    Sequel::Error,
+                    "unhandled eager_loader_key #{key.inspect} for " +
+                      "association #{association}",
+                  )
+                end
+              end
+            end
+
+            loader = reflection[:eager_loader]
+
+            loader.call(
+              key_hash: key_hash,
+              rows: models,
+              associations: nested_associations,
+              self: self,
+              eager_block: eager_block,
+              id_map: id_map,
+            )
+
+            if reflection[:after_load]
+              models.each do |model|
+                model.send(
+                  :run_association_callbacks,
+                  reflection,
+                  :after_load,
+                  model.associations[association],
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sequel/packer/version.rb

@@ -1,5 +1,5 @@
 module Sequel
   class Packer
-    VERSION = "0.3.0"
+    VERSION = "1.0.1"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sequel-packer
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Paul Julius Martinez
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-14 00:00:00.000000000 Z
+date: 2021-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -128,6 +128,7 @@ files:
 - bin/setup
 - lib/sequel/packer.rb
 - lib/sequel/packer/eager_hash.rb
+- lib/sequel/packer/eager_loading.rb
 - lib/sequel/packer/validation.rb
 - lib/sequel/packer/version.rb
 - sequel-packer.gemspec