activerecord-summarize 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0f4063a016e57d85371ba91aa751c223c5830f29bf46a7f693c00af2b808fb48
-  data.tar.gz: c63ee2b1ed0e2c7e39f71125f759a4f933cd56281c414b0e694f8f46511b18f4
+  metadata.gz: fbba2c0577555f891c1f860b76cc9f0194504d47d59922c8337afb9a1b3ab1a0
+  data.tar.gz: 3c1945b1652a4305ec4df2d13a0c5a6966e308018d2b34c5c21e396f02a1e31c
 SHA512:
-  metadata.gz: 3252c9e8bc8eb0e5ca6eef4b3a089d68f5f67f17d42824f5478b02181f7a618a9feee961c9636f1b696acc6661975580ab75aef8c9edcc6a08b480eae11ae188
-  data.tar.gz: 062a2d2557969c0ae4fefd6e656dcd5bb8b4981e0b71899a726025fd3c6ef081e085c8c87729c85555cdd3999f58f8daf323f1a2f6b00f59f71b5de6f8a3a78c
+  metadata.gz: a2a9e27420e30861c5c713e757ae66ead1af9d5aa9e10b259620b10d9289652464fc561745e720bae100cbda1b265e9f551cfd1f05c78f53c8dae39bfd788c33
+  data.tar.gz: 7e8632ae8ced794bdfb539624a64ebeef8d4a0a6568e3f625cca13b0d669365dd309981d351649c2af3a33f21c45e0fed2848853d581681dd0656d55b8163005

data/Gemfile

@@ -6,7 +6,8 @@ source "https://rubygems.org"
 gemspec
 
 gem "rake", "~> 13.0"
-
 gem "minitest", "~> 5.0"
-
 gem "standard", "~> 1.3"
+
+gem "activerecord", "7.0.3"
+gem "sqlite3", "1.4.2"

data/Gemfile.lock

@@ -1,24 +1,24 @@
 PATH
   remote: .
   specs:
-    activerecord-summarize (0.2.1)
+    activerecord-summarize (0.3.0)
       activerecord (>= 5.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (7.0.2.2)
-      activesupport (= 7.0.2.2)
-    activerecord (7.0.2.2)
-      activemodel (= 7.0.2.2)
-      activesupport (= 7.0.2.2)
-    activesupport (7.0.2.2)
+    activemodel (7.0.3)
+      activesupport (= 7.0.3)
+    activerecord (7.0.3)
+      activemodel (= 7.0.3)
+      activesupport (= 7.0.3)
+    activesupport (7.0.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
     ast (2.4.2)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.1.10)
     i18n (1.10.0)
       concurrent-ruby (~> 1.0)
     minitest (5.15.0)
@@ -44,6 +44,7 @@ GEM
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
     ruby-progressbar (1.11.0)
+    sqlite3 (1.4.2)
     standard (1.7.2)
       rubocop (= 1.25.1)
       rubocop-performance (= 1.13.2)
@@ -56,9 +57,11 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  activerecord (= 7.0.3)
   activerecord-summarize!
   minitest (~> 5.0)
   rake (~> 13.0)
+  sqlite3 (= 1.4.2)
   standard (~> 1.3)
 
 BUNDLED WITH

data/README.md

@@ -6,6 +6,8 @@
 
 2. For more complex reporting requirements, including nested `.group` calls, use `summarize` for fast, legible code that you just couldn't have written before without unacceptable performance or lengthy custom SQL and data-wrangling.
 
+Sidebar: Are you wondering [how `summarize` compares to `load_async`](./docs/summarize_compared_with_load_async.md)?
+
 ## Installation
 
 Add this line to your Rails application's Gemfile:
@@ -138,7 +140,11 @@ end
 # }
 ```
 
-The ActiveRecord API has no direct analog for this, so `noop: true` is not allowed when `summarize` is called on a grouped relation.
+See [Use case: moderator dashboard](./docs/use_case_moderator_dashboard.md) for a more-complete example comparing ActiveRecord-only code with `summarize`.
+
+### Caveat
+
+The ActiveRecord API has no direct analog for this mode, so `noop: true` is not allowed when `summarize` is called on a grouped relation.
 
 When the relation already has `group` applied, for correct results, `summarize` requires that the block mutate no state and return all values you care about: functional purity, no side effects. `ChainableResult` values referenced by instance variables or local variables not returned from the block won't be evaluated. I.e., `pure: true` is implied and `pure: false` is not allowed. To see why:
 

data/docs/summarize_compared_with_load_async.md

@@ -0,0 +1,11 @@
+# How does `summarize` compare to `load_async`?
+
+`load_async` is cool, and it serves an almost completely different use case—you can't even use it with calculations out of the box.
+
+⭐️ `load_async` is for "I know I'm going to need these `Post` records, but I probably won't actually do anything with them till render, so start loading them in the background now while I do some other work."
+
+If there's only one collection to load for a given controller action, the benefits will be very modest. But if you have (e.g.) 2 collections to load, `load_async` lets you hide the load time of the faster one inside the slower one, since the queries will run simultaneously—at the cost of using an additional database connection. The most straightforward wins for `load_async`, IMO, are when you have one slow-ish load and several quick queries or you have a slow-ish load *and* you will have to wait on some other 3rd-party API. (In both cases, start the slow load first with `load_async`.)
+
+⭐️ `summarize` is for "Over the last 30 days, for each subreddit that I'm a moderator of, I need to count how many `Post` were created, and I also need to count how many of them ended up with negative karma, and I also need to see, grouped by date, what percentage of posts ended up with `karma > :karma_threshold`, and I also want to know the average number of comments per post, for all posts with karma >= 0, grouped by day of the week."
+
+`summarize` can get all that for you in a single query and return the data in a useful shape. See [Use case: moderator dashboard](./use_case_moderator_dashboard.md) for how it might be done.

data/docs/use_case_moderator_dashboard.md

@@ -0,0 +1,97 @@
+
+# Using `summarize` for a moderator dashboard at reddit (in my imagination)
+
+I'm an only-occasional reddit user and not a moderator at all, but let's imagine we're building a dashboard for moderators with activity and engagement stats for each subreddit they moderate. Let's suppose a straightforward Rails-y schema and a Postgres database. (I have no inside knowledge about how reddit actually works, and maybe at reddit's scale you'd need an entirely different approach.)
+
+## Requirements
+
+For each subreddit that a user moderates, the user should see these stats with respect to the last 30 days:
+
+- count of how many posts were created
+- count of how many posts from this period were buried, i.e., ended up with negative karma
+- grouped by post creation date, the percentage of posts that ended up being popular, where popular means having a karma score greater than a per-subreddit-configured threshold
+- grouped by post creation day of the week, the average number of comments per non-buried post
+  
+  > *Below, grouping by day of the week is handled with `.group("EXTRACT(DOW FROM posts.created_at)")`*
+
+## Background
+
+Before we get into the dashboard, someone must have written something for selecting popular posts already, right? Here it is!
+
+```ruby
+class Post < ApplicationModel
+  # Grab our subreddit's popularity_threshold directly: no need to join through subreddits
+  has_one :popularity_threshold_setting, -> { where(key: "popularity_threshold") },
+    class_name: 'Setting', foreign_key: :subreddit_id, primary_key: :subreddit_id
+  scope :popular, -> { left_joins(:popularity_threshold_setting)
+    .where("posts.karma >= coalesce(settings.value,?)", DEFAULT_POPULAR_THRESHOLD) }
+end
+```
+
+## Without `summarize`
+
+You might start with something like this:
+
+```ruby
+def dashboard
+  @subreddits = current_user.moderated_subreddits
+  @subreddit_stats = subreddits.each_with_object({}) do |subreddit, all_stats|
+    stats = all_stats[subreddit.id] = {}
+    posts = subreddit.posts.where(created_at: 30.days.ago..).order(:created_at)
+    stats[:posts_created] = posts.count
+    stats[:buried_posts] = posts.where(karma: ...0).count
+    daily_posts = posts.group("posts.created_at::date")
+    daily_popular = daily_posts.popular.count
+    daily_total = daily_posts.count
+    stats[:daily_popular_rate] = daily_total.map {|k,v| [k,(daily_popular[k]||0).to_f / v] }.to_h
+    dow_not_buried = posts.where(karma: 0..).group("EXTRACT(DOW FROM posts.created_at)")
+    dow_posts = dow_not_buried.count
+    dow_comments = dow_not_buried.sum(:comments_count)
+    stats[:dow_avg_comments] = dow_posts.map {|k,v| [k,(dow_comments[k]||0).to_f / v] }.to_h
+  end
+end
+```
+
+This code is straightforward and easy to read and reason about, but for a user who moderates 3 subreddits, just this part of the dashboard is going to involve 18 database queries. And anything else we want to add to the subreddit stats will be another 1-2 queries per subreddit. So if you're building this dashboard, as requirements evolve over time, it's going to get slower and slower, and eventually you're going to push back on requirements and/or rewrite the whole action as a wall of hand-crafted SQL and another wall of ruby code to get the data back into the right shape.
+
+## With `summarize`
+
+Or you could do it **with `summarize`** and get identical results in a single query.
+
+This is the more-advanced `.group(*cols).summarize` mode that has no direct ActiveRecord equivalent: just as above, `@subreddit_stats` will  be a hash with `subreddit_id` keys, and each value will be a hash with a couple of simple count values and a couple grouped calculations. But to do this with `ActiveRecord` alone, we had to iterate a list, run queries, and build the `@subreddit_stats` hash ourself.
+
+I've also given one requirement that implies a join, so you can see how that works just a touch differently with `summarize`.
+
+```ruby
+def dashboard
+  @subreddits = current_user.moderated_subreddits
+  # Join :popularity_threshold_setting before .summarize to use it within the summarize block.
+  # If you forget, `daily_posts.popular.count` will raise `Unsummarizable` with a helpful message.
+  all_posts = Post.where(subreddit: @subreddits.select(:id)).where(created_at: 30.days.ago..)
+                  .left_joins(:popularity_threshold_setting).order(:created_at)
+  @subreddit_stats = all_posts.group(:subreddit_id).summarize do |posts, with|
+    daily_posts = posts.group("posts.created_at::date")
+    dow_not_burried = posts.where(karma: 0..).group("EXTRACT(DOW FROM posts.created_at)")
+    {
+      posts_created: posts.count,
+      buried_posts: posts.where(karma: ...0).count,
+      daily_popular_rate: with[
+          daily_posts.popular.count,
+          daily_posts.count
+        ] do |popular, total|
+          total.map { |date, count| [date, (popular[date]||0).to_f / count] }.to_h
+        end,
+      dow_avg_comments: with[
+          dow_not_buried.sum(:comments_count),
+          dow_not_buried.count
+        ] do |comments, posts|
+          posts.map { |dow, count| [dow, (comments[dow]||0).to_f / count] }.to_h
+        end
+    }
+  end
+end
+```
+
+Since `summarize` runs a single query that visits each relevant `posts` row just once, adding additional calculations is pretty close to free.
+
+Even with the mental overhead of needing to join outside the block and use `with` to combine calculations (see [README](../README.md) for details), I think this is still easy to read, write, and reason about, and it beats the heck out of walls of SQL. What do you think?

data/lib/activerecord/summarize/version.rb

@@ -2,6 +2,6 @@
 
 module ActiveRecord
   module Summarize
-    VERSION = "0.2.1"
+    VERSION = "0.3.0"
   end
 end

data/lib/activerecord/summarize.rb

@@ -127,28 +127,36 @@ module ActiveRecord::Summarize
       # grouped_query = groups.any? ? from_where.group(*groups) : from_where
       grouped_query = groups.any? ? from_where.group(*1..groups.size) : from_where
       data = grouped_query.pluck(*groups, *value_selects)
+      # .pluck(:one_column) returns an array of values instead of an array of arrays,
+      # which breaks the aggregation and assignment below in case anyone ever asks
+      # `summarize` for only one thing.
+      data = data.map { |d| [d] } if (groups.size + value_selects.size) == 1
 
       # Aggregate & assign results
       group_idx = groups.each_with_index.to_h
       starting_values, reducers = @calculations.each_with_index.map do |f, i|
         value_column = groups.size + i
         group_columns = f.relation.group_values.map { |k| group_idx[k] }
+        # `row[value_column] || 0` pattern in reducers because SQL SUM(NULL)
+        # returns NULL, but like ActiveRecord we always want .sum to return a
+        # number, and our "starting_values and reducers" implementation means
+        # we sometimes will have to add NULL to our numbers.
         case group_columns.size
         when 0 then [
           0,
-          ->(memo, row) { memo + row[value_column] }
+          ->(memo, row) { memo + (row[value_column] || 0) }
         ]
         when 1 then [
           Hash.new(0), # Default 0 makes the reducer much cleaner, but we have to clean it up later
           ->(memo, row) {
-            memo[row[group_columns[0]]] += row[value_column] unless row[value_column].zero?
+            memo[row[group_columns[0]]] += row[value_column] unless (row[value_column] || 0).zero?
             memo
           }
         ]
         else [
           Hash.new(0),
           ->(memo, row) {
-            memo[group_columns.map { |i| row[i] }] += row[value_column] unless row[value_column].zero?
+            memo[group_columns.map { |i| row[i] }] += row[value_column] unless (row[value_column] || 0).zero?
             memo
           }
         ]
@@ -233,14 +241,14 @@ module ActiveRecord::Summarize
     def select_value(base_relation)
       where = relation.where_clause - base_relation.where_clause
       for_select = column
-      for_select = Arel::Nodes::Case.new(where.ast, unmatch_value).when(true, for_select) unless where.empty?
+      for_select = Arel::Nodes::Case.new(where.ast).when(true, for_select).else(unmatch_arel_node) unless where.empty?
       function.new([for_select]).tap { |f| f.distinct = relation.distinct_value }
     end
 
-    def unmatch_value
+    def unmatch_arel_node
       case method
-      when "sum" then 0
-      when "count" then nil
+      when "sum" then 0 # Adding zero to a sum does nothing
+      when "count" then nil # In SQL, null is no value and is not counted
       else raise "Unknown calculation method"
       end
     end
@@ -268,6 +276,7 @@ module ActiveRecord::Summarize
       case operation = operation.to_s.downcase
       when "count", "sum"
         column_name = :id if [nil, "*", :all].include? column_name
+        raise Unsummarizable, "DISTINCT in SQL is not reliably correct with summarize" if column_name.is_a?(String) && /\bdistinct\b/i === column_name
         @summarize.add_calculation(self, operation, aggregate_column(column_name))
       else super
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-summarize
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Joshua Paine
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-03-17 00:00:00.000000000 Z
+date: 2022-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -56,6 +56,8 @@ files:
 - activerecord-summarize.gemspec
 - bin/console
 - bin/setup
+- docs/summarize_compared_with_load_async.md
+- docs/use_case_moderator_dashboard.md
 - lib/activerecord/summarize.rb
 - lib/activerecord/summarize/version.rb
 - lib/chainable_result.rb