top_n_loader 1.0.0 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cd29e09d74f26981a3772f7ac8871d7fe00bbaa736588f2de733acea20ad7bc5
-  data.tar.gz: 19a89bc3aaf851949e203fd0c29402f0c99fa6d3b3ce40227b4c7faa39c95370
+  metadata.gz: d905751360ae053d80faa6332daad7e5fce710c2b44031483d71018e15d67c1a
+  data.tar.gz: dc175dd9e135880cb26c1c81c2784500e8ee0950dda9b03a39799af6d0b98710
 SHA512:
-  metadata.gz: 42e80f413a55f32365bcc68425ae2d070f56a708387db2b91f867018604b1b20076e303ce07212369cb04cc86eb66632aa70fde0f932608199c6bfc31c363c8f
-  data.tar.gz: bbbbe954a980640ad53014604198fed10ff1047e09e1850ae18b7cb304e38240029ce9e583cdbb634923ac2f99819789cc9eec7a06680a1e32f2bcf861e42402
+  metadata.gz: fb24707fdea8d9162a87b6960cce21ee38d5f9c5ab8cadee549a97518e0da9c1f1fea5e2c6ae50b0b1db0171d868ec5cb2c5f3c058b8e1cefba14491b058d2ec
+  data.tar.gz: a69198a3697ebc97e49c5c36d71103b9c8afb837b1416a7668e0190323042b3af51af3cd14bd1e22351646f271b825e385db53a80bb8090f3baf34e6e47626d4

data/.github/workflows/test.yml

@@ -0,0 +1,22 @@
+name: Test
+on: [push, pull_request]
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.7', '3.0', '3.1' ]
+        gemfiles:
+          - gemfiles/Gemfile-rails-6.0
+          - gemfiles/Gemfile-rails-7.0
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - run: |
+        sudo apt-get update
+        sudo apt-get install -y libsqlite3-dev
+    - run: bundle install --gemfile ${{ matrix.gemfiles }} --jobs 4 --retry 3
+    - run: bundle exec --gemfile ${{ matrix.gemfiles }} rake

data/.rubocop.yml

@@ -0,0 +1,16 @@
+require:
+  - rubocop-rubycw
+
+AllCops:
+  TargetRubyVersion: 2.7
+  DisabledByDefault: true
+
+Rubycw/Rubycw:
+  Enabled: true
+  Exclude:
+    - 'test/**/*_test.rb'
+
+Lint/DuplicateMethods:
+  Enabled: true
+  Include:
+    - 'test/**/*_test.rb'

data/Gemfile.lock

@@ -1,45 +1,59 @@
 PATH
   remote: .
   specs:
-    top_n_loader (1.0.0)
+    top_n_loader (1.0.3)
       activerecord
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (5.2.3)
-      activesupport (= 5.2.3)
-    activerecord (5.2.3)
-      activemodel (= 5.2.3)
-      activesupport (= 5.2.3)
-      arel (>= 9.0)
-    activesupport (5.2.3)
+    activemodel (7.0.3)
+      activesupport (= 7.0.3)
+    activerecord (7.0.3)
+      activemodel (= 7.0.3)
+      activesupport (= 7.0.3)
+    activesupport (7.0.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-    arel (9.0.0)
-    coderay (1.1.2)
-    concurrent-ruby (1.1.5)
-    docile (1.1.5)
-    i18n (1.6.0)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    ast (2.4.2)
+    concurrent-ruby (1.1.10)
+    docile (1.4.0)
+    i18n (1.10.0)
       concurrent-ruby (~> 1.0)
-    json (2.1.0)
-    method_source (0.9.0)
-    minitest (5.11.3)
-    pry (0.11.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    rake (10.5.0)
-    simplecov (0.15.1)
-      docile (~> 1.1.0)
-      json (>= 1.8, < 3)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.2)
-    sqlite3 (1.3.13)
-    thread_safe (0.3.6)
-    tzinfo (1.2.5)
-      thread_safe (~> 0.1)
+    minitest (5.16.1)
+    parallel (1.22.1)
+    parser (3.1.2.0)
+      ast (~> 2.4.1)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    regexp_parser (2.5.0)
+    rexml (3.2.5)
+    rubocop (1.31.0)
+      parallel (~> 1.10)
+      parser (>= 3.1.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.18.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.18.0)
+      parser (>= 3.1.1.0)
+    rubocop-rubycw (0.1.6)
+      rubocop (~> 1.0)
+    ruby-progressbar (1.11.0)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    sqlite3 (1.4.4)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.2.0)
 
 PLATFORMS
   ruby
@@ -47,11 +61,12 @@ PLATFORMS
 DEPENDENCIES
   bundler
   minitest
-  pry
   rake
+  rubocop
+  rubocop-rubycw
   simplecov
   sqlite3
   top_n_loader!
 
 BUNDLED WITH
-   1.17.2
+   2.1.4

data/README.md

@@ -35,7 +35,7 @@ end
 
 ```ruby
 # Gemfile
-gem 'top_n_loader', github: 'tompng/top_n_loader'
+gem 'top_n_loader'
 ```
 
 ```ruby

data/Rakefile

@@ -7,4 +7,9 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList["test/**/*_test.rb"]
 end
 
-task :default => :test
+desc "Run rubocop"
+task :rubocop do
+  sh 'bundle exec rubocop'
+end
+
+task default: [:rubocop, :test]

data/bin/console

@@ -2,7 +2,7 @@
 
 require 'bundler/setup'
 require 'top_n_loader'
-require 'pry'
 require_relative '../test/db'
 
-Pry.start
+DB.connect DB::DATABASE_CONFIG_SQLITE3
+binding.irb

data/gemfiles/Gemfile-rails-6.0

@@ -0,0 +1,8 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in top_n_loader.gemspec
+gemspec path: ".."
+
+gem "activerecord", "~> 6.0.0"

data/gemfiles/Gemfile-rails-7.0

@@ -0,0 +1,8 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in top_n_loader.gemspec
+gemspec path: ".."
+
+gem "activerecord", "~> 7.0.0"

data/lib/top_n_loader/sql_builder.rb

@@ -9,71 +9,92 @@ module TopNLoader::SQLBuilder
   end
 
   def self.top_n_association_sql(klass, target_klass, relation, limit:, order_mode:, order_key:)
+    limit = limit.to_i
     parent_table = klass.table_name
     joins = klass.joins relation.to_sym
     target_table = target_klass.table_name
+    nullable = nullable_column? target_klass, order_key
     if target_table == klass.table_name
       target_table = "#{joins.joins_values.first.to_s.pluralize}_#{target_table}"
     end
     join_sql = joins.to_sql.match(/FROM.+/)[0]
-    %(
+    parent_primary_key = "#{qt parent_table}.#{q klass.primary_key}"
+    target_order_key = "#{qt target_table}.#{q order_key}"
+    target_primary_key = "#{qt target_table}.#{q target_klass.primary_key}"
+    top_n_key, top_n_alias, t_join_cond = limit == 1 ? [
+      target_primary_key, :top_n_primary_key, "#{target_primary_key} = top_n_primary_key"
+    ] : [
+      target_order_key, :top_n_order_key, compare_cond(target_order_key, order_mode, includes_nil: nullable)
+    ]
+    order_columns = limit == 1 ? [target_order_key, target_primary_key].uniq : [target_order_key]
+    order_cond = order_columns.map {|column| "#{column} #{order_mode.to_s.upcase}"}.join(', ')
+    <<~SQL.squish
       SELECT #{qt target_table}.*, top_n_group_key
       #{join_sql}
       INNER JOIN
       (
-        SELECT T.#{q klass.primary_key} as top_n_group_key,
+        SELECT T.#{q klass.primary_key} AS top_n_group_key,
         (
-          SELECT #{qt target_table}.#{q order_key}
+          SELECT #{top_n_key}
           #{join_sql}
-          WHERE #{qt parent_table}.#{q klass.primary_key} = T.#{q klass.primary_key}
-          ORDER BY #{qt target_table}.#{q order_key} #{order_mode.upcase}
-          LIMIT 1 OFFSET #{limit.to_i - 1}
-        ) AS last_value
-        FROM #{qt parent_table} as T where T.#{q klass.primary_key} in (?)
+          WHERE #{parent_primary_key} = T.#{q klass.primary_key}
+          ORDER BY #{order_cond}
+          LIMIT 1#{" OFFSET #{limit - 1}" if limit != 1}
+        ) AS #{top_n_alias}
+        FROM #{qt parent_table} AS T WHERE T.#{q klass.primary_key} IN (?)
       ) T
-      ON #{qt parent_table}.#{q klass.primary_key} = T.top_n_group_key
-      AND (
-        T.last_value IS NULL
-        OR #{qt target_table}.#{q order_key} #{{ asc: :<=, desc: :>= }[order_mode]} T.last_value
-        OR #{qt target_table}.#{q order_key} is NULL
-      )
-    )
+      ON #{parent_primary_key} = T.top_n_group_key AND #{t_join_cond}
+    SQL
   end
 
-
   def self.top_n_group_sql(klass:, group_column:, group_keys:, condition:, limit:, order_mode:, order_key:)
-    order_op = order_mode == :asc ? :<= : :>=
-    group_key_table = value_table(:T, :top_n_group_key, group_keys)
+    limit = limit.to_i
     table_name = klass.table_name
     sql = condition_sql klass, condition
-    join_cond = %(#{qt table_name}.#{q group_column} = T.top_n_group_key)
-    if group_keys.include? nil
-      nil_join_cond = %((#{qt table_name}.#{q group_column} IS NULL AND T.top_n_group_key IS NULL))
-      join_cond = %((#{join_cond} OR #{nil_join_cond}))
-    end
-    %(
+    group_key_nullable = group_keys.include?(nil) && nullable_column?(klass, group_column)
+    order_key_nullable = nullable_column? klass, order_key
+    group_key_table = value_table :T, :top_n_group_key, group_keys
+    table_order_key = "#{qt table_name}.#{q order_key}"
+    join_cond = equals_cond "#{qt table_name}.#{q group_column}", includes_nil: group_key_nullable
+    table_primary_key = "#{qt table_name}.#{q klass.primary_key}"
+    top_n_key, top_n_alias, t_join_cond = limit == 1 ? [
+      table_primary_key, :top_n_primary_key, "#{table_primary_key} = top_n_primary_key"
+    ] : [
+      table_order_key, :top_n_order_key,
+      "#{compare_cond table_order_key, order_mode, includes_nil: order_key_nullable}#{" WHERE #{sql}" if sql}"
+    ]
+    order_columns = limit == 1 ? [table_order_key, table_primary_key].uniq : [table_order_key]
+    order_cond = order_columns.map {|column| "#{column} #{order_mode.to_s.upcase}"}.join(', ')
+    <<~SQL.squish
       SELECT #{qt table_name}.*, top_n_group_key
       FROM #{qt table_name}
       INNER JOIN
       (
         SELECT top_n_group_key,
         (
-          SELECT #{qt table_name}.#{q order_key} FROM #{qt table_name}
-          WHERE #{join_cond}
-          #{"AND #{sql}" if sql}
-          ORDER BY #{qt table_name}.#{q order_key} #{order_mode.to_s.upcase}
-          LIMIT 1 OFFSET #{limit.to_i - 1}
-        ) AS last_value
+          SELECT #{top_n_key} FROM #{qt table_name}
+          WHERE #{join_cond}#{" AND #{sql}" if sql}
+          ORDER BY #{order_cond}
+          LIMIT 1#{" OFFSET #{limit - 1}" if limit != 1}
+        ) AS #{top_n_alias}
         FROM #{group_key_table}
       ) T
-      ON #{join_cond}
-      AND (
-        T.last_value IS NULL
-        OR #{qt table_name}.#{q order_key} #{order_op} T.last_value
-        OR #{qt table_name}.#{q order_key} is NULL
-      )
-      #{"WHERE #{sql}" if sql}
-    )
+      ON #{join_cond} AND #{t_join_cond}
+    SQL
+  end
+
+  def self.equals_cond(column, includes_nil:, t_column: 'T.top_n_group_key')
+    cond = "#{column} = #{t_column}"
+    includes_nil ? "(#{cond} OR (#{column} IS NULL AND #{t_column} IS NULL))" : cond
+  end
+
+  def self.compare_cond(column, order_mode, includes_nil:, t_column: 'T.top_n_order_key')
+    op = order_mode == :asc ? '<=' : '>='
+    if includes_nil && (nil_first? ? order_mode == :asc : order_mode == :desc)
+      "(#{column} #{op} #{t_column} OR #{column} IS NULL OR #{t_column} IS NULL)"
+    else
+      "(#{column} #{op} #{t_column} OR #{t_column} IS NULL)"
+    end
   end
 
   def self.q(name)
@@ -84,24 +105,56 @@ module TopNLoader::SQLBuilder
     ActiveRecord::Base.connection.quote_table_name name
   end
 
+  def self.nullable_column?(klass, column)
+    klass.column_for_attribute(column).null
+  end
+
+  def self.type_values(values)
+    return [nil, values] if sqlite?
+    groups = values.group_by { _1.is_a?(Time) || _1.is_a?(DateTime) ? 0 : _1.is_a?(Date) ? 1 : 2 }
+    type = groups[0] ? :TIMESTAMP : groups[1] ? :DATE : nil
+    [type, groups.sort.flat_map(&:last)]
+  end
+
   def self.value_table(table, column, values)
-    if ActiveRecord::Base.connection.adapter_name == 'PostgreSQL'
-      values_value_table(table, column, values)
+    type, values = type_values values
+    if postgres?
+      values_value_table table, column, values, type
     else
-      union_value_table(table, column, values)
+      union_value_table table, column, values, type
     end
   end
 
-  def self.union_value_table(table, column, values)
+  def self.values_table_batch_size
+    sqlite? ? 200 : 1000
+  end
+
+  def self.nil_first?
+    !postgres?
+  end
+
+  def self.adapter_name
+    ActiveRecord::Base.connection.adapter_name
+  end
+
+  def self.postgres?
+    adapter_name == 'PostgreSQL'
+  end
+
+  def self.sqlite?
+    adapter_name == 'SQLite'
+  end
+
+  def self.union_value_table(table, column, values, type)
     sanitize_sql_array [
-      "(SELECT ? AS #{column}#{' UNION SELECT ?' * (values.size - 1)}) AS #{table}",
+      "(SELECT #{"#{type} " if type}? AS #{column}#{' UNION SELECT ?' * (values.size - 1)}) AS #{table}",
       *values
     ]
   end
 
-  def self.values_value_table(table, column, values)
+  def self.values_value_table(table, column, values, type)
     sanitize_sql_array [
-      "(VALUES #{(['(?)'] * values.size).join(',')}) AS #{table} (#{column})",
+      "(VALUES (#{"#{type} " if type}?) #{', (?)' * (values.size - 1)}) AS #{table} (#{column})",
       *values
     ]
   end
@@ -122,12 +175,12 @@ module TopNLoader::SQLBuilder
     sql_binds = begin
       case value
       when NilClass
-        %(#{q key} IS NULL)
+        "#{q key} IS NULL"
       when Range
         if value.exclude_end?
-          [%(#{q key} >= ? AND #{q key} < ?), value.begin, value.end]
+          ["#{q key} >= ? AND #{q key} < ?", value.begin, value.end]
         else
-          [%(#{q key} BETWEEN ? AND ?), value.begin, value.end]
+          ["#{q key} BETWEEN ? AND ?", value.begin, value.end]
         end
       when Hash
         raise ArgumentError, '' unless value.keys == [:not]
@@ -135,12 +188,12 @@ module TopNLoader::SQLBuilder
       when Enumerable
         array = value.to_a
         if array.include? nil
-          [%((#{q key} IS NULL OR #{q key} IN (?))), array.reject(&:nil?)]
+          ["(#{q key} IS NULL OR #{q key} IN (?))", array.reject(&:nil?)]
         else
-          [%(#{q key} IN (?)), array]
+          ["#{q key} IN (?)", array]
         end
       else
-        [%(#{q key} = ?), value]
+        ["#{q key} = ?", value]
       end
     end
     sanitize_sql_array sql_binds

data/lib/top_n_loader/version.rb

@@ -1,3 +1,3 @@
 module TopNLoader
-  VERSION = '1.0.0'
+  VERSION = '1.0.3'
 end

data/lib/top_n_loader.rb

@@ -10,7 +10,7 @@ module TopNLoader
       return Hash.new { [] } if ids.empty? || limit.zero?
       klass = base_klass.reflect_on_association(relation.to_sym).klass
       order_option = { limit: limit, **parse_order(klass, order) }
-      sql = SQLBuilder.top_n_association_sql base_klass, klass, relation, order_option
+      sql = SQLBuilder.top_n_association_sql base_klass, klass, relation, **order_option
       records = klass.find_by_sql([sql, ids])
       format_result(records, klass: klass, **order_option)
     end
@@ -25,14 +25,18 @@ module TopNLoader
         limit: limit,
         **parse_order(klass, order)
       }
-      records = klass.find_by_sql(
-        SQLBuilder.top_n_group_sql(
-          group_keys: keys,
-          condition: condition,
-          **options
+      keys = keys.uniq
+      batch_size = keys.size.fdiv(keys.size.fdiv(SQLBuilder.values_table_batch_size).ceil).ceil
+      records = keys.each_slice(batch_size).flat_map do |batch_keys|
+        klass.find_by_sql(
+          SQLBuilder.top_n_group_sql(
+            group_keys: batch_keys,
+            condition: condition,
+            **options
+          )
         )
-      )
-      format_result records, options
+      end
+      format_result records, **options
     end
 
     private
@@ -74,7 +78,7 @@ module TopNLoader
         existings, blanks = grouped_records.partition { |o| o[order_key] }
         existings.sort_by! { |o| [o[order_key], o[primary_key]] }
         blanks.sort_by! { |o| o[primary_key] }
-        ordered = blanks + existings
+        ordered = SQLBuilder.nil_first? ? blanks + existings : existings + blanks
         ordered.reverse! if order_mode == :desc
         ordered.take limit
       end

data/top_n_loader.gemspec

@@ -10,6 +10,7 @@ Gem::Specification.new do |spec|
 
   spec.summary       = %q{load top n records for each group}
   spec.description   = %q{load top n records for each group}
+  spec.homepage      = "https://github.com/tompng/#{spec.name}"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
@@ -21,7 +22,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "activerecord"
 
-  %w[bundler rake minitest sqlite3 pry simplecov].each do |gem_name|
+  %w[bundler rake minitest sqlite3 simplecov rubocop rubocop-rubycw].each do |gem_name|
     spec.add_development_dependency gem_name
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: top_n_loader
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.3
 platform: ruby
 authors:
 - tompng
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-05-11 00:00:00.000000000 Z
+date: 2022-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -81,7 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -95,7 +95,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: simplecov
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rubycw
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -115,8 +129,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
-- ".travis.yml"
+- ".rubocop.yml"
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -124,15 +139,17 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- gemfiles/Gemfile-rails-6.0
+- gemfiles/Gemfile-rails-7.0
 - lib/top_n_loader.rb
 - lib/top_n_loader/sql_builder.rb
 - lib/top_n_loader/version.rb
 - top_n_loader.gemspec
-homepage: 
+homepage: https://github.com/tompng/top_n_loader
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -147,8 +164,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
-signing_key: 
+rubygems_version: 3.3.3
+signing_key:
 specification_version: 4
 summary: load top n records for each group
 test_files: []

data/.travis.yml

@@ -1,5 +0,0 @@
-sudo: false
-language: ruby
-rvm:
-  - 2.5.0
-before_install: gem install bundler -v 1.16.1