top_n_loader 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6eb6d3a03ffce49b32f78bb04809b3ce0b9c0cd114c3e32dae441c815873c8a5
-  data.tar.gz: 1b5426e4d320baed476489d56928480d46604028b798a975faa7d104533d7606
+  metadata.gz: d905751360ae053d80faa6332daad7e5fce710c2b44031483d71018e15d67c1a
+  data.tar.gz: dc175dd9e135880cb26c1c81c2784500e8ee0950dda9b03a39799af6d0b98710
 SHA512:
-  metadata.gz: 7adc3640113ca456040bfcdc08c1c35d9c9ed18d7ddc54d865785ee8a49a06531b2c0067ae84f90ecb5198c088683226892e6c03e3b38edbdcd48635d669dc7a
-  data.tar.gz: f6c99dac0cbe3cd32fc02393f085baf853d88f8680b848e99efccf42ce9eebfffd7a8605192a04e75a396e45bda642a53a0372aa78241101b12680d6d93e6a8a
+  metadata.gz: fb24707fdea8d9162a87b6960cce21ee38d5f9c5ab8cadee549a97518e0da9c1f1fea5e2c6ae50b0b1db0171d868ec5cb2c5f3c058b8e1cefba14491b058d2ec
+  data.tar.gz: a69198a3697ebc97e49c5c36d71103b9c8afb837b1416a7668e0190323042b3af51af3cd14bd1e22351646f271b825e385db53a80bb8090f3baf34e6e47626d4

data/.github/workflows/test.yml

@@ -5,15 +5,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.5', '2.6', '2.7' ]
+        ruby: [ '2.7', '3.0', '3.1' ]
         gemfiles:
-          - gemfiles/Gemfile-rails-5.2
           - gemfiles/Gemfile-rails-6.0
-        exclude:
-          - ruby: '2.6'
-            gemfiles: gemfiles/Gemfile-rails-5.2
-          - ruby: '2.7'
-            gemfiles: gemfiles/Gemfile-rails-5.2
+          - gemfiles/Gemfile-rails-7.0
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2

data/Gemfile.lock

@@ -1,61 +1,59 @@
 PATH
   remote: .
   specs:
-    top_n_loader (1.0.2)
+    top_n_loader (1.0.3)
       activerecord
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (7.0.1)
-      activesupport (= 7.0.1)
-    activerecord (7.0.1)
-      activemodel (= 7.0.1)
-      activesupport (= 7.0.1)
-    activesupport (7.0.1)
+    activemodel (7.0.3)
+      activesupport (= 7.0.3)
+    activerecord (7.0.3)
+      activemodel (= 7.0.3)
+      activesupport (= 7.0.3)
+    activesupport (7.0.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-    ast (2.4.0)
-    coderay (1.1.2)
-    concurrent-ruby (1.1.9)
-    docile (1.1.5)
-    i18n (1.8.11)
+    ast (2.4.2)
+    concurrent-ruby (1.1.10)
+    docile (1.4.0)
+    i18n (1.10.0)
       concurrent-ruby (~> 1.0)
-    jaro_winkler (1.5.4)
-    json (2.1.0)
-    method_source (0.9.0)
-    minitest (5.11.3)
-    parallel (1.19.1)
-    parser (2.7.1.2)
-      ast (~> 2.4.0)
-    pry (0.11.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    rainbow (3.0.0)
-    rake (10.5.0)
-    rexml (3.2.4)
-    rubocop (0.82.0)
-      jaro_winkler (~> 1.5.1)
+    minitest (5.16.1)
+    parallel (1.22.1)
+    parser (3.1.2.0)
+      ast (~> 2.4.1)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    regexp_parser (2.5.0)
+    rexml (3.2.5)
+    rubocop (1.31.0)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 3.1.0.0)
       rainbow (>= 2.2.2, < 4.0)
-      rexml
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.18.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-rubycw (0.1.4)
-      rubocop
-    ruby-progressbar (1.10.1)
-    simplecov (0.15.1)
-      docile (~> 1.1.0)
-      json (>= 1.8, < 3)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.2)
-    sqlite3 (1.4.2)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.18.0)
+      parser (>= 3.1.1.0)
+    rubocop-rubycw (0.1.6)
+      rubocop (~> 1.0)
+    ruby-progressbar (1.11.0)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    sqlite3 (1.4.4)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (1.7.0)
+    unicode-display_width (2.2.0)
 
 PLATFORMS
   ruby
@@ -63,7 +61,6 @@ PLATFORMS
 DEPENDENCIES
   bundler
   minitest
-  pry
   rake
   rubocop
   rubocop-rubycw

data/bin/console

@@ -2,7 +2,7 @@
 
 require 'bundler/setup'
 require 'top_n_loader'
-require 'pry'
 require_relative '../test/db'
 
-Pry.start
+DB.connect DB::DATABASE_CONFIG_SQLITE3
+binding.irb

data/gemfiles/{Gemfile-rails-5.2 → Gemfile-rails-7.0}

@@ -5,4 +5,4 @@ git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
 # Specify your gem's dependencies in top_n_loader.gemspec
 gemspec path: ".."
 
-gem "activerecord", "~> 5.2.0"
+gem "activerecord", "~> 7.0.0"

data/lib/top_n_loader/sql_builder.rb

@@ -9,71 +9,92 @@ module TopNLoader::SQLBuilder
   end
 
   def self.top_n_association_sql(klass, target_klass, relation, limit:, order_mode:, order_key:)
+    limit = limit.to_i
     parent_table = klass.table_name
     joins = klass.joins relation.to_sym
     target_table = target_klass.table_name
+    nullable = nullable_column? target_klass, order_key
     if target_table == klass.table_name
       target_table = "#{joins.joins_values.first.to_s.pluralize}_#{target_table}"
     end
     join_sql = joins.to_sql.match(/FROM.+/)[0]
-    %(
+    parent_primary_key = "#{qt parent_table}.#{q klass.primary_key}"
+    target_order_key = "#{qt target_table}.#{q order_key}"
+    target_primary_key = "#{qt target_table}.#{q target_klass.primary_key}"
+    top_n_key, top_n_alias, t_join_cond = limit == 1 ? [
+      target_primary_key, :top_n_primary_key, "#{target_primary_key} = top_n_primary_key"
+    ] : [
+      target_order_key, :top_n_order_key, compare_cond(target_order_key, order_mode, includes_nil: nullable)
+    ]
+    order_columns = limit == 1 ? [target_order_key, target_primary_key].uniq : [target_order_key]
+    order_cond = order_columns.map {|column| "#{column} #{order_mode.to_s.upcase}"}.join(', ')
+    <<~SQL.squish
       SELECT #{qt target_table}.*, top_n_group_key
       #{join_sql}
       INNER JOIN
       (
-        SELECT T.#{q klass.primary_key} as top_n_group_key,
+        SELECT T.#{q klass.primary_key} AS top_n_group_key,
         (
-          SELECT #{qt target_table}.#{q order_key}
+          SELECT #{top_n_key}
           #{join_sql}
-          WHERE #{qt parent_table}.#{q klass.primary_key} = T.#{q klass.primary_key}
-          ORDER BY #{qt target_table}.#{q order_key} #{order_mode.upcase}
-          LIMIT 1 OFFSET #{limit.to_i - 1}
-        ) AS last_value_of_key
-        FROM #{qt parent_table} as T where T.#{q klass.primary_key} in (?)
+          WHERE #{parent_primary_key} = T.#{q klass.primary_key}
+          ORDER BY #{order_cond}
+          LIMIT 1#{" OFFSET #{limit - 1}" if limit != 1}
+        ) AS #{top_n_alias}
+        FROM #{qt parent_table} AS T WHERE T.#{q klass.primary_key} IN (?)
       ) T
-      ON #{qt parent_table}.#{q klass.primary_key} = T.top_n_group_key
-      AND (
-        T.last_value_of_key IS NULL
-        OR #{qt target_table}.#{q order_key} #{{ asc: :<=, desc: :>= }[order_mode]} T.last_value_of_key
-        OR #{qt target_table}.#{q order_key} is NULL
-      )
-    )
+      ON #{parent_primary_key} = T.top_n_group_key AND #{t_join_cond}
+    SQL
   end
 
-
   def self.top_n_group_sql(klass:, group_column:, group_keys:, condition:, limit:, order_mode:, order_key:)
-    order_op = order_mode == :asc ? :<= : :>=
-    group_key_table = value_table(:T, :top_n_group_key, group_keys)
+    limit = limit.to_i
     table_name = klass.table_name
     sql = condition_sql klass, condition
-    join_cond = %(#{qt table_name}.#{q group_column} = T.top_n_group_key)
-    if group_keys.include? nil
-      nil_join_cond = %((#{qt table_name}.#{q group_column} IS NULL AND T.top_n_group_key IS NULL))
-      join_cond = %((#{join_cond} OR #{nil_join_cond}))
-    end
-    %(
+    group_key_nullable = group_keys.include?(nil) && nullable_column?(klass, group_column)
+    order_key_nullable = nullable_column? klass, order_key
+    group_key_table = value_table :T, :top_n_group_key, group_keys
+    table_order_key = "#{qt table_name}.#{q order_key}"
+    join_cond = equals_cond "#{qt table_name}.#{q group_column}", includes_nil: group_key_nullable
+    table_primary_key = "#{qt table_name}.#{q klass.primary_key}"
+    top_n_key, top_n_alias, t_join_cond = limit == 1 ? [
+      table_primary_key, :top_n_primary_key, "#{table_primary_key} = top_n_primary_key"
+    ] : [
+      table_order_key, :top_n_order_key,
+      "#{compare_cond table_order_key, order_mode, includes_nil: order_key_nullable}#{" WHERE #{sql}" if sql}"
+    ]
+    order_columns = limit == 1 ? [table_order_key, table_primary_key].uniq : [table_order_key]
+    order_cond = order_columns.map {|column| "#{column} #{order_mode.to_s.upcase}"}.join(', ')
+    <<~SQL.squish
       SELECT #{qt table_name}.*, top_n_group_key
       FROM #{qt table_name}
       INNER JOIN
       (
         SELECT top_n_group_key,
         (
-          SELECT #{qt table_name}.#{q order_key} FROM #{qt table_name}
-          WHERE #{join_cond}
-          #{"AND #{sql}" if sql}
-          ORDER BY #{qt table_name}.#{q order_key} #{order_mode.to_s.upcase}
-          LIMIT 1 OFFSET #{limit.to_i - 1}
-        ) AS last_value_of_key
+          SELECT #{top_n_key} FROM #{qt table_name}
+          WHERE #{join_cond}#{" AND #{sql}" if sql}
+          ORDER BY #{order_cond}
+          LIMIT 1#{" OFFSET #{limit - 1}" if limit != 1}
+        ) AS #{top_n_alias}
         FROM #{group_key_table}
       ) T
-      ON #{join_cond}
-      AND (
-        T.last_value_of_key IS NULL
-        OR #{qt table_name}.#{q order_key} #{order_op} T.last_value_of_key
-        OR #{qt table_name}.#{q order_key} is NULL
-      )
-      #{"WHERE #{sql}" if sql}
-    )
+      ON #{join_cond} AND #{t_join_cond}
+    SQL
+  end
+
+  def self.equals_cond(column, includes_nil:, t_column: 'T.top_n_group_key')
+    cond = "#{column} = #{t_column}"
+    includes_nil ? "(#{cond} OR (#{column} IS NULL AND #{t_column} IS NULL))" : cond
+  end
+
+  def self.compare_cond(column, order_mode, includes_nil:, t_column: 'T.top_n_order_key')
+    op = order_mode == :asc ? '<=' : '>='
+    if includes_nil && (nil_first? ? order_mode == :asc : order_mode == :desc)
+      "(#{column} #{op} #{t_column} OR #{column} IS NULL OR #{t_column} IS NULL)"
+    else
+      "(#{column} #{op} #{t_column} OR #{t_column} IS NULL)"
+    end
   end
 
   def self.q(name)
@@ -84,24 +105,56 @@ module TopNLoader::SQLBuilder
     ActiveRecord::Base.connection.quote_table_name name
   end
 
+  def self.nullable_column?(klass, column)
+    klass.column_for_attribute(column).null
+  end
+
+  def self.type_values(values)
+    return [nil, values] if sqlite?
+    groups = values.group_by { _1.is_a?(Time) || _1.is_a?(DateTime) ? 0 : _1.is_a?(Date) ? 1 : 2 }
+    type = groups[0] ? :TIMESTAMP : groups[1] ? :DATE : nil
+    [type, groups.sort.flat_map(&:last)]
+  end
+
   def self.value_table(table, column, values)
-    if ActiveRecord::Base.connection.adapter_name == 'PostgreSQL'
-      values_value_table(table, column, values)
+    type, values = type_values values
+    if postgres?
+      values_value_table table, column, values, type
     else
-      union_value_table(table, column, values)
+      union_value_table table, column, values, type
     end
   end
 
-  def self.union_value_table(table, column, values)
+  def self.values_table_batch_size
+    sqlite? ? 200 : 1000
+  end
+
+  def self.nil_first?
+    !postgres?
+  end
+
+  def self.adapter_name
+    ActiveRecord::Base.connection.adapter_name
+  end
+
+  def self.postgres?
+    adapter_name == 'PostgreSQL'
+  end
+
+  def self.sqlite?
+    adapter_name == 'SQLite'
+  end
+
+  def self.union_value_table(table, column, values, type)
     sanitize_sql_array [
-      "(SELECT ? AS #{column}#{' UNION SELECT ?' * (values.size - 1)}) AS #{table}",
+      "(SELECT #{"#{type} " if type}? AS #{column}#{' UNION SELECT ?' * (values.size - 1)}) AS #{table}",
       *values
     ]
   end
 
-  def self.values_value_table(table, column, values)
+  def self.values_value_table(table, column, values, type)
     sanitize_sql_array [
-      "(VALUES #{(['(?)'] * values.size).join(',')}) AS #{table} (#{column})",
+      "(VALUES (#{"#{type} " if type}?) #{', (?)' * (values.size - 1)}) AS #{table} (#{column})",
       *values
     ]
   end
@@ -122,12 +175,12 @@ module TopNLoader::SQLBuilder
     sql_binds = begin
       case value
       when NilClass
-        %(#{q key} IS NULL)
+        "#{q key} IS NULL"
       when Range
         if value.exclude_end?
-          [%(#{q key} >= ? AND #{q key} < ?), value.begin, value.end]
+          ["#{q key} >= ? AND #{q key} < ?", value.begin, value.end]
         else
-          [%(#{q key} BETWEEN ? AND ?), value.begin, value.end]
+          ["#{q key} BETWEEN ? AND ?", value.begin, value.end]
         end
       when Hash
         raise ArgumentError, '' unless value.keys == [:not]
@@ -135,12 +188,12 @@ module TopNLoader::SQLBuilder
       when Enumerable
         array = value.to_a
         if array.include? nil
-          [%((#{q key} IS NULL OR #{q key} IN (?))), array.reject(&:nil?)]
+          ["(#{q key} IS NULL OR #{q key} IN (?))", array.reject(&:nil?)]
         else
-          [%(#{q key} IN (?)), array]
+          ["#{q key} IN (?)", array]
         end
       else
-        [%(#{q key} = ?), value]
+        ["#{q key} = ?", value]
       end
     end
     sanitize_sql_array sql_binds

data/lib/top_n_loader/version.rb

@@ -1,3 +1,3 @@
 module TopNLoader
-  VERSION = '1.0.2'
+  VERSION = '1.0.3'
 end

data/lib/top_n_loader.rb

@@ -25,13 +25,17 @@ module TopNLoader
         limit: limit,
         **parse_order(klass, order)
       }
-      records = klass.find_by_sql(
-        SQLBuilder.top_n_group_sql(
-          group_keys: keys,
-          condition: condition,
-          **options
+      keys = keys.uniq
+      batch_size = keys.size.fdiv(keys.size.fdiv(SQLBuilder.values_table_batch_size).ceil).ceil
+      records = keys.each_slice(batch_size).flat_map do |batch_keys|
+        klass.find_by_sql(
+          SQLBuilder.top_n_group_sql(
+            group_keys: batch_keys,
+            condition: condition,
+            **options
+          )
         )
-      )
+      end
       format_result records, **options
     end
 
@@ -74,7 +78,7 @@ module TopNLoader
         existings, blanks = grouped_records.partition { |o| o[order_key] }
         existings.sort_by! { |o| [o[order_key], o[primary_key]] }
         blanks.sort_by! { |o| o[primary_key] }
-        ordered = blanks + existings
+        ordered = SQLBuilder.nil_first? ? blanks + existings : existings + blanks
         ordered.reverse! if order_mode == :desc
         ordered.take limit
       end

data/top_n_loader.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "activerecord"
 
-  %w[bundler rake minitest sqlite3 pry simplecov rubocop rubocop-rubycw].each do |gem_name|
+  %w[bundler rake minitest sqlite3 simplecov rubocop rubocop-rubycw].each do |gem_name|
     spec.add_development_dependency gem_name
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: top_n_loader
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - tompng
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-01-13 00:00:00.000000000 Z
+date: 2022-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -80,20 +80,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -146,7 +132,6 @@ files:
 - ".github/workflows/test.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".travis.yml"
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -154,8 +139,8 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
-- gemfiles/Gemfile-rails-5.2
 - gemfiles/Gemfile-rails-6.0
+- gemfiles/Gemfile-rails-7.0
 - lib/top_n_loader.rb
 - lib/top_n_loader/sql_builder.rb
 - lib/top_n_loader/version.rb

data/.travis.yml

@@ -1,5 +0,0 @@
-sudo: false
-language: ruby
-rvm:
-  - 2.5.0
-before_install: gem install bundler -v 1.16.1