activerecord_cursor_paginate 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea74d80632e02c285a03b087219a635b30b678225dfbe7c17b949a83d74d2444
-  data.tar.gz: 06b461f0429a9527dd0180fc94ff5d984649aa412e78c641321cb463c63a40e3
+  metadata.gz: 198c2d0c2e71f89e73fca64d7ab8711949cf25faa20af6297d1d6045587ec7cf
+  data.tar.gz: 6f60b98579b2dcd132c09d4f75105b5cab7219ae8841c6970df9e77df6b59722
 SHA512:
-  metadata.gz: a671a9f299684a25cc158f38b0c7e23eb1eb03c36141bea92f435a0857dd6b35db371f4cfcf9e6a0ddb2ac10da0a88b7efb101e44a7da319a143e6da3320e3ed
-  data.tar.gz: 0f2e04a9dbb66a10253ea6f6309aacd1634a4a20cbde2f212ed1b3bf4d270931664cb6ad25e9d84c266b6bf9b815abda732a4fc4bacbe3d2f92cbb25dc7c1b40
+  metadata.gz: 85fd85af08637f1c22e52430f51bae57c5b116af2f843703dd2ab528d36616bf323d686c53c7b059fb04aa176a01bc0ec2a3c5a2b40acb6f54cf5412e2770e10
+  data.tar.gz: be0771bacd8fbd5f110c9f90927a12792ef85d8f4e4278e46321e96d82f93f87b848ad71b86cd9150c9465c39e5d8dd3a4e2a71672e51d7d7c04f00cbb561990

data/CHANGELOG.md

@@ -1,5 +1,30 @@
 ## master (unreleased)
 
+## 0.3.0 (2025-01-06)
+
+- Allow paginating over nullable columns
+
+    Previously, the error was raised when cursor values contained `nil`s. Now, it is possible to paginate
+    over columns containing `nil` values. You need to explicitly configure which columns are nullable,
+    otherwise columns are considered as non-nullable by the gem.
+
+    ```ruby
+    paginator = users.cursor_paginate(order: [:name, :id], nullable_columns: [:name])
+    ```
+
+    Note that it is not recommended to use this feature, because the complexity of produced SQL queries can have
+    a very negative impact on the database performance. It is better to paginate using only non-nullable columns.
+
+- Fix paginating over relations with joins, includes and custom ordering
+- Add ability to incrementally configure a paginator
+
+- Add ability to get the total number of records
+
+    ```ruby
+    paginator = posts.cursor_paginate
+    paginator.total_count # => 145
+    ```
+
 ## 0.2.0 (2024-05-23)
 
 - Fix prefixing selected columns when iterating over joined tables

data/README.md

@@ -8,7 +8,7 @@ Where a regular limit / offset pagination would jump in results if a record on a
 
 To learn more about cursor pagination, check out the _"How does it work?"_ section below.
 
-[![Build Status](https://github.com/fatkodima/activerecord_cursor_paginate/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/fatkodima/activerecord_cursor_paginate/actions/workflows/test.yml)
+[![Build Status](https://github.com/healthie/activerecord_cursor_paginate/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/healthie/activerecord_cursor_paginate/actions/workflows/test.yml)
 
 ## Requirements
 
@@ -44,9 +44,16 @@ And then we create our paginator to fetch the first response page:
 
 ```ruby
 paginator = posts.cursor_paginate
+
+# Total number of records to iterate by this paginator
+paginator.total_count # => 145
+
 page = paginator.fetch
 page.records # => [#<Post:0x00007fd7071b2ea8 @id=1>, #<Post:0x00007fd7071bb738 @id=2>, ..., #<Post:0x00007fd707238260 @id=10>]
+
+# Number of records in this page
 page.count # => 10
+
 page.empty? # => false
 page.cursors            # => ["MQ", "Mg", ..., "MTA"]
                                |                 |
@@ -375,7 +382,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/fatkodima/activerecord_cursor_paginate.
+Bug reports and pull requests are welcome on GitHub at https://github.com/healthie/activerecord_cursor_paginate.
 
 ## License
 

data/lib/activerecord_cursor_paginate/cursor.rb

@@ -7,12 +7,12 @@ module ActiveRecordCursorPaginate
   # @private
   class Cursor
     class << self
-      def from_record(record, columns:)
+      def from_record(record, columns:, nullable_columns: nil)
         values = columns.map { |column| record[column] }
-        new(columns: columns, values: values)
+        new(columns: columns, values: values, nullable_columns: nullable_columns)
       end
 
-      def decode(cursor_string:, columns:)
+      def decode(cursor_string:, columns:, nullable_columns: nil)
         decoded = JSON.parse(Base64.urlsafe_decode64(cursor_string))
 
         if (columns.size == 1 && decoded.is_a?(Array)) ||
@@ -28,7 +28,7 @@ module ActiveRecordCursorPaginate
             deserialize_time_if_needed(decoded)
           end
 
-        new(columns: columns, values: decoded)
+        new(columns: columns, values: decoded, nullable_columns: nullable_columns)
       rescue ArgumentError, JSON::ParserError # ArgumentError is raised by strict_decode64
         raise InvalidCursorError, "The given cursor `#{cursor_string}` could not be decoded"
       end
@@ -46,11 +46,16 @@ module ActiveRecordCursorPaginate
 
     attr_reader :columns, :values
 
-    def initialize(columns:, values:)
+    def initialize(columns:, values:, nullable_columns: nil)
       @columns = Array.wrap(columns)
       @values = Array.wrap(values)
+      @nullable_columns = Array.wrap(nullable_columns)
+
+      nil_index = @values.index(nil)
+      if nil_index && !@nullable_columns.include?(@columns[nil_index])
+        raise ArgumentError, "Cursor value is nil for a column that is not in the :nullable_columns list"
+      end
 
-      raise ArgumentError, "Cursor values can not be nil" if @values.any?(nil)
       raise ArgumentError, ":columns and :values have different sizes" if @columns.size != @values.size
     end
 

data/lib/activerecord_cursor_paginate/extension.rb

@@ -10,9 +10,8 @@ module ActiveRecordCursorPaginate
     #   paginator = Post.cursor_paginate(limit: 2, after: "Mg")
     #   page = paginator.fetch
     #
-    def cursor_paginate(after: nil, before: nil, limit: nil, order: nil, append_primary_key: true)
-      relation = (is_a?(ActiveRecord::Relation) ? self : all)
-      Paginator.new(relation, after: after, before: before, limit: limit, order: order, append_primary_key: append_primary_key)
+    def cursor_paginate(**options)
+      Paginator.new(all, **options)
     end
     alias cursor_pagination cursor_paginate
   end

data/lib/activerecord_cursor_paginate/page.rb

@@ -6,15 +6,16 @@ module ActiveRecordCursorPaginate
   #
   class Page
     # Records this page contains.
-    # @return [ActiveRecord::Base]
+    # @return [Array<ActiveRecord::Base>]
     #
     attr_reader :records
 
-    def initialize(records, order_columns:, has_previous: false, has_next: false)
+    def initialize(records, order_columns:, has_previous: false, has_next: false, nullable_columns: nil)
       @records = records
       @order_columns = order_columns
       @has_previous = has_previous
       @has_next = has_next
+      @nullable_columns = nullable_columns
     end
 
     # Number of records in this page.
@@ -79,7 +80,7 @@ module ActiveRecordCursorPaginate
     private
       def cursor_for_record(record)
         if record
-          cursor = Cursor.from_record(record, columns: @order_columns)
+          cursor = Cursor.from_record(record, columns: @order_columns, nullable_columns: @nullable_columns)
           cursor.encode
         end
       end

data/lib/activerecord_cursor_paginate/paginator.rb

@@ -19,6 +19,8 @@ module ActiveRecordCursorPaginate
   #     end
   #
   class Paginator
+    attr_reader :relation, :before, :after, :limit, :order, :append_primary_key
+
     # Create a new instance of the `ActiveRecordCursorPaginate::Paginator`
     #
     # @param relation [ActiveRecord::Relation] Relation that will be paginated.
@@ -38,30 +40,87 @@ module ActiveRecordCursorPaginate
     #   should be implicitly appended to the list of sorting columns. It may be useful
     #   to disable it for the table with a UUID primary key or when the sorting is done by a
     #   combination of columns that are already unique.
+    # @param nullable_columns [Symbol, String, nil, Array] Columns which are nullable.
+    #   By default, all columns are considered as non-nullable, if not in this list.
+    #   It is not recommended to use this feature, because the complexity of produced SQL
+    #   queries can have a very negative impact on the database performance. It is better
+    #   to paginate using only non-nullable columns.
+    #
     # @raise [ArgumentError] If any parameter is not valid
     #
-    def initialize(relation, before: nil, after: nil, limit: nil, order: nil, append_primary_key: true)
+    def initialize(relation, before: nil, after: nil, limit: nil, order: nil, append_primary_key: true, nullable_columns: nil)
       unless relation.is_a?(ActiveRecord::Relation)
         raise ArgumentError, "relation is not an ActiveRecord::Relation"
       end
 
-      if before.present? && after.present?
+      @relation = relation
+      @primary_key = @relation.primary_key
+      @append_primary_key = append_primary_key
+
+      @cursor = @current_cursor = nil
+      @is_forward_pagination = true
+      @before = @after = nil
+      @page_size = nil
+      @limit = nil
+      @columns = []
+      @directions = []
+      @order = nil
+
+      self.before = before
+      self.after = after
+      self.limit = limit
+      self.order = order
+      self.nullable_columns = nullable_columns
+    end
+
+    def before=(value)
+      if value.present? && after.present?
+        raise ArgumentError, "Only one of :before and :after can be provided"
+      end
+
+      @cursor = value || after
+      @current_cursor = @cursor
+      @is_forward_pagination = value.blank?
+      @before = value
+    end
+
+    def after=(value)
+      if before.present? && value.present?
         raise ArgumentError, "Only one of :before and :after can be provided"
       end
 
-      @relation = relation
-      @primary_key = @relation.primary_key
-      @cursor = before || after
-      @is_forward_pagination = before.blank?
+      @cursor = before || value
+      @current_cursor = @cursor
+      @after = value
+    end
 
+    def limit=(value)
       config = ActiveRecordCursorPaginate.config
-      @page_size = limit || config.default_page_size
+      @page_size = value || config.default_page_size
       @page_size = [@page_size, config.max_page_size].min if config.max_page_size
+      @limit = value
+    end
 
-      @append_primary_key = append_primary_key
-      order = normalize_order(order)
+    def order=(value)
+      order = normalize_order(value)
       @columns = order.keys
       @directions = order.values
+      @order = value
+    end
+
+    def nullable_columns=(value)
+      value = Array.wrap(value)
+      value = value.map { |column| column.is_a?(Symbol) ? column.to_s : column }
+
+      if (value - @columns).any?
+        raise ArgumentError, ":nullable_columns should include only column names from the :order option"
+      end
+
+      if value.include?(@columns.last)
+        raise ArgumentError, "Last order column can not be nullable"
+      end
+
+      @nullable_columns = value
     end
 
     # Get the paginated result.
@@ -70,32 +129,7 @@ module ActiveRecordCursorPaginate
     # @note Calling this method advances the paginator.
     #
     def fetch
-      relation = @relation
-
-      # Non trivial columns (expressions or joined tables columns).
-      if @columns.any?(/\W/)
-        arel_columns = @columns.map.with_index do |column, i|
-          arel_column(column).as("cursor_column_#{i + 1}")
-        end
-        cursor_column_names = arel_columns.map { |column| column.right.to_s }
-
-        relation =
-          if relation.select_values.empty?
-            relation.select(relation.arel_table[Arel.star], arel_columns)
-          else
-            relation.select(arel_columns)
-          end
-      else
-        cursor_column_names = @columns
-      end
-
-      pagination_directions = @directions.map { |direction| pagination_direction(direction) }
-      relation = relation.reorder(cursor_column_names.zip(pagination_directions).to_h)
-
-      if @cursor
-        decoded_cursor = Cursor.decode(cursor_string: @cursor, columns: @columns)
-        relation = apply_cursor(relation, decoded_cursor)
-      end
+      relation = build_cursor_relation(@current_cursor)
 
       relation = relation.limit(@page_size + 1)
       records_plus_one = relation.to_a
@@ -106,9 +140,9 @@ module ActiveRecordCursorPaginate
 
       if @is_forward_pagination
         has_next_page = has_additional
-        has_previous_page = @cursor.present?
+        has_previous_page = @current_cursor.present?
       else
-        has_next_page = @cursor.present?
+        has_next_page = @current_cursor.present?
         has_previous_page = has_additional
       end
 
@@ -116,7 +150,8 @@ module ActiveRecordCursorPaginate
         records,
         order_columns: cursor_column_names,
         has_next: has_next_page,
-        has_previous: has_previous_page
+        has_previous: has_previous_page,
+        nullable_columns: nullable_cursor_column_names
       )
 
       advance_by_page(page) unless page.empty?
@@ -139,6 +174,13 @@ module ActiveRecordCursorPaginate
       end
     end
 
+    # Total number of records to iterate by this paginator.
+    # @return [Integer]
+    #
+    def total_count
+      @total_count ||= build_cursor_relation(@cursor).count(:all)
+    end
+
     private
       def normalize_order(order)
         order ||= {}
@@ -168,20 +210,81 @@ module ActiveRecordCursorPaginate
         result
       end
 
+      def build_cursor_relation(cursor)
+        relation = @relation
+
+        # Non trivial columns (expressions or joined tables columns).
+        if @columns.any?(/\W/)
+          arel_columns = @columns.map.with_index do |column, i|
+            arel_column(column).as("cursor_column_#{i + 1}")
+          end
+
+          relation =
+            if relation.select_values.empty?
+              relation.select(relation.arel_table[Arel.star], arel_columns)
+            else
+              relation.select(arel_columns)
+            end
+        end
+
+        pagination_directions = @directions.map { |direction| pagination_direction(direction) }
+        relation = relation.reorder(@columns.zip(pagination_directions).to_h)
+
+        if cursor
+          decoded_cursor = Cursor.decode(cursor_string: cursor, columns: cursor_column_names, nullable_columns: nullable_cursor_column_names)
+          relation = apply_cursor(relation, decoded_cursor)
+        end
+
+        relation
+      end
+
+      def nullable_cursor_column_names
+        @nullable_columns.map do |column|
+          cursor_column_names[@columns.index(column)]
+        end
+      end
+
+      def cursor_column_names
+        if @columns.any?(/\W/)
+          @columns.size.times.map { |i| "cursor_column_#{i + 1}" }
+        else
+          @columns
+        end
+      end
+
       def apply_cursor(relation, cursor)
-        operators = @directions.map { |direction| pagination_operator(direction) }
-        cursor_positions = cursor.columns.zip(cursor.values, operators)
+        cursor_positions = @columns.zip(cursor.values, @directions)
 
         where_clause = nil
-        cursor_positions.reverse_each.with_index do |(column, value, operator), index|
-          where_clause =
-            if index == 0
-              arel_column(column).public_send(operator, value)
+
+        cursor_positions.reverse_each.with_index do |(column, value, direction), index|
+          previous_where_clause = where_clause
+
+          operator = pagination_operator(direction)
+          arel_column = arel_column(column)
+
+          # The last column can't be nil.
+          if index == 0
+            where_clause = arel_column.public_send(operator, value)
+          elsif value.nil?
+            if nulls_at_end?(direction)
+              # We are at the section with nulls, which is at the end ([x, x, null, null, null])
+              where_clause = arel_column.eq(nil).and(previous_where_clause)
             else
-              arel_column(column).public_send(operator, value).or(
-                arel_column(column).eq(value).and(where_clause)
-              )
+              # We are at the section with nulls, which is at the beginning ([null, null, null, x, x])
+              where_clause = arel_column.not_eq(nil)
+              where_clause = arel_column.eq(nil).and(previous_where_clause).or(where_clause)
             end
+          else
+            where_clause = arel_column.public_send(operator, value).or(
+              arel_column.eq(value).and(previous_where_clause)
+            )
+
+            if nullable_column?(column) && nulls_at_end?(direction)
+              # Since column's value is not null, nulls can only be at the end.
+              where_clause = arel_column.eq(nil).or(where_clause)
+            end
+          end
         end
 
         relation.where(where_clause)
@@ -214,12 +317,27 @@ module ActiveRecordCursorPaginate
       end
 
       def advance_by_page(page)
-        @cursor =
+        @current_cursor =
           if @is_forward_pagination
             page.next_cursor
           else
             page.previous_cursor
           end
       end
+
+      def nulls_at_end?(direction)
+        (direction == :asc && !small_nulls?) || (direction == :desc && small_nulls?)
+      end
+
+      def small_nulls?
+        # PostgreSQL considers NULLs larger than any value,
+        # opposite for SQLite and MySQL.
+        db_config = @relation.klass.connection_pool.db_config
+        db_config.adapter !~ /postg/ # postgres and postgis
+      end
+
+      def nullable_column?(column)
+        @nullable_columns.include?(column)
+      end
   end
 end

data/lib/activerecord_cursor_paginate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActiveRecordCursorPaginate
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord_cursor_paginate
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - fatkodima
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-05-23 00:00:00.000000000 Z
+date: 2025-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -41,13 +41,13 @@ files:
 - lib/activerecord_cursor_paginate/page.rb
 - lib/activerecord_cursor_paginate/paginator.rb
 - lib/activerecord_cursor_paginate/version.rb
-homepage: https://github.com/fatkodima/activerecord_cursor_paginate
+homepage: https://github.com/healthie/activerecord_cursor_paginate
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://github.com/fatkodima/activerecord_cursor_paginate
-  source_code_uri: https://github.com/fatkodima/activerecord_cursor_paginate
-  changelog_uri: https://github.com/fatkodima/activerecord_cursor_paginate/blob/master/CHANGELOG.md
+  homepage_uri: https://github.com/healthie/activerecord_cursor_paginate
+  source_code_uri: https://github.com/healthie/activerecord_cursor_paginate
+  changelog_uri: https://github.com/healthie/activerecord_cursor_paginate/blob/master/CHANGELOG.md
 post_install_message:
 rdoc_options: []
 require_paths: