sequel-batches 1.0.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e24dcc9f437a98a65709c2486c46e6e3ddbeb89a1d3298af245c4d20ae758492
-  data.tar.gz: 5d5670cc31a6afcf7cdee391dc889c9636e9a201fe1f68e01860df2174e462fd
+  metadata.gz: 6ffd198d92d022335e38e2422ca6f31f790f3501ebc463b15f8f9cde318faea6
+  data.tar.gz: 14d9cdbf89a5099136a586d2a468360524df0142b653c4908a86ba3c183fd1f9
 SHA512:
-  metadata.gz: 90f54e68d80fe0872415ab53b4d217c803eaf1d711e083bc518799475c805cd050d2a5605d42603b959fcde28ddfb5ef679344f46bd601677a31ab192bf4b4aa
-  data.tar.gz: a7f69e996cdedfec3b83160b15858fb090c970d2e33516fe53c2e64e1c2c5ed5d198241ea6836352cd1a3f260ebae8667b79bcd4847e6d08e4556c22a996a306
+  metadata.gz: cc23bb340fec3c041afdda7c1cb21d4dd9af5717d66149a9494f419b169e292c24074aa378dbfadc24a8d827a43aa52dcbafff3a2fc70c287c80d688d088769e
+  data.tar.gz: faba27222e6b8d0a7a3efcfd7eedbf9d423c2f87aa837bd4173bb22cb58b9084369c0ae5de054461a64d8e419258d9f1ed04c3b97dd26294c0d271608b730dad

data/.github/workflows/ci.yml

@@ -0,0 +1,50 @@
+name: CI
+
+on: [pull_request, push]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != 'umbrellio/sequel-batches'
+
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: root
+          POSTGRES_HOST_AUTH_METHOD: trust
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    env:
+      PGHOST: localhost
+      PGUSER: root
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ["2.7", "3.0", "3.1"]
+
+    name: ${{ matrix.ruby }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - run: psql -c 'CREATE DATABASE batches_test'
+      - run: bundle exec rake bundle:audit
+      - run: bundle exec rake lint
+      - run: bundle exec rspec
+
+      - uses: coverallsapp/github-action@v1.1.2
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

data/.gitignore

@@ -11,7 +11,6 @@
 .rspec_status
 
 *.gem
-*.lock
 *.log
 .coveralls.yml
 .ruby-version

data/.rubocop.yml

@@ -3,7 +3,16 @@ inherit_gem:
 
 AllCops:
   DisplayCopNames: true
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.7
+  Include:
+    - lib/**/*.rb
+    - spec/**/*.rb
+    - Gemfile
+    - Rakefile
+    - sequel-batches.gemspec
 
-Naming/UncommunicativeMethodParamName:
+Layout/LineLength:
+  Max: 120
+
+Naming/MethodParameterName:
   Enabled: false

data/Gemfile

@@ -4,13 +4,3 @@ source "https://rubygems.org"
 
 # Specify your gem's dependencies in sequel-batches.gemspec
 gemspec
-
-# MRI/Rubinius Adapter Dependencies
-platforms :ruby do
-  gem "pg"
-end
-
-# JRuby Adapter Dependencies
-platforms :jruby do
-  gem "jdbc-postgres"
-end

data/Gemfile.lock

@@ -0,0 +1,113 @@
+PATH
+  remote: .
+  specs:
+    sequel-batches (2.0.0)
+      sequel
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (7.0.3.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    ast (2.4.2)
+    bundler-audit (0.9.1)
+      bundler (>= 1.2.0, < 3)
+      thor (~> 1.0)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.10)
+    diff-lcs (1.5.0)
+    docile (1.4.0)
+    i18n (1.12.0)
+      concurrent-ruby (~> 1.0)
+    method_source (1.0.0)
+    minitest (5.16.2)
+    parallel (1.22.1)
+    parser (3.1.2.0)
+      ast (~> 2.4.1)
+    pg (1.4.2)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rack (2.2.4)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    regexp_parser (2.5.0)
+    rexml (3.2.5)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    rubocop (1.30.1)
+      parallel (~> 1.10)
+      parser (>= 3.1.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.18.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.19.1)
+      parser (>= 3.1.1.0)
+    rubocop-config-umbrellio (1.30.0.65)
+      rubocop (~> 1.30.0)
+      rubocop-performance (~> 1.14.0)
+      rubocop-rails (~> 2.14.2)
+      rubocop-rake (~> 0.6.0)
+      rubocop-rspec (~> 2.11.1)
+      rubocop-sequel (~> 0.3.3)
+    rubocop-performance (1.14.3)
+      rubocop (>= 1.7.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
+    rubocop-rails (2.14.2)
+      activesupport (>= 4.2.0)
+      rack (>= 1.1)
+      rubocop (>= 1.7.0, < 2.0)
+    rubocop-rake (0.6.0)
+      rubocop (~> 1.0)
+    rubocop-rspec (2.11.1)
+      rubocop (~> 1.19)
+    rubocop-sequel (0.3.4)
+      rubocop (~> 1.0)
+    ruby-progressbar (1.11.0)
+    sequel (5.59.0)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov-lcov (0.8.0)
+    simplecov_json_formatter (0.1.4)
+    thor (1.2.1)
+    tzinfo (2.0.5)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.2.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  bundler-audit
+  pg
+  pry
+  rake
+  rspec
+  rubocop-config-umbrellio
+  sequel-batches!
+  simplecov
+  simplecov-lcov
+
+BUNDLED WITH
+   2.3.19

data/README.md

@@ -4,22 +4,6 @@ This dataset extension provides the `#in_batches` method. The method splits data
 
 Note: currently only PostgreSQL database is supported.
 
-You can set the following options:
-
-### pk
-Overrides primary key of your dataset. This option is required in case your table doesn't have a real PK, otherwise you will get `Sequel::Extensions::Batches::MissingPKError`.
-
-Note that you have to provide columns that don't contain NULL values, otherwise this may not work as intended. You will receive `Sequel::Extensions::Batches::NullPKError` in case batch processing detects a NULL value on it's way, but it's not guaranteed since it doesn't check all the rows for performance reasons.
-
-### of
-Sets chunk size (1000 by default).
-
-### start
-A hash `{ [column]: <start_value> }` that represents frame start for batch processing. Note that you will get `Sequel::Extensions::Batches::InvalidPKError` in case you provide a hash with wrong keys (ordering matters as well).
-
-### finish
-Same as `start` but represents the frame end.
-
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -28,14 +12,6 @@ Add this line to your application's Gemfile:
 gem 'sequel-batches'
 ```
 
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install sequel-batches
-
 ## Usage
 
 In order to use the feature you should enable the extension:
@@ -60,6 +36,7 @@ options = {
   pk: [:project_id, :external_user_id],
   start: { project_id: 2, external_user_id: 3 },
   finish: { project_id: 5, external_user_id: 70 },
+  order: :desc,
 }
 
 Event.where(type: "login").in_batches(options) do |ds|
@@ -67,6 +44,32 @@ Event.where(type: "login").in_batches(options) do |ds|
 end
 ```
 
+## Options
+
+You can set the following options:
+
+### pk
+
+Overrides primary key of your dataset. This option is required in case your table doesn't have a real PK, otherwise you will get `Sequel::Extensions::Batches::MissingPKError`.
+
+Note that you have to provide columns that don't contain NULL values, otherwise this may not work as intended. You will receive `Sequel::Extensions::Batches::NullPKError` in case batch processing detects a NULL value on it's way, but it's not guaranteed since it doesn't check all the rows for performance reasons.
+
+### of
+
+Sets chunk size (1000 by default).
+
+### start
+
+A hash `{ [column]: <start_value> }` that represents frame start for batch processing. Note that you will get `Sequel::Extensions::Batches::InvalidPKError` in case you provide a hash with wrong keys (ordering matters as well).
+
+### finish
+
+Same as `start` but represents the frame end.
+
+### order
+
+Specifies the primary key order (can be `:asc` or `:desc`). Defaults to `:asc`.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/umbrellio/sequel-batches.

data/Rakefile

@@ -1,10 +1,18 @@
 # frozen_string_literal: true
 
 require "bundler/gem_tasks"
+require "bundler/audit/task"
 require "rspec/core/rake_task"
 require "rubocop/rake_task"
 
 RSpec::Core::RakeTask.new(:spec)
-RuboCop::RakeTask.new(:lint)
+RuboCop::RakeTask.new(:lint) do |t|
+  config_path = File.expand_path(File.join(".rubocop.yml"), __dir__)
+
+  t.options = ["--config", config_path]
+  t.requires << "rubocop-rspec"
+  t.requires << "rubocop-performance"
+end
+Bundler::Audit::Task.new
 
 task default: %i[lint spec]

data/lib/sequel/extensions/batches/yielder.rb

@@ -2,32 +2,38 @@
 
 module Sequel::Extensions::Batches
   class Yielder
-    attr_accessor :ds, :of, :start, :finish
+    attr_accessor :ds, :of, :start, :finish, :order
     attr_writer :pk
 
-    def initialize(ds:, pk: nil, of: 1000, start: nil, finish: nil)
+    def initialize(ds:, **options)
       self.ds = ds
-      self.pk = pk
-      self.of = of
-      self.start = start
-      self.finish = finish
+      self.pk = options.delete(:pk)
+      self.of = options.delete(:of) || 1000
+      self.start = options.delete(:start)
+      self.finish = options.delete(:finish)
+      self.order = options.delete(:order) || :asc
+
+      raise ArgumentError, ":order must be :asc or :desc, got #{order.inspect}" unless %i[asc desc].include?(order)
+      raise ArgumentError, "unknown options: #{options.keys.inspect}" if options.any?
     end
 
     def call
       base_ds = setup_base_ds or return
+      return enum_for(:call) unless block_given?
 
       current_instance = nil
 
       loop do
         working_ds =
           if current_instance
-            base_ds.where(generate_conditions(current_instance.to_h, sign: :>))
+            base_ds.where(generate_conditions(current_instance.to_h, sign: sign_from_exclusive))
           else
             base_ds
           end
 
-        current_instance = db.from(working_ds.limit(of)).select(*pk).order(*pk).last or break
-        working_ds = working_ds.where(generate_conditions(current_instance.to_h, sign: :<=))
+        working_ds_pk = working_ds.select(*qualified_pk).order(order_by).limit(of)
+        current_instance = db.from(working_ds_pk).select(*pk).order(order_by).last or break
+        working_ds = working_ds.where(generate_conditions(current_instance.to_h, sign: sign_to_inclusive))
 
         yield working_ds
       end
@@ -47,6 +53,27 @@ module Sequel::Extensions::Batches
       end
     end
 
+    def asc_order?
+      order == :asc
+    end
+
+    def sign_from_exclusive
+      asc_order? ? :> : :<
+    end
+
+    def sign_from_inclusive
+      asc_order? ? :>= : :<=
+    end
+
+    def sign_to_inclusive
+      asc_order? ? :<= : :>=
+    end
+
+    def order_by(qualified: false)
+      columns = qualified ? qualified_pk : pk
+      asc_order? ? Sequel.asc(columns) : Sequel.desc(columns)
+    end
+
     def qualified_pk
       @qualified_pk ||= pk.map { |x| Sequel[ds.first_source][x] }
     end
@@ -63,20 +90,18 @@ module Sequel::Extensions::Batches
     end
 
     def setup_base_ds
-      base_ds = ds.order(*qualified_pk)
-      base_ds = base_ds.where(generate_conditions(check_pk(start), sign: :>=)) if start
-      base_ds = base_ds.where(generate_conditions(check_pk(finish), sign: :<=)) if finish
+      base_ds = ds.order(order_by(qualified: true))
+      base_ds = base_ds.where(generate_conditions(check_pk(start), sign: sign_from_inclusive)) if start
+      base_ds = base_ds.where(generate_conditions(check_pk(finish), sign: sign_to_inclusive)) if finish
 
-      pk_ds = db.from(base_ds).select(*pk).order(*pk)
+      pk_ds = db.from(base_ds.select(*qualified_pk)).select(*pk).order(order_by)
       actual_start = pk_ds.first
       actual_finish = pk_ds.last
 
       return unless actual_start && actual_finish
 
-      base_ds = base_ds.where(generate_conditions(actual_start, sign: :>=))
-      base_ds = base_ds.where(generate_conditions(actual_finish, sign: :<=))
-
-      base_ds
+      base_ds = base_ds.where(generate_conditions(actual_start, sign: sign_from_inclusive))
+      base_ds.where(generate_conditions(actual_finish, sign: sign_to_inclusive))
     end
   end
 end

data/sequel-batches.gemspec

@@ -5,9 +5,10 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name = "sequel-batches"
-  spec.version = "1.0.0"
-  spec.authors = ["fiscal-cliff", "umbrellio"]
+  spec.version = "2.0.0"
+  spec.authors = %w[fiscal-cliff umbrellio]
   spec.email = ["oss@umbrellio.biz"]
+  spec.required_ruby_version = ">= 2.7"
 
   spec.summary = "The extension mimics AR5 batches api"
   spec.description = "Allows you to split your dataset in batches"
@@ -20,10 +21,12 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "sequel"
 
   spec.add_development_dependency "bundler"
-  spec.add_development_dependency "coveralls"
+  spec.add_development_dependency "bundler-audit"
+  spec.add_development_dependency "pg"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "rubocop-config-umbrellio"
   spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "simplecov-lcov"
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: sequel-batches
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.0.0
 platform: ruby
 authors:
 - fiscal-cliff
 - umbrellio
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-08-19 00:00:00.000000000 Z
+date: 2022-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -40,7 +40,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: coveralls
+  name: bundler-audit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,6 +137,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov-lcov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Allows you to split your dataset in batches
 email:
 - oss@umbrellio.biz
@@ -130,12 +158,13 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -147,7 +176,7 @@ homepage: https://github.com/umbrellio/sequel-batches
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -155,15 +184,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.3.19
+signing_key:
 specification_version: 4
 summary: The extension mimics AR5 batches api
 test_files: []

data/.travis.yml

@@ -1,25 +0,0 @@
-sudo: false
-
-language: ruby
-
-rvm:
-  - 2.3
-  - 2.4
-  - 2.5
-  - 2.6
-  - jruby-9.2.8.0
-  - ruby-head
-  - jruby-head
-
-before_install: gem install bundler
-
-addons:
-  postgresql: "9.6"
-
-services:
-  - postgresql
-
-matrix:
-  allow_failures:
-    - rvm: ruby-head
-    - rvm: jruby-head