sequel-batches 0.2.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0574ea7f800e68b8316c8f03cf5375dd165640ced4cf1a41016f02848fa61954
-  data.tar.gz: 980b78345324a39f7dc8b45bf79ee0096fe638285d761e768729ac398b11ed18
+  metadata.gz: 2ba86551c2e6fa28e3c012fec381ca293267b3c04629f8295114724a2c7ff99a
+  data.tar.gz: c36a543672f8d6869e4189ad808ac15c275d0342cc41ec3a258a0299e4225829
 SHA512:
-  metadata.gz: bdca5b44da27259b09e24662024f0b10b6d095a85762b98ef4cded1685f0af02daf7cfe62f230d5eafd81fc51d45aec5f13fba9dfbf4d144e56113371d450542
-  data.tar.gz: d0443e858f6202492cb7a1a67585bdcfffab0f47f630bd9f817937bb10e21dbf9dcfda27417f345b69c92d86855a96f2947d99b50f1c2b775d198ceab0589a7e
+  metadata.gz: 655a16e69b97c23fe80c92beef06e5cc80fa242b161955ef3702193385528b67b8575a14c5b75ed0e7cc4af559e1a04e0e0141e8ab4ef60c5cfc350ac5afda9c
+  data.tar.gz: c36d801caaf7cefc37520c4db12ce9d47ff62c20eb2c9c8c2c9813775829bd5acbaa0bc8f1d6953b5fc1c6dfbcea34650eec29ca37f8c15731872bd1915f193a

data/.github/workflows/ci.yml

@@ -0,0 +1,51 @@
+name: CI
+
+on: [pull_request, push]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != 'umbrellio/sequel-batches'
+
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: root
+          POSTGRES_HOST_AUTH_METHOD: trust
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    env:
+      PGHOST: localhost
+      PGUSER: root
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.6, 2.7, "3.0", jruby-9.3.1.0, ruby-head, jruby-head]
+
+    name: ${{ matrix.ruby }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - run: psql -c 'CREATE DATABASE batches_test'
+      - run: bundle exec rake bundle:audit
+      - run: bundle exec rake lint
+      - run: bundle exec rspec
+        continue-on-error: ${{ matrix.ruby == 'jruby-head' ||  matrix.ruby == 'ruby-head' }}
+
+      - uses: coverallsapp/github-action@v1.1.2
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

data/.rubocop.yml

@@ -0,0 +1,18 @@
+inherit_gem:
+  rubocop-config-umbrellio: lib/rubocop.yml
+
+AllCops:
+  DisplayCopNames: true
+  TargetRubyVersion: 2.6
+  Include:
+    - lib/**/*.rb
+    - spec/**/*.rb
+    - Gemfile
+    - Rakefile
+    - sequel-batches.gemspec
+
+Layout/LineLength:
+  Max: 120
+
+Naming/MethodParameterName:
+  Enabled: false

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 # Specify your gem's dependencies in sequel-batches.gemspec

data/README.md

@@ -1,25 +1,9 @@
-# Sequel::Batches [![Build Status](https://travis-ci.org/umbrellio/sequel-batches.svg?branch=master)](https://travis-ci.org/umbrellio/sequel-batches) [![Coverage Status](https://coveralls.io/repos/github/umbrellio/sequel-batches/badge.svg?branch=master)](https://coveralls.io/github/umbrellio/sequel-batches?branch=master)
+# Sequel::Batches    [![Gem Version](https://badge.fury.io/rb/sequel-batches.svg)](https://badge.fury.io/rb/sequel-batches) [![Build Status](https://travis-ci.org/umbrellio/sequel-batches.svg?branch=master)](https://travis-ci.org/umbrellio/sequel-batches) [![Coverage Status](https://coveralls.io/repos/github/umbrellio/sequel-batches/badge.svg?branch=master)](https://coveralls.io/github/umbrellio/sequel-batches?branch=master)
 
-This dataset extension provides the method #in_batches. The method splits dataset in parts and yields it.
+This dataset extension provides the `#in_batches` method. The method splits dataset in parts and yields it.
 
 Note: currently only PostgreSQL database is supported.
 
-You can set the following options:
-
-### pk
-Overrides primary key of your dataset. This option is required in case your table doesn't have a real PK, otherwise you will get `Sequel::Extensions::Batches::MissingPKError`.
-
-Note that you have to provide columns that don't contain NULL values, otherwise this may not work as intended. You will receive `Sequel::Extensions::Batches::NullPKError` in case batch processing detects a NULL value on it's way, but it's not guaranteed since it doesn't check all the rows for performance reasons.
-
-### of
-Sets chunk size (1000 by default).
-
-### start
-A hash `{ [column]: <start_value> }` that represents frame start for batch processing. Note that you will get `Sequel::Extensions::Batches::InvalidPKError` in case you provide a hash with wrong keys (ordering matters as well).
-
-### finish
-Same as `start` but represents the frame end.
-
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -38,13 +22,13 @@ Or install it yourself as:
 
 ## Usage
 
-In order to use the feature you should enable the extension
+In order to use the feature you should enable the extension:
 
 ```ruby
 Sequel::DATABASES.first.extension :batches
 ```
 
-And then the method becomes available on dataset
+After that the `#in_batches` method becomes available on dataset:
 
 ```ruby
 User.where(role: "admin").in_batches(of: 4) do |ds|
@@ -52,28 +36,51 @@ User.where(role: "admin").in_batches(of: 4) do |ds|
 end
 ```
 
-Finally, here's an example including all the available options
+Finally, here's an example including all the available options:
 
 ```ruby
-Event.where(type: "login").in_batches(of: 4, pk: [:project_id, :external_user_id], start: { project_id: 2, external_user_id: 3 }, finish: { project_id: 5, external_user_id: 70 }) do |ds|
+options = {
+  of: 4,
+  pk: [:project_id, :external_user_id],
+  start: { project_id: 2, external_user_id: 3 },
+  finish: { project_id: 5, external_user_id: 70 },
+  order: :desc,
+}
+
+Event.where(type: "login").in_batches(options) do |ds|
   ds.delete
 end
 ```
 
-## Development
+## Options
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+You can set the following options:
+
+### pk
+Overrides primary key of your dataset. This option is required in case your table doesn't have a real PK, otherwise you will get `Sequel::Extensions::Batches::MissingPKError`.
+
+Note that you have to provide columns that don't contain NULL values, otherwise this may not work as intended. You will receive `Sequel::Extensions::Batches::NullPKError` in case batch processing detects a NULL value on it's way, but it's not guaranteed since it doesn't check all the rows for performance reasons.
+
+### of
+Sets chunk size (1000 by default).
+
+### start
+A hash `{ [column]: <start_value> }` that represents frame start for batch processing. Note that you will get `Sequel::Extensions::Batches::InvalidPKError` in case you provide a hash with wrong keys (ordering matters as well).
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+### finish
+Same as `start` but represents the frame end.
+
+### order
+Specifies the primary key order (can be :asc or :desc). Defaults to :asc.
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/sequel-batches. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+Bug reports and pull requests are welcome on GitHub at https://github.com/umbrellio/sequel-batches.
 
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
 
-## Code of Conduct
-
-Everyone interacting in the Sequel::Batches project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/sequel-batches/blob/master/CODE_OF_CONDUCT.md).
+<a href="https://github.com/umbrellio/">
+<img style="float: left;" src="https://umbrellio.github.io/Umbrellio/supported_by_umbrellio.svg" alt="Supported by Umbrellio" width="439" height="72">
+</a>

data/Rakefile

@@ -1,6 +1,18 @@
+# frozen_string_literal: true
+
 require "bundler/gem_tasks"
+require "bundler/audit/task"
 require "rspec/core/rake_task"
+require "rubocop/rake_task"
 
 RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new(:lint) do |t|
+  config_path = File.expand_path(File.join(".rubocop.yml"), __dir__)
+
+  t.options = ["--config", config_path]
+  t.requires << "rubocop-rspec"
+  t.requires << "rubocop-performance"
+end
+Bundler::Audit::Task.new
 
-task :default => :spec
+task default: %i[lint spec]

data/lib/sequel/extensions/batches/yielder.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Sequel::Extensions::Batches
+  class Yielder
+    attr_accessor :ds, :of, :start, :finish, :order
+    attr_writer :pk
+
+    def initialize(ds:, **options)
+      self.ds = ds
+      self.pk = options.delete(:pk)
+      self.of = options.delete(:of) || 1000
+      self.start = options.delete(:start)
+      self.finish = options.delete(:finish)
+      self.order = options.delete(:order) || :asc
+
+      raise ArgumentError, ":order must be :asc or :desc, got #{order.inspect}" unless %i[asc desc].include?(order)
+      raise ArgumentError, "unknown options: #{options.keys.inspect}" if options.any?
+    end
+
+    def call
+      base_ds = setup_base_ds or return
+      return enum_for(:call) unless block_given?
+
+      current_instance = nil
+
+      loop do
+        working_ds =
+          if current_instance
+            base_ds.where(generate_conditions(current_instance.to_h, sign: sign_from_exclusive))
+          else
+            base_ds
+          end
+
+        working_ds_pk = working_ds.select(*qualified_pk).order(order_by).limit(of)
+        current_instance = db.from(working_ds_pk).select(*pk).order(order_by).last or break
+        working_ds = working_ds.where(generate_conditions(current_instance.to_h, sign: sign_to_inclusive))
+
+        yield working_ds
+      end
+    end
+
+    private
+
+    def db
+      ds.db
+    end
+
+    def pk
+      @pk ||= begin
+        pk = db.schema(ds.first_source).select { |x| x[1][:primary_key] }.map(&:first)
+        raise MissingPKError if pk.empty?
+        pk
+      end
+    end
+
+    def asc_order?
+      order == :asc
+    end
+
+    def sign_from_exclusive
+      asc_order? ? :> : :<
+    end
+
+    def sign_from_inclusive
+      asc_order? ? :>= : :<=
+    end
+
+    def sign_to_inclusive
+      asc_order? ? :<= : :>=
+    end
+
+    def order_by(qualified: false)
+      columns = qualified ? qualified_pk : pk
+      asc_order? ? Sequel.asc(columns) : Sequel.desc(columns)
+    end
+
+    def qualified_pk
+      @qualified_pk ||= pk.map { |x| Sequel[ds.first_source][x] }
+    end
+
+    def check_pk(input_pk)
+      raise InvalidPKError if input_pk.keys != pk
+      input_pk
+    end
+
+    def generate_conditions(input_pk, sign:)
+      raise NullPKError if input_pk.values.any?(&:nil?)
+      row_expr = Sequel.function(:row, *input_pk.values)
+      Sequel.function(:row, *qualified_pk).public_send(sign, row_expr)
+    end
+
+    def setup_base_ds
+      base_ds = ds.order(order_by(qualified: true))
+      base_ds = base_ds.where(generate_conditions(check_pk(start), sign: sign_from_inclusive)) if start
+      base_ds = base_ds.where(generate_conditions(check_pk(finish), sign: sign_to_inclusive)) if finish
+
+      pk_ds = db.from(base_ds.select(*qualified_pk)).select(*pk).order(order_by)
+      actual_start = pk_ds.first
+      actual_finish = pk_ds.last
+
+      return unless actual_start && actual_finish
+
+      base_ds = base_ds.where(generate_conditions(actual_start, sign: sign_from_inclusive))
+      base_ds.where(generate_conditions(actual_finish, sign: sign_to_inclusive))
+    end
+  end
+end

data/lib/sequel/extensions/batches.rb

@@ -1,5 +1,4 @@
-require "sequel/extensions/batches/version"
-require "sequel/model"
+# frozen_string_literal: true
 
 module Sequel
   module Extensions
@@ -8,55 +7,13 @@ module Sequel
       NullPKError = Class.new(StandardError)
       InvalidPKError = Class.new(StandardError)
 
-      def in_batches(pk: nil, of: 1000, start: nil, finish: nil)
-        pk ||= db.schema(first_source).select { |x| x[1][:primary_key] }.map(&:first)
-        raise MissingPKError if pk.empty?
-
-        qualified_pk = pk.map { |x| Sequel[first_source][x] }
-
-        check_pk = lambda do |input_pk|
-          raise InvalidPKError if input_pk.keys != pk
-          input_pk
-        end
-
-        conditions = lambda do |pk, sign:|
-          raise NullPKError if pk.values.any?(&:nil?)
-          row_expr = Sequel.function(:row, *pk.values)
-          Sequel.function(:row, *qualified_pk).public_send(sign, row_expr)
-        end
-
-        base_ds = order(*qualified_pk)
-        base_ds = base_ds.where(conditions.call(check_pk.call(start), sign: :>=)) if start
-        base_ds = base_ds.where(conditions.call(check_pk.call(finish), sign: :<=)) if finish
-
-        pk_ds = db.from(base_ds).select(*pk).order(*pk)
-        actual_start = pk_ds.first
-        actual_finish = pk_ds.last
-
-        return unless actual_start && actual_finish
-
-        base_ds = base_ds.where(conditions.call(actual_start, sign: :>=))
-        base_ds = base_ds.where(conditions.call(actual_finish, sign: :<=))
-
-        current_instance = nil
-
-        loop do
-          if current_instance
-            working_ds = base_ds.where(conditions.call(current_instance.to_h, sign: :>))
-          else
-            working_ds = base_ds
-          end
-
-          current_instance = db.from(working_ds.limit(of)).select(*pk).order(*pk).last or break
-          working_ds = working_ds.where(conditions.call(current_instance.to_h, sign: :<=))
-
-          yield working_ds
-        end
+      def in_batches(**options, &block)
+        Sequel::Extensions::Batches::Yielder.new(ds: self, **options).call(&block)
       end
-
-      private
-
-      ::Sequel::Dataset.register_extension(:batches, Batches)
     end
   end
 end
+
+::Sequel::Dataset.register_extension(:batches, Sequel::Extensions::Batches)
+
+require_relative "batches/yielder"

data/sequel-batches.gemspec

@@ -1,40 +1,34 @@
+# frozen_string_literal: true
 
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "sequel/extensions/batches/version"
 
 Gem::Specification.new do |spec|
-  spec.name          = "sequel-batches"
-  spec.version       = Sequel::Extensions::Batches::VERSION
-  spec.authors       = ["fiscal-cliff", "umbrellio"]
-  spec.email         = ["oss@umbrellio.biz"]
-
-  spec.summary       = %q{The extension mimics AR5 batches api}
-  spec.description   = %q{Allows you to split your dataset in batches}
-  spec.homepage      = "https://github.com/umbrellio/sequel-batches"
-  spec.license       = "MIT"
-
-  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
-  # to allow pushing to a single host or delete this section to allow pushing to any host.
-  if spec.respond_to?(:metadata)
-    spec.metadata["allowed_push_host"] = "https://rubygems.org"
-  else
-    raise "RubyGems 2.0 or newer is required to protect against " \
-      "public gem pushes."
-  end
-
-  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
-    f.match(%r{^(test|spec|features)/})
-  end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.name = "sequel-batches"
+  spec.version = "1.1.0"
+  spec.authors = %w[fiscal-cliff umbrellio]
+  spec.email = ["oss@umbrellio.biz"]
+  spec.required_ruby_version = ">= 2.6"
+
+  spec.summary = "The extension mimics AR5 batches api"
+  spec.description = "Allows you to split your dataset in batches"
+  spec.homepage = "https://github.com/umbrellio/sequel-batches"
+  spec.license = "MIT"
+
+  spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.require_paths = ["lib"]
 
+  spec.add_runtime_dependency "sequel"
+
   spec.add_development_dependency "bundler"
-  spec.add_development_dependency "coveralls"
+  spec.add_development_dependency "bundler-audit"
+
   spec.add_development_dependency "pry"
   spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec"
 
-  spec.add_runtime_dependency "sequel"
+  spec.add_development_dependency "rubocop-config-umbrellio"
+
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "simplecov-lcov"
 end

metadata

@@ -1,16 +1,30 @@
 --- !ruby/object:Gem::Specification
 name: sequel-batches
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 1.1.0
 platform: ruby
 authors:
 - fiscal-cliff
 - umbrellio
-autorequire: 
-bindir: exe
+autorequire:
+bindir: bin
 cert_chain: []
-date: 2019-08-19 00:00:00.000000000 Z
+date: 2022-08-05 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -26,7 +40,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: coveralls
+  name: bundler-audit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,6 +81,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-config-umbrellio
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -82,13 +110,27 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: sequel
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov-lcov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -102,26 +144,24 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
+- ".rubocop.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
-- gemfiles/ci.gemfile
-- lib/sequel.rb
 - lib/sequel/extensions/batches.rb
-- lib/sequel/extensions/batches/version.rb
+- lib/sequel/extensions/batches/yielder.rb
 - log/.keep
 - sequel-batches.gemspec
 homepage: https://github.com/umbrellio/sequel-batches
 licenses:
 - MIT
-metadata:
-  allowed_push_host: https://rubygems.org
-post_install_message: 
+metadata: {}
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -129,15 +169,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.3.19
+signing_key:
 specification_version: 4
 summary: The extension mimics AR5 batches api
 test_files: []

data/.travis.yml

@@ -1,34 +0,0 @@
-sudo: false
-
-language: ruby
-
-rvm:
-  - 2.3
-  - 2.4
-  - 2.5
-  - 2.6
-  - jruby-9.2.8.0
-  - ruby-head
-  - jruby-head
-
-before_install: gem install bundler
-
-env: SEQUEL_VERSION="~> 5.0"
-
-gemfile: gemfiles/ci.gemfile
-
-addons:
-  postgresql: "9.6"
-
-services:
-  - postgresql
-
-matrix:
-  include:
-    - rvm: 2.6
-      env: SEQUEL_VERSION="~> 4.0"
-  allow_failures:
-    - rvm: ruby-head
-      env: SEQUEL_VERSION="~> 5.0"
-    - rvm: jruby-head
-      env: SEQUEL_VERSION="~> 5.0"

data/gemfiles/ci.gemfile

@@ -1,15 +0,0 @@
-source 'https://rubygems.org'
-
-gemspec :path => '../'
-
-# MRI/Rubinius Adapter Dependencies
-platforms :ruby do
-  gem "pg", "~>0.20.0"
-end
-
-# JRuby Adapter Dependencies
-platforms :jruby do
-  gem 'jdbc-postgres', '~> 9.4'
-end
-
-gem "sequel", "#{ENV['SEQUEL_VERSION']}"

data/lib/sequel/extensions/batches/version.rb

@@ -1,7 +0,0 @@
-module Sequel
-  module Extensions
-    module Batches
-      VERSION = "0.2.1"
-    end
-  end
-end

data/lib/sequel.rb

@@ -1 +0,0 @@
-require "sequel/extensions/batches"