sequel-batches 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 59714df347d4f6cb7e6e697051a8a049fda88dc1
-  data.tar.gz: 011f44112aee94dda9e0e47e10999e13fe4972fd
+SHA256:
+  metadata.gz: c22ae251b55c86ad2a6e7a62edfa43b7be2cc629da1457532ddf9d7575450130
+  data.tar.gz: 32e9759bf95501a53b12f7813a085821867b3405c5cadf9f716a3306950e7a27
 SHA512:
-  metadata.gz: eb8daebb6ba2b19abcb3ee3ce6070042b28b36e9d63d01bb9a35f367faacc228e8a1908a22631d08291d70f54d49e8b1f6d10bdb5403b9825b7bc6b8610cb7b2
-  data.tar.gz: 9b9b8ab0afcb1670d6aa517d29299ab1e5887ed322e3f848db85d10b240a5c5ef8529dc93e55ea3406a7b5a1521ef2b14d68f250c52e988fe86a675e98aa7b57
+  metadata.gz: c7d249497e4f035ac74de379de666244269adf9139b79169d0f096b78873a04601a04fca173afa73e2171ea0aeeb7f7609c2e82d3f52f67f04094a7991525b36
+  data.tar.gz: e32900947434fa3af0a04db69cfaafa34ef7832f1820d6cb7f0273ccd73655983e046bd56b5570c73b5948baa2a118520c85d79ace60fa691d5015af01a8d7dd

data/.gitignore

@@ -9,5 +9,9 @@
 
 # rspec failure tracking
 .rspec_status
+
 *.gem
+*.lock
+*.log
 .coveralls.yml
+.ruby-version

data/.travis.yml

@@ -1,17 +1,34 @@
 sudo: false
+
 language: ruby
+
 rvm:
-  - 2.3.3
-  - 2.4.2
-  - 2.5.0
-  - jruby
-before_install: gem install bundler -v 1.14.6
-env:
-  - SEQUEL_VERSION='~> 4.0'
-  - SEQUEL_VERSION='~> 5.0'
-gemfile:
-  - gemfiles/ci.gemfile
+  - 2.3
+  - 2.4
+  - 2.5
+  - 2.6
+  - jruby-9.2.8.0
+  - ruby-head
+  - jruby-head
+
+before_install: gem install bundler
+
+env: SEQUEL_VERSION="~> 5.0"
+
+gemfile: gemfiles/ci.gemfile
+
 addons:
   postgresql: "9.6"
+
 services:
   - postgresql
+
+matrix:
+  include:
+    - rvm: 2.6
+      env: SEQUEL_VERSION="~> 4.0"
+  allow_failures:
+    - rvm: ruby-head
+      env: SEQUEL_VERSION="~> 5.0"
+    - rvm: jruby-head
+      env: SEQUEL_VERSION="~> 5.0"

data/Gemfile

@@ -1,6 +1,14 @@
 source "https://rubygems.org"
 
-git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
-
 # Specify your gem's dependencies in sequel-batches.gemspec
 gemspec
+
+# MRI/Rubinius Adapter Dependencies
+platforms :ruby do
+  gem "pg"
+end
+
+# JRuby Adapter Dependencies
+platforms :jruby do
+  gem "jdbc-postgres"
+end

data/README.md

@@ -2,11 +2,23 @@
 
 This dataset extension provides the method #in_batches. The method splits dataset in parts and yields it.
 
-You can set following options:
-  - pk Overrides primary key of your dataset
-  - of sets chunk size (1000 by default)
-  - start as a hash { [column]: <start_value> } represents frame start for batch processing
-  - finish as a hash represents frame end
+Note: currently only PostgreSQL database is supported.
+
+You can set the following options:
+
+### pk
+Overrides primary key of your dataset. This option is required in case your table doesn't have a real PK, otherwise you will get `Sequel::Extensions::Batches::MissingPKError`.
+
+Note that you have to provide columns that don't contain NULL values, otherwise this may not work as intended. You will receive `Sequel::Extensions::Batches::NullPKError` in case batch processing detects a NULL value on it's way, but it's not guaranteed since it doesn't check all the rows for performance reasons.
+
+### of
+Sets chunk size (1000 by default).
+
+### start
+A hash `{ [column]: <start_value> }` that represents frame start for batch processing. Note that you will get `Sequel::Extensions::Batches::InvalidPKError` in case you provide a hash with wrong keys (ordering matters as well).
+
+### finish
+Same as `start` but represents the frame end.
 
 ## Installation
 

data/gemfiles/ci.gemfile

@@ -12,4 +12,4 @@ platforms :jruby do
   gem 'jdbc-postgres', '~> 9.4'
 end
 
-gem sequel, "#{ENV['SEQUEL_VERSION']}"
+gem "sequel", "#{ENV['SEQUEL_VERSION']}"

data/lib/sequel/extensions/batches/version.rb

@@ -1,7 +1,7 @@
 module Sequel
   module Extensions
     module Batches
-      VERSION = "0.1.3"
+      VERSION = "0.2.0"
     end
   end
 end

data/lib/sequel/extensions/batches.rb

@@ -5,63 +5,52 @@ module Sequel
   module Extensions
     module Batches
       MissingPKError = Class.new(StandardError)
+      NullPKError = Class.new(StandardError)
+      InvalidPKError = Class.new(StandardError)
 
-      def in_batches(pk: nil, of: 1000, start: {}, finish: {})
-        pk ||= self.db.schema(first_source)
-                 .select{|r| r[1][:primary_key]}
-                 .map(&:first) or raise MissingPKError
-        qualified_pk = pk.map { |c| Sequel[first_source][c] }
+      def in_batches(pk: nil, of: 1000, start: nil, finish: nil)
+        pk ||= db.schema(first_source).select { |x| x[1][:primary_key] }.map(&:first)
+        raise MissingPKError if pk.empty?
 
-        pk_expr = (-> (pk:) do
-          pk.map do |col|
-            colname = col.is_a?(Symbol) ? col : col.column
-            Sequel.as(
-              Sequel.pg_array(
-                [
-                  Sequel.function(:min, col),
-                  Sequel.function(:max, col)
-                ]
-              ), :"#{colname}"
-            )
-          end
-        end)
+        qualified_pk = pk.map { |x| Sequel[first_source][x] }
 
-        entire_min_max = self.order(*pk).select(*pk_expr.call(pk: qualified_pk)).first
-        min_max = {}
+        check_pk = lambda do |input_pk|
+          raise InvalidPKError if input_pk.keys != pk
+          input_pk
+        end
 
-        range_expr =  (-> (col, range) do
-          Sequel.&(
-            Sequel.expr(Sequel[first_source][col]) >= range[0],
-            Sequel.expr(Sequel[first_source][col]) <= range[1],
-          )
-        end)
+        conditions = lambda do |pk, sign:|
+          raise NullPKError if pk.values.any?(&:nil?)
+          row_expr = Sequel.function(:row, *pk.values)
+          Sequel.function(:row, *qualified_pk).public_send(sign, row_expr)
+        end
 
-        loop do
-          pk.each do |col|
-            entire_min_max[col][0] = start[col] || entire_min_max[col][0]
-            entire_min_max[col][1] = finish[col] || entire_min_max[col][1]
-          end
+        base_ds = order(*qualified_pk)
+        base_ds = base_ds.where(conditions.call(check_pk.call(start), sign: :>=)) if start
+        base_ds = base_ds.where(conditions.call(check_pk.call(finish), sign: :<=)) if finish
 
-          ds = self.order(*qualified_pk).limit(of).where(
-            Sequel.&(*pk.map { |col| range_expr.call(col, entire_min_max[col]) })
-          )
-          if min_max.present?
-            pk_combinations = pk.each_with_index.map { |x, i| pk[0..-i] }
-            ds = ds.where(Sequel.|(*pk_combinations.each_with_index.map do |pks, i|
-              Sequel.&(*pks.each_with_index.map do |col, j|
-                if j == i
-                  Sequel[first_source][col] > min_max[col].last
-                else
-                  Sequel[first_source][col] >= min_max[col].last
-                end
-              end)
-            end))
+        pk_ds = db.from(base_ds).select(*pk).order(*pk)
+        actual_start = pk_ds.first
+        actual_finish = pk_ds.last
+
+        return unless actual_start && actual_finish
+
+        base_ds = base_ds.where(conditions.call(actual_start, sign: :>=))
+        base_ds = base_ds.where(conditions.call(actual_finish, sign: :<=))
+
+        current_instance = nil
+
+        loop do
+          if current_instance
+            working_ds = base_ds.where(conditions.call(current_instance.to_h, sign: :>))
+          else
+            working_ds = base_ds
           end
 
-          min_max = self.db.from(ds).select(*pk_expr.call(pk: pk)).first
+          current_instance = db.from(working_ds.limit(of)).select(*pk).order(*pk).last or break
+          working_ds = working_ds.where(conditions.call(current_instance.to_h, sign: :<=))
 
-          break if min_max.values.flatten.any?(&:blank?)
-          yield self.where(Sequel.&(*pk.map { |col| range_expr.call(col, min_max[col]) }))
+          yield working_ds
         end
       end
 

data/sequel-batches.gemspec

@@ -30,10 +30,11 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "sequel", "~> 5.0"
-  spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "pry", "~> 0.10"
+  spec.add_development_dependency "bundler"
   spec.add_development_dependency "coveralls"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+
+  spec.add_runtime_dependency "sequel"
 end

metadata

@@ -1,94 +1,94 @@
 --- !ruby/object:Gem::Specification
 name: sequel-batches
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - fiscal-cliff
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-28 00:00:00.000000000 Z
+date: 2019-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
-  name: sequel
-  prerelease: false
+        version: '0'
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
+  name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
-  name: bundler
-  prerelease: false
+        version: '0'
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '0'
 - !ruby/object:Gem::Dependency
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
-  name: rake
-  prerelease: false
+        version: '0'
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
-  name: rspec
-  prerelease: false
+        version: '0'
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
-  name: pry
-  prerelease: false
+        version: '0'
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0'
 - !ruby/object:Gem::Dependency
+  name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: coveralls
+  type: :runtime
   prerelease: false
-  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -103,28 +103,26 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
-- ".ruby-version"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
 - gemfiles/ci.gemfile
-- gemfiles/ci.gemfile.lock
 - lib/sequel.rb
 - lib/sequel/extensions/batches.rb
 - lib/sequel/extensions/batches/version.rb
+- log/.keep
 - sequel-batches.gemspec
 homepage: https://github.com/fiscal-cliff
 licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -139,9 +137,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.14
-signing_key:
+rubygems_version: 3.0.3
+signing_key: 
 specification_version: 4
 summary: The extension mimics AR5 batches api
 test_files: []

data/.ruby-version

@@ -1 +0,0 @@
-jruby-9.1.15.0

data/Gemfile.lock

@@ -1,70 +0,0 @@
-PATH
-  remote: .
-  specs:
-    sequel-batches (0.1.2)
-      sequel (~> 5.0)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    coderay (1.1.2)
-    coveralls (0.8.21)
-      json (>= 1.8, < 3)
-      simplecov (~> 0.14.1)
-      term-ansicolor (~> 1.3)
-      thor (~> 0.19.4)
-      tins (~> 1.6)
-    diff-lcs (1.3)
-    docile (1.1.5)
-    ffi (1.9.23-java)
-    json (2.1.0)
-    json (2.1.0-java)
-    method_source (0.9.0)
-    pry (0.11.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    pry (0.11.3-java)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-      spoon (~> 0.0)
-    rake (10.5.0)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.1)
-    sequel (5.6.0)
-    simplecov (0.14.1)
-      docile (~> 1.1.0)
-      json (>= 1.8, < 3)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.2)
-    spoon (0.0.6)
-      ffi
-    term-ansicolor (1.6.0)
-      tins (~> 1.0)
-    thor (0.19.4)
-    tins (1.16.3)
-
-PLATFORMS
-  java
-  ruby
-
-DEPENDENCIES
-  bundler (~> 1.16)
-  coveralls
-  pry (~> 0.10)
-  rake (~> 10.0)
-  rspec (~> 3.0)
-  sequel-batches!
-
-BUNDLED WITH
-   1.16.1

data/gemfiles/ci.gemfile.lock

@@ -1,74 +0,0 @@
-PATH
-  remote: ..
-  specs:
-    sequel-batches (0.1.2)
-      sequel (~> 5.0)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    coderay (1.1.2)
-    coveralls (0.8.21)
-      json (>= 1.8, < 3)
-      simplecov (~> 0.14.1)
-      term-ansicolor (~> 1.3)
-      thor (~> 0.19.4)
-      tins (~> 1.6)
-    diff-lcs (1.3)
-    docile (1.1.5)
-    ffi (1.9.23-java)
-    jdbc-postgres (9.4.1212)
-    json (2.1.0)
-    json (2.1.0-java)
-    method_source (0.9.0)
-    pg (0.20.0)
-    pry (0.11.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    pry (0.11.3-java)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-      spoon (~> 0.0)
-    rake (10.5.0)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.1)
-    sequel (5.6.0)
-    simplecov (0.14.1)
-      docile (~> 1.1.0)
-      json (>= 1.8, < 3)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.2)
-    spoon (0.0.6)
-      ffi
-    term-ansicolor (1.6.0)
-      tins (~> 1.0)
-    thor (0.19.4)
-    tins (1.16.3)
-
-PLATFORMS
-  java
-  ruby
-
-DEPENDENCIES
-  bundler (~> 1.16)
-  coveralls
-  jdbc-postgres (~> 9.4)
-  pg (~> 0.20.0)
-  pry (~> 0.10)
-  rake (~> 10.0)
-  rspec (~> 3.0)
-  sequel-batches!
-
-BUNDLED WITH
-   1.16.1