sequel-batches 1.0.1 → 2.0.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 75f26b2a799694f8ab7c27d9b2e9fdbf12990d92c3c9a5ba2117ec9c7815cb47
4
- data.tar.gz: be0eb9ffc65170761b2000c20c2c8e628c8eacc2cfbf1cee7d4fbf54c0134760
3
+ metadata.gz: 0d8af97c53f8431d8fc6318af40c5f42e00ad83f177dc663fbdd5c0a18f7c90d
4
+ data.tar.gz: 1f5d637fff098d001694c62a4e397801a5077217ba2174533a8c87975edc8fb7
5
5
  SHA512:
6
- metadata.gz: 874d85e620b0d5cbe2a82700deff251edf004bca67a64e89ece8fca769abcbc5f117891d02767f0873634156de7680558cfeceee0abd917f7a762c8f5720c42c
7
- data.tar.gz: 39ca7cd7e363ab3fe6d8d04ef7c73f140369f3e0e6a87c8d79605d6c80882c2bcbc0fb625ee8dccc8d297f20d5516c031b126c82cbcc38f3d291474ccbe226be
6
+ metadata.gz: f26a92d8573d15f3a264d606cce33eaeab69bdb0a791d9b98b4dc9cb8cfad8c4abddf0b04bd5ff9032fe80042da81cc81d1b8759b662c4459567d6720f2bfb7c
7
+ data.tar.gz: 237d285d2febaa6f896f4e7d727a0a36884c6889c348ef68815999ee10c766390f490888adf62eda75f2a41789e93bddd0dcb205279003eba46db6dba570ff42
@@ -0,0 +1,50 @@
1
+ name: CI
2
+
3
+ on: [pull_request, push]
4
+
5
+ jobs:
6
+ test:
7
+ runs-on: ubuntu-latest
8
+
9
+ if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != 'umbrellio/sequel-batches'
10
+
11
+ services:
12
+ postgres:
13
+ image: postgres
14
+ env:
15
+ POSTGRES_USER: root
16
+ POSTGRES_HOST_AUTH_METHOD: trust
17
+ options: >-
18
+ --health-cmd pg_isready
19
+ --health-interval 10s
20
+ --health-timeout 5s
21
+ --health-retries 5
22
+ ports:
23
+ - 5432:5432
24
+
25
+ env:
26
+ PGHOST: localhost
27
+ PGUSER: root
28
+
29
+ strategy:
30
+ fail-fast: false
31
+ matrix:
32
+ ruby: ["2.7", "3.0", "3.1"]
33
+
34
+ name: ${{ matrix.ruby }}
35
+
36
+ steps:
37
+ - uses: actions/checkout@v2
38
+ - uses: ruby/setup-ruby@v1
39
+ with:
40
+ ruby-version: ${{ matrix.ruby }}
41
+ bundler-cache: true
42
+
43
+ - run: psql -c 'CREATE DATABASE batches_test'
44
+ - run: bundle exec rake bundle:audit
45
+ - run: bundle exec rake lint
46
+ - run: bundle exec rspec
47
+
48
+ - uses: coverallsapp/github-action@v1.1.2
49
+ with:
50
+ github-token: ${{ secrets.GITHUB_TOKEN }}
data/.gitignore CHANGED
@@ -11,7 +11,6 @@
11
11
  .rspec_status
12
12
 
13
13
  *.gem
14
- *.lock
15
14
  *.log
16
15
  .coveralls.yml
17
16
  .ruby-version
data/.rubocop.yml CHANGED
@@ -3,7 +3,16 @@ inherit_gem:
3
3
 
4
4
  AllCops:
5
5
  DisplayCopNames: true
6
- TargetRubyVersion: 2.3
6
+ TargetRubyVersion: 2.7
7
+ Include:
8
+ - lib/**/*.rb
9
+ - spec/**/*.rb
10
+ - Gemfile
11
+ - Rakefile
12
+ - sequel-batches.gemspec
13
+
14
+ Layout/LineLength:
15
+ Max: 120
7
16
 
8
17
  Naming/MethodParameterName:
9
18
  Enabled: false
data/Gemfile CHANGED
@@ -4,13 +4,3 @@ source "https://rubygems.org"
4
4
 
5
5
  # Specify your gem's dependencies in sequel-batches.gemspec
6
6
  gemspec
7
-
8
- # MRI/Rubinius Adapter Dependencies
9
- platforms :ruby do
10
- gem "pg"
11
- end
12
-
13
- # JRuby Adapter Dependencies
14
- platforms :jruby do
15
- gem "jdbc-postgres"
16
- end
data/Gemfile.lock ADDED
@@ -0,0 +1,113 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ sequel-batches (2.0.1)
5
+ sequel
6
+
7
+ GEM
8
+ remote: https://rubygems.org/
9
+ specs:
10
+ activesupport (7.0.3.1)
11
+ concurrent-ruby (~> 1.0, >= 1.0.2)
12
+ i18n (>= 1.6, < 2)
13
+ minitest (>= 5.1)
14
+ tzinfo (~> 2.0)
15
+ ast (2.4.2)
16
+ bundler-audit (0.9.1)
17
+ bundler (>= 1.2.0, < 3)
18
+ thor (~> 1.0)
19
+ coderay (1.1.3)
20
+ concurrent-ruby (1.1.10)
21
+ diff-lcs (1.5.0)
22
+ docile (1.4.0)
23
+ i18n (1.12.0)
24
+ concurrent-ruby (~> 1.0)
25
+ method_source (1.0.0)
26
+ minitest (5.16.2)
27
+ parallel (1.22.1)
28
+ parser (3.1.2.1)
29
+ ast (~> 2.4.1)
30
+ pg (1.4.2)
31
+ pry (0.14.1)
32
+ coderay (~> 1.1)
33
+ method_source (~> 1.0)
34
+ rack (2.2.4)
35
+ rainbow (3.1.1)
36
+ rake (13.0.6)
37
+ regexp_parser (2.5.0)
38
+ rexml (3.2.5)
39
+ rspec (3.11.0)
40
+ rspec-core (~> 3.11.0)
41
+ rspec-expectations (~> 3.11.0)
42
+ rspec-mocks (~> 3.11.0)
43
+ rspec-core (3.11.0)
44
+ rspec-support (~> 3.11.0)
45
+ rspec-expectations (3.11.0)
46
+ diff-lcs (>= 1.2.0, < 2.0)
47
+ rspec-support (~> 3.11.0)
48
+ rspec-mocks (3.11.1)
49
+ diff-lcs (>= 1.2.0, < 2.0)
50
+ rspec-support (~> 3.11.0)
51
+ rspec-support (3.11.0)
52
+ rubocop (1.30.1)
53
+ parallel (~> 1.10)
54
+ parser (>= 3.1.0.0)
55
+ rainbow (>= 2.2.2, < 4.0)
56
+ regexp_parser (>= 1.8, < 3.0)
57
+ rexml (>= 3.2.5, < 4.0)
58
+ rubocop-ast (>= 1.18.0, < 2.0)
59
+ ruby-progressbar (~> 1.7)
60
+ unicode-display_width (>= 1.4.0, < 3.0)
61
+ rubocop-ast (1.21.0)
62
+ parser (>= 3.1.1.0)
63
+ rubocop-config-umbrellio (1.30.0.65)
64
+ rubocop (~> 1.30.0)
65
+ rubocop-performance (~> 1.14.0)
66
+ rubocop-rails (~> 2.14.2)
67
+ rubocop-rake (~> 0.6.0)
68
+ rubocop-rspec (~> 2.11.1)
69
+ rubocop-sequel (~> 0.3.3)
70
+ rubocop-performance (1.14.3)
71
+ rubocop (>= 1.7.0, < 2.0)
72
+ rubocop-ast (>= 0.4.0)
73
+ rubocop-rails (2.14.2)
74
+ activesupport (>= 4.2.0)
75
+ rack (>= 1.1)
76
+ rubocop (>= 1.7.0, < 2.0)
77
+ rubocop-rake (0.6.0)
78
+ rubocop (~> 1.0)
79
+ rubocop-rspec (2.11.1)
80
+ rubocop (~> 1.19)
81
+ rubocop-sequel (0.3.4)
82
+ rubocop (~> 1.0)
83
+ ruby-progressbar (1.11.0)
84
+ sequel (5.59.0)
85
+ simplecov (0.21.2)
86
+ docile (~> 1.1)
87
+ simplecov-html (~> 0.11)
88
+ simplecov_json_formatter (~> 0.1)
89
+ simplecov-html (0.12.3)
90
+ simplecov-lcov (0.8.0)
91
+ simplecov_json_formatter (0.1.4)
92
+ thor (1.2.1)
93
+ tzinfo (2.0.5)
94
+ concurrent-ruby (~> 1.0)
95
+ unicode-display_width (2.2.0)
96
+
97
+ PLATFORMS
98
+ ruby
99
+
100
+ DEPENDENCIES
101
+ bundler
102
+ bundler-audit
103
+ pg
104
+ pry
105
+ rake
106
+ rspec
107
+ rubocop-config-umbrellio
108
+ sequel-batches!
109
+ simplecov
110
+ simplecov-lcov
111
+
112
+ BUNDLED WITH
113
+ 2.3.19
data/README.md CHANGED
@@ -12,14 +12,6 @@ Add this line to your application's Gemfile:
12
12
  gem 'sequel-batches'
13
13
  ```
14
14
 
15
- And then execute:
16
-
17
- $ bundle
18
-
19
- Or install it yourself as:
20
-
21
- $ gem install sequel-batches
22
-
23
15
  ## Usage
24
16
 
25
17
  In order to use the feature you should enable the extension:
@@ -44,6 +36,7 @@ options = {
44
36
  pk: [:project_id, :external_user_id],
45
37
  start: { project_id: 2, external_user_id: 3 },
46
38
  finish: { project_id: 5, external_user_id: 70 },
39
+ order: :desc,
47
40
  }
48
41
 
49
42
  Event.where(type: "login").in_batches(options) do |ds|
@@ -56,19 +49,27 @@ end
56
49
  You can set the following options:
57
50
 
58
51
  ### pk
52
+
59
53
  Overrides primary key of your dataset. This option is required in case your table doesn't have a real PK, otherwise you will get `Sequel::Extensions::Batches::MissingPKError`.
60
54
 
61
55
  Note that you have to provide columns that don't contain NULL values, otherwise this may not work as intended. You will receive `Sequel::Extensions::Batches::NullPKError` in case batch processing detects a NULL value on it's way, but it's not guaranteed since it doesn't check all the rows for performance reasons.
62
56
 
63
57
  ### of
58
+
64
59
  Sets chunk size (1000 by default).
65
60
 
66
61
  ### start
62
+
67
63
  A hash `{ [column]: <start_value> }` that represents frame start for batch processing. Note that you will get `Sequel::Extensions::Batches::InvalidPKError` in case you provide a hash with wrong keys (ordering matters as well).
68
64
 
69
65
  ### finish
66
+
70
67
  Same as `start` but represents the frame end.
71
68
 
69
+ ### order
70
+
71
+ Specifies the primary key order (can be `:asc` or `:desc`). Defaults to `:asc`.
72
+
72
73
  ## Contributing
73
74
 
74
75
  Bug reports and pull requests are welcome on GitHub at https://github.com/umbrellio/sequel-batches.
data/Rakefile CHANGED
@@ -1,10 +1,18 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "bundler/gem_tasks"
4
+ require "bundler/audit/task"
4
5
  require "rspec/core/rake_task"
5
6
  require "rubocop/rake_task"
6
7
 
7
8
  RSpec::Core::RakeTask.new(:spec)
8
- RuboCop::RakeTask.new(:lint)
9
+ RuboCop::RakeTask.new(:lint) do |t|
10
+ config_path = File.expand_path(File.join(".rubocop.yml"), __dir__)
11
+
12
+ t.options = ["--config", config_path]
13
+ t.requires << "rubocop-rspec"
14
+ t.requires << "rubocop-performance"
15
+ end
16
+ Bundler::Audit::Task.new
9
17
 
10
18
  task default: %i[lint spec]
@@ -2,33 +2,38 @@
2
2
 
3
3
  module Sequel::Extensions::Batches
4
4
  class Yielder
5
- attr_accessor :ds, :of, :start, :finish
5
+ attr_accessor :ds, :of, :start, :finish, :order
6
6
  attr_writer :pk
7
7
 
8
- def initialize(ds:, pk: nil, of: 1000, start: nil, finish: nil)
8
+ def initialize(ds:, **options)
9
9
  self.ds = ds
10
- self.pk = pk
11
- self.of = of
12
- self.start = start
13
- self.finish = finish
10
+ self.pk = options.delete(:pk)
11
+ self.of = options.delete(:of) || 1000
12
+ self.start = options.delete(:start)
13
+ self.finish = options.delete(:finish)
14
+ self.order = options.delete(:order) || :asc
15
+
16
+ raise ArgumentError, ":order must be :asc or :desc, got #{order.inspect}" unless %i[asc desc].include?(order)
17
+ raise ArgumentError, "unknown options: #{options.keys.inspect}" if options.any?
14
18
  end
15
19
 
16
20
  def call
17
21
  base_ds = setup_base_ds or return
22
+ return enum_for(:call) unless block_given?
18
23
 
19
24
  current_instance = nil
20
25
 
21
26
  loop do
22
27
  working_ds =
23
28
  if current_instance
24
- base_ds.where(generate_conditions(current_instance.to_h, sign: :>))
29
+ base_ds.where(generate_conditions(current_instance.to_h, sign: sign_from_exclusive))
25
30
  else
26
31
  base_ds
27
32
  end
28
33
 
29
- working_ds_pk = working_ds.select(*qualified_pk).limit(of)
30
- current_instance = db.from(working_ds_pk).select(*pk).order(*pk).last or break
31
- working_ds = working_ds.where(generate_conditions(current_instance.to_h, sign: :<=))
34
+ working_ds_pk = working_ds.select(*qualified_pk).order(order_by(qualified: true)).limit(of)
35
+ current_instance = db.from(working_ds_pk).select(*pk).order(order_by).last or break
36
+ working_ds = working_ds.where(generate_conditions(current_instance.to_h, sign: sign_to_inclusive))
32
37
 
33
38
  yield working_ds
34
39
  end
@@ -48,6 +53,27 @@ module Sequel::Extensions::Batches
48
53
  end
49
54
  end
50
55
 
56
+ def asc_order?
57
+ order == :asc
58
+ end
59
+
60
+ def sign_from_exclusive
61
+ asc_order? ? :> : :<
62
+ end
63
+
64
+ def sign_from_inclusive
65
+ asc_order? ? :>= : :<=
66
+ end
67
+
68
+ def sign_to_inclusive
69
+ asc_order? ? :<= : :>=
70
+ end
71
+
72
+ def order_by(qualified: false)
73
+ columns = qualified ? qualified_pk : pk
74
+ asc_order? ? Sequel.asc(columns) : Sequel.desc(columns)
75
+ end
76
+
51
77
  def qualified_pk
52
78
  @qualified_pk ||= pk.map { |x| Sequel[ds.first_source][x] }
53
79
  end
@@ -64,20 +90,18 @@ module Sequel::Extensions::Batches
64
90
  end
65
91
 
66
92
  def setup_base_ds
67
- base_ds = ds.order(*qualified_pk)
68
- base_ds = base_ds.where(generate_conditions(check_pk(start), sign: :>=)) if start
69
- base_ds = base_ds.where(generate_conditions(check_pk(finish), sign: :<=)) if finish
93
+ base_ds = ds.order(order_by(qualified: true))
94
+ base_ds = base_ds.where(generate_conditions(check_pk(start), sign: sign_from_inclusive)) if start
95
+ base_ds = base_ds.where(generate_conditions(check_pk(finish), sign: sign_to_inclusive)) if finish
70
96
 
71
- pk_ds = db.from(base_ds.select(*qualified_pk)).select(*pk).order(*pk)
97
+ pk_ds = db.from(base_ds.select(*qualified_pk)).select(*pk).order(order_by)
72
98
  actual_start = pk_ds.first
73
99
  actual_finish = pk_ds.last
74
100
 
75
101
  return unless actual_start && actual_finish
76
102
 
77
- base_ds = base_ds.where(generate_conditions(actual_start, sign: :>=))
78
- base_ds = base_ds.where(generate_conditions(actual_finish, sign: :<=))
79
-
80
- base_ds
103
+ base_ds = base_ds.where(generate_conditions(actual_start, sign: sign_from_inclusive))
104
+ base_ds.where(generate_conditions(actual_finish, sign: sign_to_inclusive))
81
105
  end
82
106
  end
83
107
  end
@@ -5,9 +5,10 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
5
 
6
6
  Gem::Specification.new do |spec|
7
7
  spec.name = "sequel-batches"
8
- spec.version = "1.0.1"
8
+ spec.version = "2.0.1"
9
9
  spec.authors = %w[fiscal-cliff umbrellio]
10
10
  spec.email = ["oss@umbrellio.biz"]
11
+ spec.required_ruby_version = ">= 2.7"
11
12
 
12
13
  spec.summary = "The extension mimics AR5 batches api"
13
14
  spec.description = "Allows you to split your dataset in batches"
@@ -20,10 +21,12 @@ Gem::Specification.new do |spec|
20
21
  spec.add_runtime_dependency "sequel"
21
22
 
22
23
  spec.add_development_dependency "bundler"
23
- spec.add_development_dependency "coveralls"
24
+ spec.add_development_dependency "bundler-audit"
25
+ spec.add_development_dependency "pg"
24
26
  spec.add_development_dependency "pry"
25
27
  spec.add_development_dependency "rake"
26
28
  spec.add_development_dependency "rspec"
27
29
  spec.add_development_dependency "rubocop-config-umbrellio"
28
30
  spec.add_development_dependency "simplecov"
31
+ spec.add_development_dependency "simplecov-lcov"
29
32
  end
metadata CHANGED
@@ -1,15 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sequel-batches
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.1
4
+ version: 2.0.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - fiscal-cliff
8
8
  - umbrellio
9
- autorequire:
9
+ autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2020-05-25 00:00:00.000000000 Z
12
+ date: 2022-08-09 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: sequel
@@ -40,7 +40,21 @@ dependencies:
40
40
  - !ruby/object:Gem::Version
41
41
  version: '0'
42
42
  - !ruby/object:Gem::Dependency
43
- name: coveralls
43
+ name: bundler-audit
44
+ requirement: !ruby/object:Gem::Requirement
45
+ requirements:
46
+ - - ">="
47
+ - !ruby/object:Gem::Version
48
+ version: '0'
49
+ type: :development
50
+ prerelease: false
51
+ version_requirements: !ruby/object:Gem::Requirement
52
+ requirements:
53
+ - - ">="
54
+ - !ruby/object:Gem::Version
55
+ version: '0'
56
+ - !ruby/object:Gem::Dependency
57
+ name: pg
44
58
  requirement: !ruby/object:Gem::Requirement
45
59
  requirements:
46
60
  - - ">="
@@ -123,6 +137,20 @@ dependencies:
123
137
  - - ">="
124
138
  - !ruby/object:Gem::Version
125
139
  version: '0'
140
+ - !ruby/object:Gem::Dependency
141
+ name: simplecov-lcov
142
+ requirement: !ruby/object:Gem::Requirement
143
+ requirements:
144
+ - - ">="
145
+ - !ruby/object:Gem::Version
146
+ version: '0'
147
+ type: :development
148
+ prerelease: false
149
+ version_requirements: !ruby/object:Gem::Requirement
150
+ requirements:
151
+ - - ">="
152
+ - !ruby/object:Gem::Version
153
+ version: '0'
126
154
  description: Allows you to split your dataset in batches
127
155
  email:
128
156
  - oss@umbrellio.biz
@@ -130,12 +158,13 @@ executables: []
130
158
  extensions: []
131
159
  extra_rdoc_files: []
132
160
  files:
161
+ - ".github/workflows/ci.yml"
133
162
  - ".gitignore"
134
163
  - ".rspec"
135
164
  - ".rubocop.yml"
136
- - ".travis.yml"
137
165
  - CODE_OF_CONDUCT.md
138
166
  - Gemfile
167
+ - Gemfile.lock
139
168
  - LICENSE.txt
140
169
  - README.md
141
170
  - Rakefile
@@ -147,7 +176,7 @@ homepage: https://github.com/umbrellio/sequel-batches
147
176
  licenses:
148
177
  - MIT
149
178
  metadata: {}
150
- post_install_message:
179
+ post_install_message:
151
180
  rdoc_options: []
152
181
  require_paths:
153
182
  - lib
@@ -155,15 +184,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
155
184
  requirements:
156
185
  - - ">="
157
186
  - !ruby/object:Gem::Version
158
- version: '0'
187
+ version: '2.7'
159
188
  required_rubygems_version: !ruby/object:Gem::Requirement
160
189
  requirements:
161
190
  - - ">="
162
191
  - !ruby/object:Gem::Version
163
192
  version: '0'
164
193
  requirements: []
165
- rubygems_version: 3.1.2
166
- signing_key:
194
+ rubygems_version: 3.3.19
195
+ signing_key:
167
196
  specification_version: 4
168
197
  summary: The extension mimics AR5 batches api
169
198
  test_files: []
data/.travis.yml DELETED
@@ -1,25 +0,0 @@
1
- sudo: false
2
-
3
- language: ruby
4
-
5
- rvm:
6
- - 2.3
7
- - 2.4
8
- - 2.5
9
- - 2.6
10
- - jruby-9.2.8.0
11
- - ruby-head
12
- - jruby-head
13
-
14
- before_install: gem install bundler
15
-
16
- addons:
17
- postgresql: "9.6"
18
-
19
- services:
20
- - postgresql
21
-
22
- matrix:
23
- allow_failures:
24
- - rvm: ruby-head
25
- - rvm: jruby-head