sequel-batches 0.2.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +9 -0
- data/.travis.yml +0 -9
- data/Gemfile +2 -0
- data/README.md +17 -16
- data/Rakefile +5 -1
- data/lib/sequel/extensions/batches.rb +7 -50
- data/lib/sequel/extensions/batches/yielder.rb +82 -0
- data/sequel-batches.gemspec +15 -26
- metadata +35 -9
- data/gemfiles/ci.gemfile +0 -15
- data/lib/sequel.rb +0 -1
- data/lib/sequel/extensions/batches/version.rb +0 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e24dcc9f437a98a65709c2486c46e6e3ddbeb89a1d3298af245c4d20ae758492
|
4
|
+
data.tar.gz: 5d5670cc31a6afcf7cdee391dc889c9636e9a201fe1f68e01860df2174e462fd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 90f54e68d80fe0872415ab53b4d217c803eaf1d711e083bc518799475c805cd050d2a5605d42603b959fcde28ddfb5ef679344f46bd601677a31ab192bf4b4aa
|
7
|
+
data.tar.gz: a7f69e996cdedfec3b83160b15858fb090c970d2e33516fe53c2e64e1c2c5ed5d198241ea6836352cd1a3f260ebae8667b79bcd4847e6d08e4556c22a996a306
|
data/.rubocop.yml
ADDED
data/.travis.yml
CHANGED
@@ -13,10 +13,6 @@ rvm:
|
|
13
13
|
|
14
14
|
before_install: gem install bundler
|
15
15
|
|
16
|
-
env: SEQUEL_VERSION="~> 5.0"
|
17
|
-
|
18
|
-
gemfile: gemfiles/ci.gemfile
|
19
|
-
|
20
16
|
addons:
|
21
17
|
postgresql: "9.6"
|
22
18
|
|
@@ -24,11 +20,6 @@ services:
|
|
24
20
|
- postgresql
|
25
21
|
|
26
22
|
matrix:
|
27
|
-
include:
|
28
|
-
- rvm: 2.6
|
29
|
-
env: SEQUEL_VERSION="~> 4.0"
|
30
23
|
allow_failures:
|
31
24
|
- rvm: ruby-head
|
32
|
-
env: SEQUEL_VERSION="~> 5.0"
|
33
25
|
- rvm: jruby-head
|
34
|
-
env: SEQUEL_VERSION="~> 5.0"
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
# Sequel::Batches [](https://travis-ci.org/umbrellio/sequel-batches) [](https://coveralls.io/github/umbrellio/sequel-batches?branch=master)
|
1
|
+
# Sequel::Batches [](https://badge.fury.io/rb/sequel-batches) [](https://travis-ci.org/umbrellio/sequel-batches) [](https://coveralls.io/github/umbrellio/sequel-batches?branch=master)
|
2
2
|
|
3
|
-
This dataset extension provides the method
|
3
|
+
This dataset extension provides the `#in_batches` method. The method splits dataset in parts and yields it.
|
4
4
|
|
5
5
|
Note: currently only PostgreSQL database is supported.
|
6
6
|
|
@@ -38,13 +38,13 @@ Or install it yourself as:
|
|
38
38
|
|
39
39
|
## Usage
|
40
40
|
|
41
|
-
In order to use the feature you should enable the extension
|
41
|
+
In order to use the feature you should enable the extension:
|
42
42
|
|
43
43
|
```ruby
|
44
44
|
Sequel::DATABASES.first.extension :batches
|
45
45
|
```
|
46
46
|
|
47
|
-
|
47
|
+
After that the `#in_batches` method becomes available on dataset:
|
48
48
|
|
49
49
|
```ruby
|
50
50
|
User.where(role: "admin").in_batches(of: 4) do |ds|
|
@@ -52,28 +52,29 @@ User.where(role: "admin").in_batches(of: 4) do |ds|
|
|
52
52
|
end
|
53
53
|
```
|
54
54
|
|
55
|
-
Finally, here's an example including all the available options
|
55
|
+
Finally, here's an example including all the available options:
|
56
56
|
|
57
57
|
```ruby
|
58
|
-
|
58
|
+
options = {
|
59
|
+
of: 4,
|
60
|
+
pk: [:project_id, :external_user_id],
|
61
|
+
start: { project_id: 2, external_user_id: 3 },
|
62
|
+
finish: { project_id: 5, external_user_id: 70 },
|
63
|
+
}
|
64
|
+
|
65
|
+
Event.where(type: "login").in_batches(options) do |ds|
|
59
66
|
ds.delete
|
60
67
|
end
|
61
68
|
```
|
62
69
|
|
63
|
-
## Development
|
64
|
-
|
65
|
-
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
66
|
-
|
67
|
-
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
68
|
-
|
69
70
|
## Contributing
|
70
71
|
|
71
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/
|
72
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/umbrellio/sequel-batches.
|
72
73
|
|
73
74
|
## License
|
74
75
|
|
75
76
|
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
76
77
|
|
77
|
-
|
78
|
-
|
79
|
-
|
78
|
+
<a href="https://github.com/umbrellio/">
|
79
|
+
<img style="float: left;" src="https://umbrellio.github.io/Umbrellio/supported_by_umbrellio.svg" alt="Supported by Umbrellio" width="439" height="72">
|
80
|
+
</a>
|
data/Rakefile
CHANGED
@@ -1,5 +1,4 @@
|
|
1
|
-
|
2
|
-
require "sequel/model"
|
1
|
+
# frozen_string_literal: true
|
3
2
|
|
4
3
|
module Sequel
|
5
4
|
module Extensions
|
@@ -8,55 +7,13 @@ module Sequel
|
|
8
7
|
NullPKError = Class.new(StandardError)
|
9
8
|
InvalidPKError = Class.new(StandardError)
|
10
9
|
|
11
|
-
def in_batches(
|
12
|
-
|
13
|
-
raise MissingPKError if pk.empty?
|
14
|
-
|
15
|
-
qualified_pk = pk.map { |x| Sequel[first_source][x] }
|
16
|
-
|
17
|
-
check_pk = lambda do |input_pk|
|
18
|
-
raise InvalidPKError if input_pk.keys != pk
|
19
|
-
input_pk
|
20
|
-
end
|
21
|
-
|
22
|
-
conditions = lambda do |pk, sign:|
|
23
|
-
raise NullPKError if pk.values.any?(&:nil?)
|
24
|
-
row_expr = Sequel.function(:row, *pk.values)
|
25
|
-
Sequel.function(:row, *qualified_pk).public_send(sign, row_expr)
|
26
|
-
end
|
27
|
-
|
28
|
-
base_ds = order(*qualified_pk)
|
29
|
-
base_ds = base_ds.where(conditions.call(check_pk.call(start), sign: :>=)) if start
|
30
|
-
base_ds = base_ds.where(conditions.call(check_pk.call(finish), sign: :<=)) if finish
|
31
|
-
|
32
|
-
pk_ds = db.from(base_ds).select(*pk).order(*pk)
|
33
|
-
actual_start = pk_ds.first
|
34
|
-
actual_finish = pk_ds.last
|
35
|
-
|
36
|
-
return unless actual_start && actual_finish
|
37
|
-
|
38
|
-
base_ds = base_ds.where(conditions.call(actual_start, sign: :>=))
|
39
|
-
base_ds = base_ds.where(conditions.call(actual_finish, sign: :<=))
|
40
|
-
|
41
|
-
current_instance = nil
|
42
|
-
|
43
|
-
loop do
|
44
|
-
if current_instance
|
45
|
-
working_ds = base_ds.where(conditions.call(current_instance.to_h, sign: :>))
|
46
|
-
else
|
47
|
-
working_ds = base_ds
|
48
|
-
end
|
49
|
-
|
50
|
-
current_instance = db.from(working_ds.limit(of)).select(*pk).order(*pk).last or break
|
51
|
-
working_ds = working_ds.where(conditions.call(current_instance.to_h, sign: :<=))
|
52
|
-
|
53
|
-
yield working_ds
|
54
|
-
end
|
10
|
+
def in_batches(**options, &block)
|
11
|
+
Sequel::Extensions::Batches::Yielder.new(ds: self, **options).call(&block)
|
55
12
|
end
|
56
|
-
|
57
|
-
private
|
58
|
-
|
59
|
-
::Sequel::Dataset.register_extension(:batches, Batches)
|
60
13
|
end
|
61
14
|
end
|
62
15
|
end
|
16
|
+
|
17
|
+
::Sequel::Dataset.register_extension(:batches, Sequel::Extensions::Batches)
|
18
|
+
|
19
|
+
require_relative "batches/yielder"
|
@@ -0,0 +1,82 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Sequel::Extensions::Batches
|
4
|
+
class Yielder
|
5
|
+
attr_accessor :ds, :of, :start, :finish
|
6
|
+
attr_writer :pk
|
7
|
+
|
8
|
+
def initialize(ds:, pk: nil, of: 1000, start: nil, finish: nil)
|
9
|
+
self.ds = ds
|
10
|
+
self.pk = pk
|
11
|
+
self.of = of
|
12
|
+
self.start = start
|
13
|
+
self.finish = finish
|
14
|
+
end
|
15
|
+
|
16
|
+
def call
|
17
|
+
base_ds = setup_base_ds or return
|
18
|
+
|
19
|
+
current_instance = nil
|
20
|
+
|
21
|
+
loop do
|
22
|
+
working_ds =
|
23
|
+
if current_instance
|
24
|
+
base_ds.where(generate_conditions(current_instance.to_h, sign: :>))
|
25
|
+
else
|
26
|
+
base_ds
|
27
|
+
end
|
28
|
+
|
29
|
+
current_instance = db.from(working_ds.limit(of)).select(*pk).order(*pk).last or break
|
30
|
+
working_ds = working_ds.where(generate_conditions(current_instance.to_h, sign: :<=))
|
31
|
+
|
32
|
+
yield working_ds
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def db
|
39
|
+
ds.db
|
40
|
+
end
|
41
|
+
|
42
|
+
def pk
|
43
|
+
@pk ||= begin
|
44
|
+
pk = db.schema(ds.first_source).select { |x| x[1][:primary_key] }.map(&:first)
|
45
|
+
raise MissingPKError if pk.empty?
|
46
|
+
pk
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def qualified_pk
|
51
|
+
@qualified_pk ||= pk.map { |x| Sequel[ds.first_source][x] }
|
52
|
+
end
|
53
|
+
|
54
|
+
def check_pk(input_pk)
|
55
|
+
raise InvalidPKError if input_pk.keys != pk
|
56
|
+
input_pk
|
57
|
+
end
|
58
|
+
|
59
|
+
def generate_conditions(input_pk, sign:)
|
60
|
+
raise NullPKError if input_pk.values.any?(&:nil?)
|
61
|
+
row_expr = Sequel.function(:row, *input_pk.values)
|
62
|
+
Sequel.function(:row, *qualified_pk).public_send(sign, row_expr)
|
63
|
+
end
|
64
|
+
|
65
|
+
def setup_base_ds
|
66
|
+
base_ds = ds.order(*qualified_pk)
|
67
|
+
base_ds = base_ds.where(generate_conditions(check_pk(start), sign: :>=)) if start
|
68
|
+
base_ds = base_ds.where(generate_conditions(check_pk(finish), sign: :<=)) if finish
|
69
|
+
|
70
|
+
pk_ds = db.from(base_ds).select(*pk).order(*pk)
|
71
|
+
actual_start = pk_ds.first
|
72
|
+
actual_finish = pk_ds.last
|
73
|
+
|
74
|
+
return unless actual_start && actual_finish
|
75
|
+
|
76
|
+
base_ds = base_ds.where(generate_conditions(actual_start, sign: :>=))
|
77
|
+
base_ds = base_ds.where(generate_conditions(actual_finish, sign: :<=))
|
78
|
+
|
79
|
+
base_ds
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
data/sequel-batches.gemspec
CHANGED
@@ -1,40 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
|
2
|
-
lib = File.expand_path("
|
3
|
+
lib = File.expand_path("lib", __dir__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require "sequel/extensions/batches/version"
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
|
-
spec.name
|
8
|
-
spec.version
|
9
|
-
spec.authors
|
10
|
-
spec.email
|
7
|
+
spec.name = "sequel-batches"
|
8
|
+
spec.version = "1.0.0"
|
9
|
+
spec.authors = ["fiscal-cliff", "umbrellio"]
|
10
|
+
spec.email = ["oss@umbrellio.biz"]
|
11
11
|
|
12
|
-
spec.summary
|
13
|
-
spec.description
|
14
|
-
spec.homepage
|
15
|
-
spec.license
|
12
|
+
spec.summary = "The extension mimics AR5 batches api"
|
13
|
+
spec.description = "Allows you to split your dataset in batches"
|
14
|
+
spec.homepage = "https://github.com/umbrellio/sequel-batches"
|
15
|
+
spec.license = "MIT"
|
16
16
|
|
17
|
-
|
18
|
-
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
19
|
-
if spec.respond_to?(:metadata)
|
20
|
-
spec.metadata["allowed_push_host"] = "https://rubygems.org"
|
21
|
-
else
|
22
|
-
raise "RubyGems 2.0 or newer is required to protect against " \
|
23
|
-
"public gem pushes."
|
24
|
-
end
|
25
|
-
|
26
|
-
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
27
|
-
f.match(%r{^(test|spec|features)/})
|
28
|
-
end
|
29
|
-
spec.bindir = "exe"
|
30
|
-
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
17
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
31
18
|
spec.require_paths = ["lib"]
|
32
19
|
|
20
|
+
spec.add_runtime_dependency "sequel"
|
21
|
+
|
33
22
|
spec.add_development_dependency "bundler"
|
34
23
|
spec.add_development_dependency "coveralls"
|
35
24
|
spec.add_development_dependency "pry"
|
36
25
|
spec.add_development_dependency "rake"
|
37
26
|
spec.add_development_dependency "rspec"
|
38
|
-
|
39
|
-
spec.
|
27
|
+
spec.add_development_dependency "rubocop-config-umbrellio"
|
28
|
+
spec.add_development_dependency "simplecov"
|
40
29
|
end
|
metadata
CHANGED
@@ -1,16 +1,30 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sequel-batches
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- fiscal-cliff
|
8
8
|
- umbrellio
|
9
9
|
autorequire:
|
10
|
-
bindir:
|
10
|
+
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
date: 2019-08-19 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: sequel
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - ">="
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '0'
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '0'
|
14
28
|
- !ruby/object:Gem::Dependency
|
15
29
|
name: bundler
|
16
30
|
requirement: !ruby/object:Gem::Requirement
|
@@ -82,13 +96,27 @@ dependencies:
|
|
82
96
|
- !ruby/object:Gem::Version
|
83
97
|
version: '0'
|
84
98
|
- !ruby/object:Gem::Dependency
|
85
|
-
name:
|
99
|
+
name: rubocop-config-umbrellio
|
86
100
|
requirement: !ruby/object:Gem::Requirement
|
87
101
|
requirements:
|
88
102
|
- - ">="
|
89
103
|
- !ruby/object:Gem::Version
|
90
104
|
version: '0'
|
91
|
-
type: :
|
105
|
+
type: :development
|
106
|
+
prerelease: false
|
107
|
+
version_requirements: !ruby/object:Gem::Requirement
|
108
|
+
requirements:
|
109
|
+
- - ">="
|
110
|
+
- !ruby/object:Gem::Version
|
111
|
+
version: '0'
|
112
|
+
- !ruby/object:Gem::Dependency
|
113
|
+
name: simplecov
|
114
|
+
requirement: !ruby/object:Gem::Requirement
|
115
|
+
requirements:
|
116
|
+
- - ">="
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: '0'
|
119
|
+
type: :development
|
92
120
|
prerelease: false
|
93
121
|
version_requirements: !ruby/object:Gem::Requirement
|
94
122
|
requirements:
|
@@ -104,23 +132,21 @@ extra_rdoc_files: []
|
|
104
132
|
files:
|
105
133
|
- ".gitignore"
|
106
134
|
- ".rspec"
|
135
|
+
- ".rubocop.yml"
|
107
136
|
- ".travis.yml"
|
108
137
|
- CODE_OF_CONDUCT.md
|
109
138
|
- Gemfile
|
110
139
|
- LICENSE.txt
|
111
140
|
- README.md
|
112
141
|
- Rakefile
|
113
|
-
- gemfiles/ci.gemfile
|
114
|
-
- lib/sequel.rb
|
115
142
|
- lib/sequel/extensions/batches.rb
|
116
|
-
- lib/sequel/extensions/batches/
|
143
|
+
- lib/sequel/extensions/batches/yielder.rb
|
117
144
|
- log/.keep
|
118
145
|
- sequel-batches.gemspec
|
119
146
|
homepage: https://github.com/umbrellio/sequel-batches
|
120
147
|
licenses:
|
121
148
|
- MIT
|
122
|
-
metadata:
|
123
|
-
allowed_push_host: https://rubygems.org
|
149
|
+
metadata: {}
|
124
150
|
post_install_message:
|
125
151
|
rdoc_options: []
|
126
152
|
require_paths:
|
data/gemfiles/ci.gemfile
DELETED
@@ -1,15 +0,0 @@
|
|
1
|
-
source 'https://rubygems.org'
|
2
|
-
|
3
|
-
gemspec :path => '../'
|
4
|
-
|
5
|
-
# MRI/Rubinius Adapter Dependencies
|
6
|
-
platforms :ruby do
|
7
|
-
gem "pg", "~>0.20.0"
|
8
|
-
end
|
9
|
-
|
10
|
-
# JRuby Adapter Dependencies
|
11
|
-
platforms :jruby do
|
12
|
-
gem 'jdbc-postgres', '~> 9.4'
|
13
|
-
end
|
14
|
-
|
15
|
-
gem "sequel", "#{ENV['SEQUEL_VERSION']}"
|
data/lib/sequel.rb
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
require "sequel/extensions/batches"
|