wor-batchifier 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +327 -0
- data/.rubocop.yml +25 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +83 -0
- data/README.md +134 -0
- data/Rakefile +6 -0
- data/lib/wor/batchifier.rb +37 -0
- data/lib/wor/batchifier/add.rb +9 -0
- data/lib/wor/batchifier/array_merge.rb +10 -0
- data/lib/wor/batchifier/exceptions.rb +10 -0
- data/lib/wor/batchifier/interface.rb +23 -0
- data/lib/wor/batchifier/maintain_unique.rb +9 -0
- data/lib/wor/batchifier/no_response.rb +9 -0
- data/lib/wor/batchifier/strategy.rb +28 -0
- data/lib/wor/batchifier/version.rb +5 -0
- data/wor-batchifier.gemspec +33 -0
- metadata +174 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: e398fb06ca5eb88125a6cb9e7b7f896a9a5b6c45
|
4
|
+
data.tar.gz: 6bd86f8c0458bd392044b56c22bb5463f610532b
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: bbb2127cd0d7c67c2ce3c60b6d7cf956aa5999eb21f1e19eb9bb94010bf5b1d2f36a618ca03145c0776ab2e3a7ff3b8694cdd68ac8b3a7e0003491e4fc874feb
|
7
|
+
data.tar.gz: 1a4469303461a0eadd8a9b06f1b7df297c244eee7c26b70720a8bbc25eda981dcd0420da7e4a4ff29599329c2544959ab9b04a72595355f6ce5002f1a7128ce3
|
data/.gitignore
ADDED
@@ -0,0 +1,327 @@
|
|
1
|
+
/.bundle/
|
2
|
+
/.yardoc
|
3
|
+
/Gemfile.lock
|
4
|
+
/_yardoc/
|
5
|
+
/coverage/
|
6
|
+
/doc/
|
7
|
+
/pkg/
|
8
|
+
/spec/reports/
|
9
|
+
/tmp/
|
10
|
+
*.gem
|
11
|
+
|
12
|
+
|
13
|
+
# Created by https://www.gitignore.io/api/ruby,rubymine,rails,emacs,vim,sublimetext,osx,macos,linux,windows
|
14
|
+
|
15
|
+
### Emacs ###
|
16
|
+
# -*- mode: gitignore; -*-
|
17
|
+
*~
|
18
|
+
\#*\#
|
19
|
+
/.emacs.desktop
|
20
|
+
/.emacs.desktop.lock
|
21
|
+
*.elc
|
22
|
+
auto-save-list
|
23
|
+
tramp
|
24
|
+
.\#*
|
25
|
+
|
26
|
+
# Org-mode
|
27
|
+
.org-id-locations
|
28
|
+
*_archive
|
29
|
+
|
30
|
+
# flymake-mode
|
31
|
+
*_flymake.*
|
32
|
+
|
33
|
+
# eshell files
|
34
|
+
/eshell/history
|
35
|
+
/eshell/lastdir
|
36
|
+
|
37
|
+
# elpa packages
|
38
|
+
/elpa/
|
39
|
+
|
40
|
+
# reftex files
|
41
|
+
*.rel
|
42
|
+
|
43
|
+
# AUCTeX auto folder
|
44
|
+
/auto/
|
45
|
+
|
46
|
+
# cask packages
|
47
|
+
.cask/
|
48
|
+
dist/
|
49
|
+
|
50
|
+
# Flycheck
|
51
|
+
flycheck_*.el
|
52
|
+
|
53
|
+
# server auth directory
|
54
|
+
/server/
|
55
|
+
|
56
|
+
# projectiles files
|
57
|
+
.projectile
|
58
|
+
|
59
|
+
# directory configuration
|
60
|
+
.dir-locals.el
|
61
|
+
|
62
|
+
### Linux ###
|
63
|
+
|
64
|
+
# temporary files which can be created if a process still has a handle open of a deleted file
|
65
|
+
.fuse_hidden*
|
66
|
+
|
67
|
+
# KDE directory preferences
|
68
|
+
.directory
|
69
|
+
|
70
|
+
# Linux trash folder which might appear on any partition or disk
|
71
|
+
.Trash-*
|
72
|
+
|
73
|
+
# .nfs files are created when an open file is removed but is still being accessed
|
74
|
+
.nfs*
|
75
|
+
|
76
|
+
### macOS ###
|
77
|
+
*.DS_Store
|
78
|
+
.AppleDouble
|
79
|
+
.LSOverride
|
80
|
+
|
81
|
+
# Icon must end with two \r
|
82
|
+
Icon
|
83
|
+
|
84
|
+
|
85
|
+
# Thumbnails
|
86
|
+
._*
|
87
|
+
|
88
|
+
# Files that might appear in the root of a volume
|
89
|
+
.DocumentRevisions-V100
|
90
|
+
.fseventsd
|
91
|
+
.Spotlight-V100
|
92
|
+
.TemporaryItems
|
93
|
+
.Trashes
|
94
|
+
.VolumeIcon.icns
|
95
|
+
.com.apple.timemachine.donotpresent
|
96
|
+
|
97
|
+
# Directories potentially created on remote AFP share
|
98
|
+
.AppleDB
|
99
|
+
.AppleDesktop
|
100
|
+
Network Trash Folder
|
101
|
+
Temporary Items
|
102
|
+
.apdisk
|
103
|
+
|
104
|
+
### OSX ###
|
105
|
+
|
106
|
+
# Icon must end with two \r
|
107
|
+
|
108
|
+
|
109
|
+
# Thumbnails
|
110
|
+
|
111
|
+
# Files that might appear in the root of a volume
|
112
|
+
|
113
|
+
# Directories potentially created on remote AFP share
|
114
|
+
|
115
|
+
### Rails ###
|
116
|
+
*.rbc
|
117
|
+
capybara-*.html
|
118
|
+
.rspec
|
119
|
+
/log
|
120
|
+
/tmp
|
121
|
+
/db/*.sqlite3
|
122
|
+
/db/*.sqlite3-journal
|
123
|
+
/public/system
|
124
|
+
/coverage/
|
125
|
+
/spec/tmp
|
126
|
+
**.orig
|
127
|
+
rerun.txt
|
128
|
+
pickle-email-*.html
|
129
|
+
|
130
|
+
# TODO Comment out this rule if you are OK with secrets being uploaded to the repo
|
131
|
+
config/initializers/secret_token.rb
|
132
|
+
|
133
|
+
# Only include if you have production secrets in this file, which is no longer a Rails default
|
134
|
+
# config/secrets.yml
|
135
|
+
|
136
|
+
# dotenv
|
137
|
+
# TODO Comment out this rule if environment variables can be committed
|
138
|
+
.env
|
139
|
+
|
140
|
+
## Environment normalization:
|
141
|
+
/.bundle
|
142
|
+
/vendor/bundle
|
143
|
+
|
144
|
+
# these should all be checked in to normalize the environment:
|
145
|
+
# Gemfile.lock, .ruby-version, .ruby-gemset
|
146
|
+
|
147
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
148
|
+
.rvmrc
|
149
|
+
|
150
|
+
# if using bower-rails ignore default bower_components path bower.json files
|
151
|
+
/vendor/assets/bower_components
|
152
|
+
*.bowerrc
|
153
|
+
bower.json
|
154
|
+
|
155
|
+
# Ignore pow environment settings
|
156
|
+
.powenv
|
157
|
+
|
158
|
+
# Ignore Byebug command history file.
|
159
|
+
.byebug_history
|
160
|
+
|
161
|
+
### Ruby ###
|
162
|
+
*.gem
|
163
|
+
/.config
|
164
|
+
/InstalledFiles
|
165
|
+
/pkg/
|
166
|
+
/spec/reports/
|
167
|
+
/spec/examples.txt
|
168
|
+
/test/tmp/
|
169
|
+
/test/version_tmp/
|
170
|
+
/tmp/
|
171
|
+
|
172
|
+
# Used by dotenv library to load environment variables.
|
173
|
+
# .env
|
174
|
+
|
175
|
+
## Specific to RubyMotion:
|
176
|
+
.dat*
|
177
|
+
.repl_history
|
178
|
+
build/
|
179
|
+
*.bridgesupport
|
180
|
+
build-iPhoneOS/
|
181
|
+
build-iPhoneSimulator/
|
182
|
+
|
183
|
+
## Specific to RubyMotion (use of CocoaPods):
|
184
|
+
#
|
185
|
+
# We recommend against adding the Pods directory to your .gitignore. However
|
186
|
+
# you should judge for yourself, the pros and cons are mentioned at:
|
187
|
+
# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
|
188
|
+
#
|
189
|
+
# vendor/Pods/
|
190
|
+
|
191
|
+
## Documentation cache and generated files:
|
192
|
+
/.yardoc/
|
193
|
+
/_yardoc/
|
194
|
+
/doc/
|
195
|
+
/rdoc/
|
196
|
+
|
197
|
+
## Environment normalization:
|
198
|
+
/.bundle/
|
199
|
+
/lib/bundler/man/
|
200
|
+
|
201
|
+
# for a library or gem, you might want to ignore these files since the code is
|
202
|
+
# intended to run in multiple environments; otherwise, check them in:
|
203
|
+
# Gemfile.lock
|
204
|
+
# .ruby-version
|
205
|
+
# .ruby-gemset
|
206
|
+
|
207
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
208
|
+
|
209
|
+
### RubyMine ###
|
210
|
+
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
|
211
|
+
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
|
212
|
+
|
213
|
+
# User-specific stuff:
|
214
|
+
.idea/**/workspace.xml
|
215
|
+
.idea/**/tasks.xml
|
216
|
+
|
217
|
+
# Sensitive or high-churn files:
|
218
|
+
.idea/**/dataSources/
|
219
|
+
.idea/**/dataSources.ids
|
220
|
+
.idea/**/dataSources.xml
|
221
|
+
.idea/**/dataSources.local.xml
|
222
|
+
.idea/**/sqlDataSources.xml
|
223
|
+
.idea/**/dynamic.xml
|
224
|
+
.idea/**/uiDesigner.xml
|
225
|
+
|
226
|
+
# Gradle:
|
227
|
+
.idea/**/gradle.xml
|
228
|
+
.idea/**/libraries
|
229
|
+
|
230
|
+
# Mongo Explorer plugin:
|
231
|
+
.idea/**/mongoSettings.xml
|
232
|
+
|
233
|
+
## File-based project format:
|
234
|
+
*.iws
|
235
|
+
|
236
|
+
## Plugin-specific files:
|
237
|
+
|
238
|
+
# IntelliJ
|
239
|
+
/out/
|
240
|
+
|
241
|
+
# mpeltonen/sbt-idea plugin
|
242
|
+
.idea_modules/
|
243
|
+
|
244
|
+
# JIRA plugin
|
245
|
+
atlassian-ide-plugin.xml
|
246
|
+
|
247
|
+
# Crashlytics plugin (for Android Studio and IntelliJ)
|
248
|
+
com_crashlytics_export_strings.xml
|
249
|
+
crashlytics.properties
|
250
|
+
crashlytics-build.properties
|
251
|
+
fabric.properties
|
252
|
+
|
253
|
+
### RubyMine Patch ###
|
254
|
+
# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
|
255
|
+
|
256
|
+
# *.iml
|
257
|
+
# modules.xml
|
258
|
+
# .idea/misc.xml
|
259
|
+
# *.ipr
|
260
|
+
|
261
|
+
### SublimeText ###
|
262
|
+
# cache files for sublime text
|
263
|
+
*.tmlanguage.cache
|
264
|
+
*.tmPreferences.cache
|
265
|
+
*.stTheme.cache
|
266
|
+
|
267
|
+
# workspace files are user-specific
|
268
|
+
*.sublime-workspace
|
269
|
+
|
270
|
+
# project files should be checked into the repository, unless a significant
|
271
|
+
# proportion of contributors will probably not be using SublimeText
|
272
|
+
# *.sublime-project
|
273
|
+
|
274
|
+
# sftp configuration file
|
275
|
+
sftp-config.json
|
276
|
+
|
277
|
+
# Package control specific files
|
278
|
+
Package Control.last-run
|
279
|
+
Package Control.ca-list
|
280
|
+
Package Control.ca-bundle
|
281
|
+
Package Control.system-ca-bundle
|
282
|
+
Package Control.cache/
|
283
|
+
Package Control.ca-certs/
|
284
|
+
Package Control.merged-ca-bundle
|
285
|
+
Package Control.user-ca-bundle
|
286
|
+
oscrypto-ca-bundle.crt
|
287
|
+
bh_unicode_properties.cache
|
288
|
+
|
289
|
+
# Sublime-github package stores a github token in this file
|
290
|
+
# https://packagecontrol.io/packages/sublime-github
|
291
|
+
GitHub.sublime-settings
|
292
|
+
|
293
|
+
### Vim ###
|
294
|
+
# swap
|
295
|
+
[._]*.s[a-v][a-z]
|
296
|
+
[._]*.sw[a-p]
|
297
|
+
[._]s[a-v][a-z]
|
298
|
+
[._]sw[a-p]
|
299
|
+
# session
|
300
|
+
Session.vim
|
301
|
+
# temporary
|
302
|
+
.netrwhist
|
303
|
+
# auto-generated tag files
|
304
|
+
tags
|
305
|
+
|
306
|
+
### Windows ###
|
307
|
+
# Windows thumbnail cache files
|
308
|
+
Thumbs.db
|
309
|
+
ehthumbs.db
|
310
|
+
ehthumbs_vista.db
|
311
|
+
|
312
|
+
# Folder config file
|
313
|
+
Desktop.ini
|
314
|
+
|
315
|
+
# Recycle Bin used on file shares
|
316
|
+
$RECYCLE.BIN/
|
317
|
+
|
318
|
+
# Windows Installer files
|
319
|
+
*.cab
|
320
|
+
*.msi
|
321
|
+
*.msm
|
322
|
+
*.msp
|
323
|
+
|
324
|
+
# Windows shortcuts
|
325
|
+
*.lnk
|
326
|
+
|
327
|
+
# End of https://www.gitignore.io/api/ruby,rubymine,rails,emacs,vim,sublimetext,osx,macos,linux,windows
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
# This is the configuration used to check the rubocop source code.
|
2
|
+
|
3
|
+
AllCops:
|
4
|
+
Exclude:
|
5
|
+
- spec/spec_helper.rb
|
6
|
+
- db/**/*
|
7
|
+
- docs/**/*
|
8
|
+
- config/**/*
|
9
|
+
- bin/**/*
|
10
|
+
- spec/**/**
|
11
|
+
|
12
|
+
Documentation:
|
13
|
+
Enabled: false
|
14
|
+
|
15
|
+
LineLength:
|
16
|
+
Max: 99
|
17
|
+
|
18
|
+
HasAndBelongsToMany:
|
19
|
+
Enabled: false
|
20
|
+
|
21
|
+
Style/FrozenStringLiteralComment:
|
22
|
+
Enabled: false
|
23
|
+
|
24
|
+
Metrics/BlockLength:
|
25
|
+
ExcludedMethods: ['register', 'describe', 'show', 'shared_examples', 'context', 'form', 'has_many', 'inputs']
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,83 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
wor-batchifier (0.0.1)
|
5
|
+
httparty (~> 0.13)
|
6
|
+
|
7
|
+
GEM
|
8
|
+
remote: https://rubygems.org/
|
9
|
+
specs:
|
10
|
+
addressable (2.6.0)
|
11
|
+
public_suffix (>= 2.0.2, < 4.0)
|
12
|
+
ast (2.4.0)
|
13
|
+
byebug (9.1.0)
|
14
|
+
crack (0.4.3)
|
15
|
+
safe_yaml (~> 1.0.0)
|
16
|
+
diff-lcs (1.3)
|
17
|
+
docile (1.3.1)
|
18
|
+
hashdiff (0.3.8)
|
19
|
+
httparty (0.16.4)
|
20
|
+
mime-types (~> 3.0)
|
21
|
+
multi_xml (>= 0.5.2)
|
22
|
+
jaro_winkler (1.5.2)
|
23
|
+
json (2.1.0)
|
24
|
+
mime-types (3.2.2)
|
25
|
+
mime-types-data (~> 3.2015)
|
26
|
+
mime-types-data (3.2018.0812)
|
27
|
+
multi_xml (0.6.0)
|
28
|
+
parallel (1.13.0)
|
29
|
+
parser (2.6.0.0)
|
30
|
+
ast (~> 2.4.0)
|
31
|
+
powerpack (0.1.2)
|
32
|
+
public_suffix (3.0.3)
|
33
|
+
rainbow (3.0.0)
|
34
|
+
rake (10.5.0)
|
35
|
+
rspec (3.8.0)
|
36
|
+
rspec-core (~> 3.8.0)
|
37
|
+
rspec-expectations (~> 3.8.0)
|
38
|
+
rspec-mocks (~> 3.8.0)
|
39
|
+
rspec-core (3.8.0)
|
40
|
+
rspec-support (~> 3.8.0)
|
41
|
+
rspec-expectations (3.8.2)
|
42
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
43
|
+
rspec-support (~> 3.8.0)
|
44
|
+
rspec-mocks (3.8.0)
|
45
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
46
|
+
rspec-support (~> 3.8.0)
|
47
|
+
rspec-support (3.8.0)
|
48
|
+
rubocop (0.63.1)
|
49
|
+
jaro_winkler (~> 1.5.1)
|
50
|
+
parallel (~> 1.10)
|
51
|
+
parser (>= 2.5, != 2.5.1.1)
|
52
|
+
powerpack (~> 0.1)
|
53
|
+
rainbow (>= 2.2.2, < 4.0)
|
54
|
+
ruby-progressbar (~> 1.7)
|
55
|
+
unicode-display_width (~> 1.4.0)
|
56
|
+
ruby-progressbar (1.10.0)
|
57
|
+
safe_yaml (1.0.4)
|
58
|
+
simplecov (0.16.1)
|
59
|
+
docile (~> 1.1)
|
60
|
+
json (>= 1.8, < 3)
|
61
|
+
simplecov-html (~> 0.10.0)
|
62
|
+
simplecov-html (0.10.2)
|
63
|
+
unicode-display_width (1.4.1)
|
64
|
+
webmock (3.5.1)
|
65
|
+
addressable (>= 2.3.6)
|
66
|
+
crack (>= 0.3.2)
|
67
|
+
hashdiff
|
68
|
+
|
69
|
+
PLATFORMS
|
70
|
+
ruby
|
71
|
+
|
72
|
+
DEPENDENCIES
|
73
|
+
bundler
|
74
|
+
byebug (~> 9.0)
|
75
|
+
rake (~> 10.0)
|
76
|
+
rspec (~> 3.0)
|
77
|
+
rubocop (~> 0.47)
|
78
|
+
simplecov
|
79
|
+
webmock
|
80
|
+
wor-batchifier!
|
81
|
+
|
82
|
+
BUNDLED WITH
|
83
|
+
1.11.2
|
data/README.md
ADDED
@@ -0,0 +1,134 @@
|
|
1
|
+
# Wolox on Rails - Batchifier
|
2
|
+
[![Gem Version](https://badge.fury.io/rb/wor-batchifier.svg)](https://badge.fury.io/rb/wor-batchifier)
|
3
|
+
[![Dependency Status](https://gemnasium.com/badges/github.com/Wolox/wor-batchifier.svg)](https://gemnasium.com/github.com/Wolox/wor-batchifier)
|
4
|
+
[![Build Status](https://travis-ci.org/Wolox/wor-batchifier.svg)](https://travis-ci.org/Wolox/wor-batchifier)
|
5
|
+
[![Code Climate](https://codeclimate.com/github/Wolox/wor-batchifier/badges/gpa.svg)](https://codeclimate.com/github/Wolox/wor-authentication)
|
6
|
+
[![Test Coverage](https://codeclimate.com/github/Wolox/wor-batchifier/badges/coverage.svg)](https://codeclimate.com/github/Wolox/wor-batchifier/coverage)
|
7
|
+
|
8
|
+
Gem that allows you to easily divide requests that send a lot of information into several requests with smaller chunks of data, taking care of the response from each one and providing it joint together based on different strategies for parsing said response.
|
9
|
+
|
10
|
+
## Installation
|
11
|
+
|
12
|
+
Add this line to your application's Gemfile:
|
13
|
+
|
14
|
+
```ruby
|
15
|
+
gem 'wor-batchifier'
|
16
|
+
```
|
17
|
+
|
18
|
+
And then execute:
|
19
|
+
|
20
|
+
$ bundle
|
21
|
+
|
22
|
+
Or install it yourself as:
|
23
|
+
|
24
|
+
$ gem install wor-batchifier
|
25
|
+
|
26
|
+
## Usage
|
27
|
+
|
28
|
+
### Basic use:
|
29
|
+
|
30
|
+
The first step is to define a parent controller from which all other controllers will have to extend to have access to the batchifier's methods. So, let's do that in our `ApplicationController.rb`:
|
31
|
+
|
32
|
+
```ruby
|
33
|
+
class ApplicationController < ActionController::Base
|
34
|
+
include Wor::Batchifier
|
35
|
+
end
|
36
|
+
```
|
37
|
+
|
38
|
+
You can also include the batchifier just in the controllers you intend to use it in.
|
39
|
+
|
40
|
+
The final step, is to find any request you wish to perform with smaller chunks of data and utilize the batchifier's methods to divide it into smaller tasks.
|
41
|
+
|
42
|
+
For example, let's pretend we have an endpoint called bulk_request that communicates with a third API and sends a lot of information to be utilized.
|
43
|
+
|
44
|
+
```ruby
|
45
|
+
def bulk_request
|
46
|
+
ThirdAPI.bulk_request(params[:information])
|
47
|
+
end
|
48
|
+
```
|
49
|
+
|
50
|
+
Now we will partition that request into chunks using the batchifier:
|
51
|
+
|
52
|
+
```ruby
|
53
|
+
def bulk_request
|
54
|
+
execute_in_batches(params[:information], batch_size: 100, strategy: :add) do |chunks|
|
55
|
+
ThirdAPI.bulk_request(chunks)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
```
|
59
|
+
|
60
|
+
The batchifier will take three parameters, the first one being the information that needs to be partitioned, then the batch_size we wish to utilize and finally the symbol of the strategy that will be implemented on the response of each batch.
|
61
|
+
|
62
|
+
### Available strategies
|
63
|
+
|
64
|
+
- Add: For each request, it joins together each response no matter the result.
|
65
|
+
- Maintain-Unique: It will only add the results that are not present already in the response.
|
66
|
+
- No-Response: It will not provide any response whatsoever.
|
67
|
+
|
68
|
+
### Adding new strategies
|
69
|
+
|
70
|
+
Should you desire to add new strategies, it's as simple as creating a new class and defining a method called `merge_strategy` which will hold the logic that will be implemented to parse the response. Let's look at an example:
|
71
|
+
|
72
|
+
```ruby
|
73
|
+
module Wor
|
74
|
+
module Batchifier
|
75
|
+
class MaintainUnique < Strategy
|
76
|
+
def merge_strategy(response,rec)
|
77
|
+
return response.merge(rec) { |_, v1, _| v1 }
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
```
|
83
|
+
|
84
|
+
The `merge_strategy` will receive two parameters, the first being "response" which is the total response which will be returned from `execute_in_batches`, and "rec" is the recursive response from each batch that will be added to response in each iteration. If you want to merge or do something else entirely, you have the option to do so.
|
85
|
+
|
86
|
+
All strategies have a `base_case` which by default is `{}` but if you wish to override it, you can define your own in your strategy by simply adding a method called `base_case` which should return the value you desire for your own personal case.
|
87
|
+
|
88
|
+
```ruby
|
89
|
+
def base_case
|
90
|
+
# Your base value for your batches.
|
91
|
+
end
|
92
|
+
```
|
93
|
+
|
94
|
+
The new class that will hold the method `merge_strategy` should inherit the class `Strategy`. If the strategy doesn't define the method, an exception will be raised when trying to utilize it
|
95
|
+
warning that it does not respect the contract set by the `Strategy` Interface.
|
96
|
+
|
97
|
+
## Contributing
|
98
|
+
|
99
|
+
1. Fork it
|
100
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
101
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
102
|
+
4. Run rubocop lint (`rubocop -R --format simple`)
|
103
|
+
5. Run rspec tests (`bundle exec rspec`)
|
104
|
+
6. Push your branch (`git push origin my-new-feature`)
|
105
|
+
7. Create a new Pull Request
|
106
|
+
|
107
|
+
## About ##
|
108
|
+
|
109
|
+
This project is maintained by [Diego Raffo](https://github.com/enanodr) along with [Pedro Jara](https://github.com/redwarewolf) and it was written by [Wolox](http://www.wolox.com.ar).
|
110
|
+
![Wolox](https://raw.githubusercontent.com/Wolox/press-kit/master/logos/logo_banner.png)
|
111
|
+
|
112
|
+
## License
|
113
|
+
|
114
|
+
**wor-batchifier** is available under the MIT [license](https://raw.githubusercontent.com/Wolox/wor-batchifier/master/LICENSE.md).
|
115
|
+
|
116
|
+
Copyright (c) 2017 Wolox
|
117
|
+
|
118
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
119
|
+
of this software and associated documentation files (the "Software"), to deal
|
120
|
+
in the Software without restriction, including without limitation the rights
|
121
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
122
|
+
copies of the Software, and to permit persons to whom the Software is
|
123
|
+
furnished to do so, subject to the following conditions:
|
124
|
+
|
125
|
+
The above copyright notice and this permission notice shall be included in
|
126
|
+
all copies or substantial portions of the Software.
|
127
|
+
|
128
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
129
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
130
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
131
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
132
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
133
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
134
|
+
THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
require_relative 'batchifier/exceptions'
|
2
|
+
require_relative 'batchifier/interface'
|
3
|
+
require_relative 'batchifier/strategy'
|
4
|
+
require_relative 'batchifier/add'
|
5
|
+
require_relative 'batchifier/no_response'
|
6
|
+
require_relative 'batchifier/array_merge'
|
7
|
+
require_relative 'batchifier/maintain_unique'
|
8
|
+
|
9
|
+
module Wor
|
10
|
+
module Batchifier
|
11
|
+
|
12
|
+
# This module exports the function execute_in_batches, that needs a collections and
|
13
|
+
# => optionaly a batch_size and a merge strategy. It will slice the collection and
|
14
|
+
# => apply the chozen strategy to all chunks and merge the results. It expects the responses
|
15
|
+
# => to be Hash. It can ignore them if the given strategy is no_response
|
16
|
+
def execute_in_batches(collection, batch_size: 100, strategy: :add)
|
17
|
+
strategy_class = classify_strategy(strategy)
|
18
|
+
collection.lazy.each_slice(batch_size).inject(strategy_class.new.base_case) do |rec, chunk|
|
19
|
+
response = yield(chunk)
|
20
|
+
merge(response, rec, strategy)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def merge(response, rec, strategy)
|
27
|
+
return Wor::Batchifier.classify_strategy(strategy).new.merge_strategy(response,rec)
|
28
|
+
end
|
29
|
+
|
30
|
+
def classify_strategy(strategy)
|
31
|
+
strategy_class_name = strategy.to_s.split('_').collect(&:capitalize).join
|
32
|
+
Kernel.const_get(strategy_class_name)
|
33
|
+
rescue NameError => e
|
34
|
+
raise Wor::Batchifier::Exceptions::StrategyNotFound
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Wor
|
2
|
+
module Batchifier
|
3
|
+
module Interface
|
4
|
+
attr_writer :contract
|
5
|
+
|
6
|
+
def implements(*selectors)
|
7
|
+
self.contract += selectors
|
8
|
+
end
|
9
|
+
|
10
|
+
def contract
|
11
|
+
@contract ||= []
|
12
|
+
end
|
13
|
+
|
14
|
+
def full_contract
|
15
|
+
(contract + ancestors.flat_map(&:contract)).uniq
|
16
|
+
end
|
17
|
+
|
18
|
+
def breaches_contract?
|
19
|
+
full_contract.any? { |it| !method_defined? it }
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Wor
|
2
|
+
module Batchifier
|
3
|
+
class Strategy
|
4
|
+
extend Wor::Batchifier::Interface
|
5
|
+
|
6
|
+
implements :merge_strategy
|
7
|
+
|
8
|
+
class << self
|
9
|
+
alias_method :__new__, :new
|
10
|
+
|
11
|
+
def new(*args)
|
12
|
+
raise Wor::Batchifier::Exceptions::InterfaceNotImplemented.new "class #{name} does not implement contract #{contract}!" if breaches_contract?
|
13
|
+
__new__(*args)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def base_case
|
18
|
+
{}
|
19
|
+
end
|
20
|
+
|
21
|
+
# When defining your own strategy for merging, you should define a new class that extends from
|
22
|
+
# this class, "Strategy", and implement the method "merge_strategy" which will take care
|
23
|
+
# of parsing the response of the batchified endpoint.
|
24
|
+
# Should you not implement the method "merge_strategy" the exception "InterfaceNotImplemented"
|
25
|
+
# will be raised to notify the developer of such issue.
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'wor/batchifier/version'
|
5
|
+
require 'date'
|
6
|
+
|
7
|
+
Gem::Specification.new do |spec|
|
8
|
+
spec.name = 'wor-batchifier'
|
9
|
+
spec.version = Wor::Batchifier::VERSION
|
10
|
+
spec.platform = Gem::Platform::RUBY
|
11
|
+
spec.date = Date.today
|
12
|
+
spec.authors = ['enanodr', 'redwarewolf']
|
13
|
+
spec.email = ['diego.raffo@wolox.com.ar', 'pedro.jara@wolox.com.ar']
|
14
|
+
|
15
|
+
spec.summary = 'Easily batchify your requests!.'
|
16
|
+
spec.description = 'Gem to batchify your requests.'
|
17
|
+
spec.homepage = 'https://github.com/Wolox/wor-batchifier'
|
18
|
+
spec.license = 'MIT'
|
19
|
+
|
20
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec)/}) }
|
21
|
+
spec.test_files = spec.files.grep(%r{^(test|spec)/})
|
22
|
+
spec.require_paths = ['lib']
|
23
|
+
|
24
|
+
spec.add_dependency 'httparty', '~> 0.13'
|
25
|
+
|
26
|
+
spec.add_development_dependency 'byebug', '~> 9.0'
|
27
|
+
spec.add_development_dependency 'rubocop', '~> 0.47'
|
28
|
+
spec.add_development_dependency 'bundler'
|
29
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
30
|
+
spec.add_development_dependency 'rspec', '~> 3.0'
|
31
|
+
spec.add_development_dependency 'simplecov'
|
32
|
+
spec.add_development_dependency 'webmock'
|
33
|
+
end
|
metadata
ADDED
@@ -0,0 +1,174 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: wor-batchifier
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- enanodr
|
8
|
+
- redwarewolf
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2019-04-17 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: httparty
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - "~>"
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '0.13'
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - "~>"
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '0.13'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: byebug
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - "~>"
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '9.0'
|
35
|
+
type: :development
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - "~>"
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '9.0'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: rubocop
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - "~>"
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '0.47'
|
49
|
+
type: :development
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - "~>"
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '0.47'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: bundler
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
type: :development
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: rake
|
72
|
+
requirement: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - "~>"
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '10.0'
|
77
|
+
type: :development
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - "~>"
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '10.0'
|
84
|
+
- !ruby/object:Gem::Dependency
|
85
|
+
name: rspec
|
86
|
+
requirement: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - "~>"
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: '3.0'
|
91
|
+
type: :development
|
92
|
+
prerelease: false
|
93
|
+
version_requirements: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - "~>"
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: '3.0'
|
98
|
+
- !ruby/object:Gem::Dependency
|
99
|
+
name: simplecov
|
100
|
+
requirement: !ruby/object:Gem::Requirement
|
101
|
+
requirements:
|
102
|
+
- - ">="
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
version: '0'
|
105
|
+
type: :development
|
106
|
+
prerelease: false
|
107
|
+
version_requirements: !ruby/object:Gem::Requirement
|
108
|
+
requirements:
|
109
|
+
- - ">="
|
110
|
+
- !ruby/object:Gem::Version
|
111
|
+
version: '0'
|
112
|
+
- !ruby/object:Gem::Dependency
|
113
|
+
name: webmock
|
114
|
+
requirement: !ruby/object:Gem::Requirement
|
115
|
+
requirements:
|
116
|
+
- - ">="
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: '0'
|
119
|
+
type: :development
|
120
|
+
prerelease: false
|
121
|
+
version_requirements: !ruby/object:Gem::Requirement
|
122
|
+
requirements:
|
123
|
+
- - ">="
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
description: Gem to batchify your requests.
|
127
|
+
email:
|
128
|
+
- diego.raffo@wolox.com.ar
|
129
|
+
- pedro.jara@wolox.com.ar
|
130
|
+
executables: []
|
131
|
+
extensions: []
|
132
|
+
extra_rdoc_files: []
|
133
|
+
files:
|
134
|
+
- ".gitignore"
|
135
|
+
- ".rubocop.yml"
|
136
|
+
- Gemfile
|
137
|
+
- Gemfile.lock
|
138
|
+
- README.md
|
139
|
+
- Rakefile
|
140
|
+
- lib/wor/batchifier.rb
|
141
|
+
- lib/wor/batchifier/add.rb
|
142
|
+
- lib/wor/batchifier/array_merge.rb
|
143
|
+
- lib/wor/batchifier/exceptions.rb
|
144
|
+
- lib/wor/batchifier/interface.rb
|
145
|
+
- lib/wor/batchifier/maintain_unique.rb
|
146
|
+
- lib/wor/batchifier/no_response.rb
|
147
|
+
- lib/wor/batchifier/strategy.rb
|
148
|
+
- lib/wor/batchifier/version.rb
|
149
|
+
- wor-batchifier.gemspec
|
150
|
+
homepage: https://github.com/Wolox/wor-batchifier
|
151
|
+
licenses:
|
152
|
+
- MIT
|
153
|
+
metadata: {}
|
154
|
+
post_install_message:
|
155
|
+
rdoc_options: []
|
156
|
+
require_paths:
|
157
|
+
- lib
|
158
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
159
|
+
requirements:
|
160
|
+
- - ">="
|
161
|
+
- !ruby/object:Gem::Version
|
162
|
+
version: '0'
|
163
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
164
|
+
requirements:
|
165
|
+
- - ">="
|
166
|
+
- !ruby/object:Gem::Version
|
167
|
+
version: '0'
|
168
|
+
requirements: []
|
169
|
+
rubyforge_project:
|
170
|
+
rubygems_version: 2.5.2.1
|
171
|
+
signing_key:
|
172
|
+
specification_version: 4
|
173
|
+
summary: Easily batchify your requests!.
|
174
|
+
test_files: []
|