kraps 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +6 -0
- data/CHANGELOG.md +30 -0
- data/Gemfile +4 -3
- data/README.md +20 -6
- data/lib/kraps/runner.rb +1 -1
- data/lib/kraps/version.rb +1 -1
- data/lib/kraps/worker.rb +1 -1
- metadata +2 -45
- data/Gemfile.lock +0 -108
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8b24f67ff2122dc82d372eb421c379ac42f415674999958f42f71a2cdbee1a33
|
4
|
+
data.tar.gz: d8297482eb38a30cb8ff6a7761a544833e3795a8f3a045343f6a10c370d886ca
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5543d1a8af8fa12007d38f00d9aa515eb1edf254d8bbc2aa8c133c0101dbe7ccaef5f0330929b6b84456a81eec7e5744091deace2d16ed2826f14cd56432db8f
|
7
|
+
data.tar.gz: 14992d608157562da3af98207681a66cf0a9a9566861fe79c949ccdec9db6cb344bfb070147fc820eac70b076c15db373896f80b5bd8d5f8714f9ce75d7eb7c8
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,35 @@
|
|
1
1
|
# CHANGELOG
|
2
2
|
|
3
|
+
## v0.9.0
|
4
|
+
|
5
|
+
* Argments are no longer passed to the `call` method, but to the
|
6
|
+
initializer instead
|
7
|
+
|
8
|
+
Before:
|
9
|
+
|
10
|
+
```ruby
|
11
|
+
class MyJob
|
12
|
+
def call(arg1, arg2)
|
13
|
+
# ...
|
14
|
+
end
|
15
|
+
end
|
16
|
+
```
|
17
|
+
|
18
|
+
After:
|
19
|
+
|
20
|
+
```ruby
|
21
|
+
class MyJob
|
22
|
+
def initializer(arg1, arg2)
|
23
|
+
@arg1 = arg1
|
24
|
+
@arg2 = arg2
|
25
|
+
end
|
26
|
+
|
27
|
+
def call
|
28
|
+
# ...
|
29
|
+
end
|
30
|
+
end
|
31
|
+
```
|
32
|
+
|
3
33
|
## v0.8.0
|
4
34
|
|
5
35
|
* Use number of partitions of previous step for `jobs` option by default
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -38,16 +38,21 @@ Kraps.configure(
|
|
38
38
|
|
39
39
|
Afterwards, create a job class, which tells Kraps what your job should do.
|
40
40
|
Therefore, you create some class with a `call` method, and optionally some
|
41
|
-
arguments. Let's create a simple job, which reads
|
42
|
-
how often search queries have been searched:
|
41
|
+
arguments passed to its initializer. Let's create a simple job, which reads
|
42
|
+
search log files to analyze how often search queries have been searched:
|
43
43
|
|
44
44
|
```ruby
|
45
45
|
class SearchLogCounter
|
46
|
-
def
|
46
|
+
def initialize(start_date:, end_date:)
|
47
|
+
@start_date = start_date
|
48
|
+
@end_date = end_date
|
49
|
+
end
|
50
|
+
|
51
|
+
def call
|
47
52
|
job = Kraps::Job.new(worker: MyKrapsWorker)
|
48
53
|
|
49
54
|
job = job.parallelize(partitions: 128) do |collector|
|
50
|
-
(Date.parse(start_date)..Date.parse(end_date)).each do |date|
|
55
|
+
(Date.parse(@start_date)..Date.parse(@end_date)).each do |date|
|
51
56
|
collector.call(date.to_s)
|
52
57
|
end
|
53
58
|
end
|
@@ -214,6 +219,10 @@ job.parallelize(partitions: 128, partitioner: partitioner, worker: MyKrapsWorker
|
|
214
219
|
end
|
215
220
|
```
|
216
221
|
|
222
|
+
Please note, that `parallelize` itself is not parallelized but rather
|
223
|
+
parallelizes the data you feed into Kraps within `parallelize` by splitting it
|
224
|
+
into the number of `partitions` specified.
|
225
|
+
|
217
226
|
The block must use the collector to feed Kraps with individual items. The
|
218
227
|
items are used as keys and the values are set to `nil`.
|
219
228
|
|
@@ -345,11 +354,16 @@ of searches made:
|
|
345
354
|
|
346
355
|
```ruby
|
347
356
|
class SearchLogCounter
|
348
|
-
def
|
357
|
+
def initialize(start_date:, end_date:)
|
358
|
+
@start_date = start_date
|
359
|
+
@end_date = end_date
|
360
|
+
end
|
361
|
+
|
362
|
+
def call
|
349
363
|
count_job = Kraps::Job.new(worker: SomeBackgroundWorker)
|
350
364
|
|
351
365
|
count_job = count_job.parallelize(partitions: 128) do |collector|
|
352
|
-
(Date.parse(start_date)..Date.parse(end_date)).each do |date|
|
366
|
+
(Date.parse(@start_date)..Date.parse(@end_date)).each do |date|
|
353
367
|
collector.call(date.to_s)
|
354
368
|
end
|
355
369
|
end
|
data/lib/kraps/runner.rb
CHANGED
@@ -5,7 +5,7 @@ module Kraps
|
|
5
5
|
end
|
6
6
|
|
7
7
|
def call(*args, **kwargs)
|
8
|
-
JobResolver.new.call(@klass.new
|
8
|
+
JobResolver.new.call(@klass.new(*args, **kwargs).call).tap do |jobs|
|
9
9
|
jobs.each_with_index do |job, job_index|
|
10
10
|
job.steps.each_with_index.inject(nil) do |frame, (_, step_index)|
|
11
11
|
StepRunner.new(
|
data/lib/kraps/version.rb
CHANGED
data/lib/kraps/worker.rb
CHANGED
@@ -274,7 +274,7 @@ module Kraps
|
|
274
274
|
end
|
275
275
|
|
276
276
|
def jobs
|
277
|
-
@jobs ||= JobResolver.new.call(@args["klass"].constantize.new
|
277
|
+
@jobs ||= JobResolver.new.call(@args["klass"].constantize.new(*@args["args"], **@args["kwargs"].transform_keys(&:to_sym)).call)
|
278
278
|
end
|
279
279
|
|
280
280
|
def job
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kraps
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Benjamin Vetter
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-03-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: attachie
|
@@ -66,48 +66,6 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
-
- !ruby/object:Gem::Dependency
|
70
|
-
name: bundler
|
71
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - ">="
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version: '0'
|
76
|
-
type: :development
|
77
|
-
prerelease: false
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
79
|
-
requirements:
|
80
|
-
- - ">="
|
81
|
-
- !ruby/object:Gem::Version
|
82
|
-
version: '0'
|
83
|
-
- !ruby/object:Gem::Dependency
|
84
|
-
name: rspec
|
85
|
-
requirement: !ruby/object:Gem::Requirement
|
86
|
-
requirements:
|
87
|
-
- - ">="
|
88
|
-
- !ruby/object:Gem::Version
|
89
|
-
version: '0'
|
90
|
-
type: :development
|
91
|
-
prerelease: false
|
92
|
-
version_requirements: !ruby/object:Gem::Requirement
|
93
|
-
requirements:
|
94
|
-
- - ">="
|
95
|
-
- !ruby/object:Gem::Version
|
96
|
-
version: '0'
|
97
|
-
- !ruby/object:Gem::Dependency
|
98
|
-
name: rubocop
|
99
|
-
requirement: !ruby/object:Gem::Requirement
|
100
|
-
requirements:
|
101
|
-
- - ">="
|
102
|
-
- !ruby/object:Gem::Version
|
103
|
-
version: '0'
|
104
|
-
type: :development
|
105
|
-
prerelease: false
|
106
|
-
version_requirements: !ruby/object:Gem::Requirement
|
107
|
-
requirements:
|
108
|
-
- - ">="
|
109
|
-
- !ruby/object:Gem::Version
|
110
|
-
version: '0'
|
111
69
|
description: Kraps allows to process and perform calculations on very large datasets
|
112
70
|
in parallel
|
113
71
|
email:
|
@@ -121,7 +79,6 @@ files:
|
|
121
79
|
- CHANGELOG.md
|
122
80
|
- CODE_OF_CONDUCT.md
|
123
81
|
- Gemfile
|
124
|
-
- Gemfile.lock
|
125
82
|
- LICENSE.txt
|
126
83
|
- README.md
|
127
84
|
- Rakefile
|
data/Gemfile.lock
DELETED
@@ -1,108 +0,0 @@
|
|
1
|
-
PATH
|
2
|
-
remote: .
|
3
|
-
specs:
|
4
|
-
kraps (0.7.0)
|
5
|
-
attachie
|
6
|
-
map-reduce-ruby (>= 3.0.0)
|
7
|
-
redis
|
8
|
-
ruby-progressbar
|
9
|
-
|
10
|
-
GEM
|
11
|
-
remote: https://rubygems.org/
|
12
|
-
specs:
|
13
|
-
activesupport (7.0.4)
|
14
|
-
concurrent-ruby (~> 1.0, >= 1.0.2)
|
15
|
-
i18n (>= 1.6, < 2)
|
16
|
-
minitest (>= 5.1)
|
17
|
-
tzinfo (~> 2.0)
|
18
|
-
ast (2.4.2)
|
19
|
-
attachie (1.2.0)
|
20
|
-
activesupport
|
21
|
-
aws-sdk-s3
|
22
|
-
connection_pool
|
23
|
-
mime-types
|
24
|
-
aws-eventstream (1.2.0)
|
25
|
-
aws-partitions (1.657.0)
|
26
|
-
aws-sdk-core (3.166.0)
|
27
|
-
aws-eventstream (~> 1, >= 1.0.2)
|
28
|
-
aws-partitions (~> 1, >= 1.651.0)
|
29
|
-
aws-sigv4 (~> 1.5)
|
30
|
-
jmespath (~> 1, >= 1.6.1)
|
31
|
-
aws-sdk-kms (1.59.0)
|
32
|
-
aws-sdk-core (~> 3, >= 3.165.0)
|
33
|
-
aws-sigv4 (~> 1.1)
|
34
|
-
aws-sdk-s3 (1.117.1)
|
35
|
-
aws-sdk-core (~> 3, >= 3.165.0)
|
36
|
-
aws-sdk-kms (~> 1)
|
37
|
-
aws-sigv4 (~> 1.4)
|
38
|
-
aws-sigv4 (1.5.2)
|
39
|
-
aws-eventstream (~> 1, >= 1.0.2)
|
40
|
-
concurrent-ruby (1.1.10)
|
41
|
-
connection_pool (2.3.0)
|
42
|
-
diff-lcs (1.5.0)
|
43
|
-
i18n (1.12.0)
|
44
|
-
concurrent-ruby (~> 1.0)
|
45
|
-
jmespath (1.6.1)
|
46
|
-
json (2.6.2)
|
47
|
-
lazy_priority_queue (0.1.1)
|
48
|
-
map-reduce-ruby (3.0.0)
|
49
|
-
json
|
50
|
-
lazy_priority_queue
|
51
|
-
mime-types (3.4.1)
|
52
|
-
mime-types-data (~> 3.2015)
|
53
|
-
mime-types-data (3.2022.0105)
|
54
|
-
minitest (5.16.3)
|
55
|
-
parallel (1.22.1)
|
56
|
-
parser (3.1.2.1)
|
57
|
-
ast (~> 2.4.1)
|
58
|
-
rainbow (3.1.1)
|
59
|
-
rake (13.0.6)
|
60
|
-
redis (5.0.5)
|
61
|
-
redis-client (>= 0.9.0)
|
62
|
-
redis-client (0.11.2)
|
63
|
-
connection_pool
|
64
|
-
regexp_parser (2.6.0)
|
65
|
-
rexml (3.2.5)
|
66
|
-
rspec (3.12.0)
|
67
|
-
rspec-core (~> 3.12.0)
|
68
|
-
rspec-expectations (~> 3.12.0)
|
69
|
-
rspec-mocks (~> 3.12.0)
|
70
|
-
rspec-core (3.12.0)
|
71
|
-
rspec-support (~> 3.12.0)
|
72
|
-
rspec-expectations (3.12.0)
|
73
|
-
diff-lcs (>= 1.2.0, < 2.0)
|
74
|
-
rspec-support (~> 3.12.0)
|
75
|
-
rspec-mocks (3.12.0)
|
76
|
-
diff-lcs (>= 1.2.0, < 2.0)
|
77
|
-
rspec-support (~> 3.12.0)
|
78
|
-
rspec-support (3.12.0)
|
79
|
-
rubocop (1.38.0)
|
80
|
-
json (~> 2.3)
|
81
|
-
parallel (~> 1.10)
|
82
|
-
parser (>= 3.1.2.1)
|
83
|
-
rainbow (>= 2.2.2, < 4.0)
|
84
|
-
regexp_parser (>= 1.8, < 3.0)
|
85
|
-
rexml (>= 3.2.5, < 4.0)
|
86
|
-
rubocop-ast (>= 1.23.0, < 2.0)
|
87
|
-
ruby-progressbar (~> 1.7)
|
88
|
-
unicode-display_width (>= 1.4.0, < 3.0)
|
89
|
-
rubocop-ast (1.23.0)
|
90
|
-
parser (>= 3.1.1.0)
|
91
|
-
ruby-progressbar (1.11.0)
|
92
|
-
tzinfo (2.0.5)
|
93
|
-
concurrent-ruby (~> 1.0)
|
94
|
-
unicode-display_width (2.3.0)
|
95
|
-
|
96
|
-
PLATFORMS
|
97
|
-
ruby
|
98
|
-
x86_64-linux
|
99
|
-
|
100
|
-
DEPENDENCIES
|
101
|
-
bundler
|
102
|
-
kraps!
|
103
|
-
rake (~> 13.0)
|
104
|
-
rspec (~> 3.0)
|
105
|
-
rubocop
|
106
|
-
|
107
|
-
BUNDLED WITH
|
108
|
-
2.3.24
|