kraps 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +3 -0
  3. data/CHANGELOG.md +5 -0
  4. data/Gemfile.lock +28 -30
  5. data/lib/kraps/version.rb +1 -1
  6. data/lib/kraps/worker.rb +14 -6
  7. metadata +5 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d1f08b6fa0f725c63e3750f4b3bf04479622b40160a6364f708d91d37c0b1948
4
- data.tar.gz: '0681d837d852846cc6c115fe9dae3075a0e5b3bb8b8eae2d90d8a23ec26581e3'
3
+ metadata.gz: 57d06d20f406e72e26424dcba5c2af296a1551085113371c2cae9894f18e72ff
4
+ data.tar.gz: a25c8c3440cdd26eeb9b32655e77556e4be4966d47bd3e290d0a702c3b4cde9f
5
5
  SHA512:
6
- metadata.gz: 354ab3129ef1713c8229af54945251069c98d681e2db5c716d93b5925576b601751c23c1d502cb72ddaea5df5fc91e6eceb4590a619de730ae65f0762662da21
7
- data.tar.gz: 0647fc85f445bc634f70e2e10feab325c3df6aec3a30d2af4b1a792e82b9adf1a31e8bb339465e10944bce927933956e8d88e72b37fb7d84027ec569441781d6
6
+ metadata.gz: 655e2d0f525e136b72e87231a8d359eda952b7e3b30a8cd38b25f7b4fba27baebedacced0a00071915166325b621437869026c6785ab7353eea928ca736bc2a7
7
+ data.tar.gz: 0d450b3032a6809d2a5300e18e3cfc9a948ca5f589f1161823d9923084939d249282d28bf0619c3b9ace10dc742118dfcf5895c898e2d9fe7c55f98d00e683b4
data/.rubocop.yml CHANGED
@@ -77,3 +77,6 @@ Lint/EmptyClass:
77
77
 
78
78
  Style/WordArray:
79
79
  Enabled: false
80
+
81
+ Style/RedundantEach:
82
+ Enabled: false
data/CHANGELOG.md ADDED
@@ -0,0 +1,5 @@
1
+ # CHANGELOG
2
+
3
+ ## v0.2.0
4
+
5
+ * Updated map-reduce-ruby to allow concurrent uploads
data/Gemfile.lock CHANGED
@@ -1,22 +1,21 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- kraps (0.1.0)
4
+ kraps (0.2.0)
5
5
  attachie
6
6
  distributed_job
7
- map-reduce-ruby (>= 2.1.1)
7
+ map-reduce-ruby (>= 3.0.0)
8
8
  redis
9
9
  ruby-progressbar
10
10
 
11
11
  GEM
12
12
  remote: https://rubygems.org/
13
13
  specs:
14
- activesupport (6.1.7)
14
+ activesupport (7.0.4)
15
15
  concurrent-ruby (~> 1.0, >= 1.0.2)
16
16
  i18n (>= 1.6, < 2)
17
17
  minitest (>= 5.1)
18
18
  tzinfo (~> 2.0)
19
- zeitwerk (~> 2.3)
20
19
  ast (2.4.2)
21
20
  attachie (1.2.0)
22
21
  activesupport
@@ -24,17 +23,17 @@ GEM
24
23
  connection_pool
25
24
  mime-types
26
25
  aws-eventstream (1.2.0)
27
- aws-partitions (1.649.0)
28
- aws-sdk-core (3.164.0)
26
+ aws-partitions (1.654.0)
27
+ aws-sdk-core (3.166.0)
29
28
  aws-eventstream (~> 1, >= 1.0.2)
30
- aws-partitions (~> 1, >= 1.525.0)
31
- aws-sigv4 (~> 1.1)
29
+ aws-partitions (~> 1, >= 1.651.0)
30
+ aws-sigv4 (~> 1.5)
32
31
  jmespath (~> 1, >= 1.6.1)
33
- aws-sdk-kms (1.58.0)
34
- aws-sdk-core (~> 3, >= 3.127.0)
32
+ aws-sdk-kms (1.59.0)
33
+ aws-sdk-core (~> 3, >= 3.165.0)
35
34
  aws-sigv4 (~> 1.1)
36
- aws-sdk-s3 (1.116.0)
37
- aws-sdk-core (~> 3, >= 3.127.0)
35
+ aws-sdk-s3 (1.117.1)
36
+ aws-sdk-core (~> 3, >= 3.165.0)
38
37
  aws-sdk-kms (~> 1)
39
38
  aws-sigv4 (~> 1.4)
40
39
  aws-sigv4 (1.5.2)
@@ -49,7 +48,7 @@ GEM
49
48
  jmespath (1.6.1)
50
49
  json (2.6.2)
51
50
  lazy_priority_queue (0.1.1)
52
- map-reduce-ruby (2.1.1)
51
+ map-reduce-ruby (3.0.0)
53
52
  json
54
53
  lazy_priority_queue
55
54
  mime-types (3.4.1)
@@ -63,40 +62,39 @@ GEM
63
62
  rake (13.0.6)
64
63
  redis (5.0.5)
65
64
  redis-client (>= 0.9.0)
66
- redis-client (0.10.0)
65
+ redis-client (0.11.0)
67
66
  connection_pool
68
- regexp_parser (2.5.0)
67
+ regexp_parser (2.6.0)
69
68
  rexml (3.2.5)
70
- rspec (3.11.0)
71
- rspec-core (~> 3.11.0)
72
- rspec-expectations (~> 3.11.0)
73
- rspec-mocks (~> 3.11.0)
74
- rspec-core (3.11.0)
75
- rspec-support (~> 3.11.0)
76
- rspec-expectations (3.11.1)
69
+ rspec (3.12.0)
70
+ rspec-core (~> 3.12.0)
71
+ rspec-expectations (~> 3.12.0)
72
+ rspec-mocks (~> 3.12.0)
73
+ rspec-core (3.12.0)
74
+ rspec-support (~> 3.12.0)
75
+ rspec-expectations (3.12.0)
77
76
  diff-lcs (>= 1.2.0, < 2.0)
78
- rspec-support (~> 3.11.0)
79
- rspec-mocks (3.11.1)
77
+ rspec-support (~> 3.12.0)
78
+ rspec-mocks (3.12.0)
80
79
  diff-lcs (>= 1.2.0, < 2.0)
81
- rspec-support (~> 3.11.0)
82
- rspec-support (3.11.1)
83
- rubocop (1.36.0)
80
+ rspec-support (~> 3.12.0)
81
+ rspec-support (3.12.0)
82
+ rubocop (1.38.0)
84
83
  json (~> 2.3)
85
84
  parallel (~> 1.10)
86
85
  parser (>= 3.1.2.1)
87
86
  rainbow (>= 2.2.2, < 4.0)
88
87
  regexp_parser (>= 1.8, < 3.0)
89
88
  rexml (>= 3.2.5, < 4.0)
90
- rubocop-ast (>= 1.20.1, < 2.0)
89
+ rubocop-ast (>= 1.23.0, < 2.0)
91
90
  ruby-progressbar (~> 1.7)
92
91
  unicode-display_width (>= 1.4.0, < 3.0)
93
- rubocop-ast (1.21.0)
92
+ rubocop-ast (1.23.0)
94
93
  parser (>= 3.1.1.0)
95
94
  ruby-progressbar (1.11.0)
96
95
  tzinfo (2.0.5)
97
96
  concurrent-ruby (~> 1.0)
98
97
  unicode-display_width (2.3.0)
99
- zeitwerk (2.6.1)
100
98
 
101
99
  PLATFORMS
102
100
  ruby
data/lib/kraps/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Kraps
2
- VERSION = "0.1.0"
2
+ VERSION = "0.2.0"
3
3
  end
data/lib/kraps/worker.rb CHANGED
@@ -31,8 +31,12 @@ module Kraps
31
31
  mapper = MapReduce::Mapper.new(implementation.new, partitioner: partitioner, memory_limit: @memory_limit)
32
32
  mapper.map(@args["item"])
33
33
 
34
- mapper.shuffle do |partition, tempfile| # TODO: upload in parallel
35
- Kraps.driver.driver.store(Kraps.driver.with_prefix("#{@args["token"]}/#{partition}/chunk.#{@args["part"]}.json"), tempfile, Kraps.driver.bucket)
34
+ mapper.shuffle(chunk_limit: @chunk_limit) do |partitions|
35
+ Parallelizer.each(partitions.to_a, @concurrency) do |partition, path|
36
+ File.open(path) do |stream|
37
+ Kraps.driver.driver.store(Kraps.driver.with_prefix("#{@args["token"]}/#{partition}/chunk.#{@args["part"]}.json"), stream, Kraps.driver.bucket)
38
+ end
39
+ end
36
40
  end
37
41
  end
38
42
 
@@ -68,10 +72,14 @@ module Kraps
68
72
  end
69
73
  end
70
74
 
71
- mapper.shuffle do |partition, tempfile| # TODO: upload in parallel
72
- Kraps.driver.driver.store(
73
- Kraps.driver.with_prefix("#{@args["token"]}/#{partition}/chunk.#{@args["part"]}.json"), tempfile, Kraps.driver.bucket
74
- )
75
+ mapper.shuffle(chunk_limit: @chunk_limit) do |partitions|
76
+ Parallelizer.each(partitions.to_a, @concurrency) do |partition, path|
77
+ File.open(path) do |stream|
78
+ Kraps.driver.driver.store(
79
+ Kraps.driver.with_prefix("#{@args["token"]}/#{partition}/chunk.#{@args["part"]}.json"), stream, Kraps.driver.bucket
80
+ )
81
+ end
82
+ end
75
83
  end
76
84
  ensure
77
85
  temp_paths&.unlink
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: kraps
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Benjamin Vetter
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2022-10-28 00:00:00.000000000 Z
11
+ date: 2022-11-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: attachie
@@ -44,14 +44,14 @@ dependencies:
44
44
  requirements:
45
45
  - - ">="
46
46
  - !ruby/object:Gem::Version
47
- version: 2.1.1
47
+ version: 3.0.0
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
- version: 2.1.1
54
+ version: 3.0.0
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: redis
57
57
  requirement: !ruby/object:Gem::Requirement
@@ -132,6 +132,7 @@ extra_rdoc_files: []
132
132
  files:
133
133
  - ".rspec"
134
134
  - ".rubocop.yml"
135
+ - CHANGELOG.md
135
136
  - CODE_OF_CONDUCT.md
136
137
  - Gemfile
137
138
  - Gemfile.lock