kraps 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d1f08b6fa0f725c63e3750f4b3bf04479622b40160a6364f708d91d37c0b1948
-  data.tar.gz: '0681d837d852846cc6c115fe9dae3075a0e5b3bb8b8eae2d90d8a23ec26581e3'
+  metadata.gz: f5bb601e7ee415b95b4b258a0241c25e6fe19eb3e772c06d4149afbfcfbe6c3d
+  data.tar.gz: cb948c05947e48d2d8e970eebbc6e2c4a5b0a88cb162ad87bf0743196f6bcaef
 SHA512:
-  metadata.gz: 354ab3129ef1713c8229af54945251069c98d681e2db5c716d93b5925576b601751c23c1d502cb72ddaea5df5fc91e6eceb4590a619de730ae65f0762662da21
-  data.tar.gz: 0647fc85f445bc634f70e2e10feab325c3df6aec3a30d2af4b1a792e82b9adf1a31e8bb339465e10944bce927933956e8d88e72b37fb7d84027ec569441781d6
+  metadata.gz: 1d1c5a16205c5584626fed5bca9b6c7dd6fae3b4f3c725b158e7740f6fa05a17abdcb483b43cbdad813576e2fc2c7621b89b94d61b32776d85ae774f5a4332d1
+  data.tar.gz: 2670dbc002633e801d8cf98fc8454c8881295f72b505bd4baf6cf0c8685a8c97a8a2dbf26e8a617c74b452ef627e820807e7af6e05b20a627fb99ce2eb216a1a

data/.rubocop.yml

@@ -77,3 +77,6 @@ Lint/EmptyClass:
 
 Style/WordArray:
   Enabled: false
+
+Style/RedundantEach:
+  Enabled: false

data/CHANGELOG.md

@@ -0,0 +1,10 @@
+# CHANGELOG
+
+## v0.3.0
+
+* Changed partitioners to receive the number of partitions
+  as second parameter
+
+## v0.2.0
+
+* Updated map-reduce-ruby to allow concurrent uploads

data/Gemfile.lock

@@ -1,22 +1,21 @@
 PATH
   remote: .
   specs:
-    kraps (0.1.0)
+    kraps (0.2.0)
       attachie
       distributed_job
-      map-reduce-ruby (>= 2.1.1)
+      map-reduce-ruby (>= 3.0.0)
       redis
       ruby-progressbar
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.7)
+    activesupport (7.0.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
     ast (2.4.2)
     attachie (1.2.0)
       activesupport
@@ -24,17 +23,17 @@ GEM
       connection_pool
       mime-types
     aws-eventstream (1.2.0)
-    aws-partitions (1.649.0)
-    aws-sdk-core (3.164.0)
+    aws-partitions (1.654.0)
+    aws-sdk-core (3.166.0)
       aws-eventstream (~> 1, >= 1.0.2)
-      aws-partitions (~> 1, >= 1.525.0)
-      aws-sigv4 (~> 1.1)
+      aws-partitions (~> 1, >= 1.651.0)
+      aws-sigv4 (~> 1.5)
       jmespath (~> 1, >= 1.6.1)
-    aws-sdk-kms (1.58.0)
-      aws-sdk-core (~> 3, >= 3.127.0)
+    aws-sdk-kms (1.59.0)
+      aws-sdk-core (~> 3, >= 3.165.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.116.0)
-      aws-sdk-core (~> 3, >= 3.127.0)
+    aws-sdk-s3 (1.117.1)
+      aws-sdk-core (~> 3, >= 3.165.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.4)
     aws-sigv4 (1.5.2)
@@ -49,7 +48,7 @@ GEM
     jmespath (1.6.1)
     json (2.6.2)
     lazy_priority_queue (0.1.1)
-    map-reduce-ruby (2.1.1)
+    map-reduce-ruby (3.0.0)
       json
       lazy_priority_queue
     mime-types (3.4.1)
@@ -63,40 +62,39 @@ GEM
     rake (13.0.6)
     redis (5.0.5)
       redis-client (>= 0.9.0)
-    redis-client (0.10.0)
+    redis-client (0.11.0)
       connection_pool
-    regexp_parser (2.5.0)
+    regexp_parser (2.6.0)
     rexml (3.2.5)
-    rspec (3.11.0)
-      rspec-core (~> 3.11.0)
-      rspec-expectations (~> 3.11.0)
-      rspec-mocks (~> 3.11.0)
-    rspec-core (3.11.0)
-      rspec-support (~> 3.11.0)
-    rspec-expectations (3.11.1)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.0)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-mocks (3.11.1)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-support (3.11.1)
-    rubocop (1.36.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    rubocop (1.38.0)
       json (~> 2.3)
       parallel (~> 1.10)
       parser (>= 3.1.2.1)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.20.1, < 2.0)
+      rubocop-ast (>= 1.23.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.21.0)
+    rubocop-ast (1.23.0)
       parser (>= 3.1.1.0)
     ruby-progressbar (1.11.0)
     tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.3.0)
-    zeitwerk (2.6.1)
 
 PLATFORMS
   ruby

data/README.md

@@ -143,17 +143,18 @@ split. Kraps assigns every `key` to a partition, either using a custom
 `partitioner` or the default built in hash partitioner. The hash partitioner
 simply calculates a hash of your key modulo the number of partitions and the
 resulting partition number is the partition where the respective key is
-assigned to. A partitioner is a callable which gets the key as argument and
-returns a partition number. The built in hash partitioner looks similar to this
-one:
+assigned to. A partitioner is a callable which gets the key and the number of
+partitions as argument and returns a partition number. The built in hash
+partitioner looks similar to this one:
 
 ```ruby
-partitioner = proc { |key| Digest::SHA1.hexdigest(key.inspect)[0..4].to_i(16) % 128 } # 128 partitions
+partitioner = proc { |key, num_partitions| Digest::SHA1.hexdigest(key.inspect)[0..4].to_i(16) % num_partitions }
 ```
 
 Please note, it's important that the partitioner and the specified number of
 partitions stays in sync. When you use a custom partitioner, please make sure
-that the partitioner operates on the same number of partitions you specify.
+that the partitioner correctly returns a partition number in the range of
+`0...num_partitions`.
 
 ## Datatypes
 

data/lib/kraps/hash_partitioner.rb

@@ -0,0 +1,7 @@
+module Kraps
+  class HashPartitioner
+    def call(key, num_partitions)
+      Digest::SHA1.hexdigest(JSON.generate(key))[0..4].to_i(16) % num_partitions
+    end
+  end
+end

data/lib/kraps/job.rb

@@ -6,10 +6,10 @@ module Kraps
       @worker = worker
       @steps = []
       @partitions = 0
-      @partitioner = MapReduce::HashPartitioner.new(@partitions)
+      @partitioner = HashPartitioner.new
     end
 
-    def parallelize(partitions:, partitioner: MapReduce::HashPartitioner.new(partitions), worker: @worker, &block)
+    def parallelize(partitions:, partitioner: HashPartitioner.new, worker: @worker, &block)
       fresh.tap do |job|
         job.instance_eval do
           @partitions = partitions
@@ -24,7 +24,7 @@ module Kraps
       fresh.tap do |job|
         job.instance_eval do
           @partitions = partitions if partitions
-          @partitioner = partitioner || MapReduce::HashPartitioner.new(partitions) if partitioner || partitions
+          @partitioner = partitioner if partitioner
 
           @steps << Step.new(action: Actions::MAP, args: { partitions: @partitions, partitioner: @partitioner, worker: worker }, block: block)
         end

data/lib/kraps/version.rb

@@ -1,3 +1,3 @@
 module Kraps
-  VERSION = "0.1.0"
+  VERSION = "0.3.0"
 end

data/lib/kraps/worker.rb

@@ -31,8 +31,12 @@ module Kraps
       mapper = MapReduce::Mapper.new(implementation.new, partitioner: partitioner, memory_limit: @memory_limit)
       mapper.map(@args["item"])
 
-      mapper.shuffle do |partition, tempfile| # TODO: upload in parallel
-        Kraps.driver.driver.store(Kraps.driver.with_prefix("#{@args["token"]}/#{partition}/chunk.#{@args["part"]}.json"), tempfile, Kraps.driver.bucket)
+      mapper.shuffle(chunk_limit: @chunk_limit) do |partitions|
+        Parallelizer.each(partitions.to_a, @concurrency) do |partition, path|
+          File.open(path) do |stream|
+            Kraps.driver.driver.store(Kraps.driver.with_prefix("#{@args["token"]}/#{partition}/chunk.#{@args["part"]}.json"), stream, Kraps.driver.bucket)
+          end
+        end
       end
     end
 
@@ -68,10 +72,14 @@ module Kraps
         end
       end
 
-      mapper.shuffle do |partition, tempfile| # TODO: upload in parallel
-        Kraps.driver.driver.store(
-          Kraps.driver.with_prefix("#{@args["token"]}/#{partition}/chunk.#{@args["part"]}.json"), tempfile, Kraps.driver.bucket
-        )
+      mapper.shuffle(chunk_limit: @chunk_limit) do |partitions|
+        Parallelizer.each(partitions.to_a, @concurrency) do |partition, path|
+          File.open(path) do |stream|
+            Kraps.driver.driver.store(
+              Kraps.driver.with_prefix("#{@args["token"]}/#{partition}/chunk.#{@args["part"]}.json"), stream, Kraps.driver.bucket
+            )
+          end
+        end
       end
     ensure
       temp_paths&.unlink
@@ -173,7 +181,7 @@ module Kraps
     end
 
     def partitioner
-      @partitioner ||= step.args[:partitioner]
+      @partitioner ||= proc { |key| step.args[:partitioner].call(key, step.args[:partitions]) }
     end
 
     def distributed_job

data/lib/kraps.rb

@@ -2,6 +2,7 @@ require_relative "kraps/version"
 require_relative "kraps/drivers"
 require_relative "kraps/actions"
 require_relative "kraps/parallelizer"
+require_relative "kraps/hash_partitioner"
 require_relative "kraps/temp_path"
 require_relative "kraps/temp_paths"
 require_relative "kraps/timeout_queue"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kraps
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Benjamin Vetter
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-10-28 00:00:00.000000000 Z
+date: 2022-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: attachie
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.1
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.1
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -132,6 +132,7 @@ extra_rdoc_files: []
 files:
 - ".rspec"
 - ".rubocop.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
@@ -143,6 +144,7 @@ files:
 - lib/kraps/actions.rb
 - lib/kraps/drivers.rb
 - lib/kraps/frame.rb
+- lib/kraps/hash_partitioner.rb
 - lib/kraps/interval.rb
 - lib/kraps/job.rb
 - lib/kraps/parallelizer.rb