kraps 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 57d06d20f406e72e26424dcba5c2af296a1551085113371c2cae9894f18e72ff
-  data.tar.gz: a25c8c3440cdd26eeb9b32655e77556e4be4966d47bd3e290d0a702c3b4cde9f
+  metadata.gz: f5bb601e7ee415b95b4b258a0241c25e6fe19eb3e772c06d4149afbfcfbe6c3d
+  data.tar.gz: cb948c05947e48d2d8e970eebbc6e2c4a5b0a88cb162ad87bf0743196f6bcaef
 SHA512:
-  metadata.gz: 655e2d0f525e136b72e87231a8d359eda952b7e3b30a8cd38b25f7b4fba27baebedacced0a00071915166325b621437869026c6785ab7353eea928ca736bc2a7
-  data.tar.gz: 0d450b3032a6809d2a5300e18e3cfc9a948ca5f589f1161823d9923084939d249282d28bf0619c3b9ace10dc742118dfcf5895c898e2d9fe7c55f98d00e683b4
+  metadata.gz: 1d1c5a16205c5584626fed5bca9b6c7dd6fae3b4f3c725b158e7740f6fa05a17abdcb483b43cbdad813576e2fc2c7621b89b94d61b32776d85ae774f5a4332d1
+  data.tar.gz: 2670dbc002633e801d8cf98fc8454c8881295f72b505bd4baf6cf0c8685a8c97a8a2dbf26e8a617c74b452ef627e820807e7af6e05b20a627fb99ce2eb216a1a

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # CHANGELOG
 
+## v0.3.0
+
+* Changed partitioners to receive the number of partitions
+  as second parameter
+
 ## v0.2.0
 
 * Updated map-reduce-ruby to allow concurrent uploads

data/README.md

@@ -143,17 +143,18 @@ split. Kraps assigns every `key` to a partition, either using a custom
 `partitioner` or the default built in hash partitioner. The hash partitioner
 simply calculates a hash of your key modulo the number of partitions and the
 resulting partition number is the partition where the respective key is
-assigned to. A partitioner is a callable which gets the key as argument and
-returns a partition number. The built in hash partitioner looks similar to this
-one:
+assigned to. A partitioner is a callable which gets the key and the number of
+partitions as argument and returns a partition number. The built in hash
+partitioner looks similar to this one:
 
 ```ruby
-partitioner = proc { |key| Digest::SHA1.hexdigest(key.inspect)[0..4].to_i(16) % 128 } # 128 partitions
+partitioner = proc { |key, num_partitions| Digest::SHA1.hexdigest(key.inspect)[0..4].to_i(16) % num_partitions }
 ```
 
 Please note, it's important that the partitioner and the specified number of
 partitions stays in sync. When you use a custom partitioner, please make sure
-that the partitioner operates on the same number of partitions you specify.
+that the partitioner correctly returns a partition number in the range of
+`0...num_partitions`.
 
 ## Datatypes
 

data/lib/kraps/hash_partitioner.rb

@@ -0,0 +1,7 @@
+module Kraps
+  class HashPartitioner
+    def call(key, num_partitions)
+      Digest::SHA1.hexdigest(JSON.generate(key))[0..4].to_i(16) % num_partitions
+    end
+  end
+end

data/lib/kraps/job.rb

@@ -6,10 +6,10 @@ module Kraps
       @worker = worker
       @steps = []
       @partitions = 0
-      @partitioner = MapReduce::HashPartitioner.new(@partitions)
+      @partitioner = HashPartitioner.new
     end
 
-    def parallelize(partitions:, partitioner: MapReduce::HashPartitioner.new(partitions), worker: @worker, &block)
+    def parallelize(partitions:, partitioner: HashPartitioner.new, worker: @worker, &block)
       fresh.tap do |job|
         job.instance_eval do
           @partitions = partitions
@@ -24,7 +24,7 @@ module Kraps
       fresh.tap do |job|
         job.instance_eval do
           @partitions = partitions if partitions
-          @partitioner = partitioner || MapReduce::HashPartitioner.new(partitions) if partitioner || partitions
+          @partitioner = partitioner if partitioner
 
           @steps << Step.new(action: Actions::MAP, args: { partitions: @partitions, partitioner: @partitioner, worker: worker }, block: block)
         end

data/lib/kraps/version.rb

@@ -1,3 +1,3 @@
 module Kraps
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/kraps/worker.rb

@@ -181,7 +181,7 @@ module Kraps
     end
 
     def partitioner
-      @partitioner ||= step.args[:partitioner]
+      @partitioner ||= proc { |key| step.args[:partitioner].call(key, step.args[:partitions]) }
     end
 
     def distributed_job

data/lib/kraps.rb

@@ -2,6 +2,7 @@ require_relative "kraps/version"
 require_relative "kraps/drivers"
 require_relative "kraps/actions"
 require_relative "kraps/parallelizer"
+require_relative "kraps/hash_partitioner"
 require_relative "kraps/temp_path"
 require_relative "kraps/temp_paths"
 require_relative "kraps/timeout_queue"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kraps
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Benjamin Vetter
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-11-01 00:00:00.000000000 Z
+date: 2022-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: attachie
@@ -144,6 +144,7 @@ files:
 - lib/kraps/actions.rb
 - lib/kraps/drivers.rb
 - lib/kraps/frame.rb
+- lib/kraps/hash_partitioner.rb
 - lib/kraps/interval.rb
 - lib/kraps/job.rb
 - lib/kraps/parallelizer.rb