map-reduce-ruby 1.0.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19ac70d25d6622d30398997cae0a4d6730ebc13f8bbc31df896c5acf480923c4
-  data.tar.gz: ede2a8f736053a268175a7948324a15954475526d0589bf47416347b57a50740
+  metadata.gz: 68967e02da6776738d27e48228374396dc613f4ee4a48f08268b71c92764eaa0
+  data.tar.gz: 0f0633adb6f2c51617ea3234284b30433da4a9ceb25d167ce8eaef2527dc09db
 SHA512:
-  metadata.gz: 650125350b5f166ccc0fb1346c77dfa9f17471ac9ae9290b4cbc4731a5baec872a810551e0b15af064b37f73c7e103ab6281d48cd839f26d4aab6780d5cd66d1
-  data.tar.gz: 70d9f8668143cb507537e369ad0ea79d9bb9f2c33469dffccd5952116f09758a7a880831d81aa4e3b67bc548b4b36aafe321a8d553946d8734fb78c11cd1686e
+  metadata.gz: d27e08793121dd81d4f8cc887f350d30ec9ea4039dad3490a4c7969164413cf575bdba0aa4c767097fc4af124aa3bb17b55f6781ff03919ef7d36164b3b14fa6
+  data.tar.gz: 71165ebbab647370de51c709b34f8e5cc8348300a41809354472ac125ef542255ec767a4613046cb964443b698fa7ec5dc8334d40df2a64728822aab0b170b43

data/.rubocop.yml

@@ -49,3 +49,9 @@ Layout/LineLength:
 
 Style/FrozenStringLiteralComment:
   EnforcedStyle: never
+
+Style/ObjectThen:
+  Enabled: false
+
+Gemspec/RequireMFA:
+  Enabled: false

data/CHANGELOG.md

@@ -1 +1,28 @@
 # CHANGELOG
+
+## v2.0.0
+
+* [BREAKING] Keys are no longer automatically converted to json before using
+  them for sorting
+  * This allows to have proper semantic sort order for numeric keys in addition
+    to just the clustering of keys
+  * Examples of valid keys: `"key"`, `["foo", 1.0]`, `["foo", ["bar"]]`
+  * Examples of problematic keys: `nil`, `true`, `["foo", nil]`, `{ "foo" => "bar" }`
+  * For migration purposes it is recommended to convert your keys to and from
+    json manually if you have complex keys using `JSON.generate`/`JSON.parse`:
+
+```ruby
+class WordCounter
+  def map(url)
+    HTTP.get(url).to_s.split.each do |word|
+      yield(JSON.generate("key" => word), 1) # if you use a hash for the key
+    end
+  end
+
+  def reduce(json_key, count1, count2)
+    key = JSON.parse(json_key) # if you want to access the original key
+
+    count1 + count2
+  end
+end
+```

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    map-reduce-ruby (1.0.0)
+    map-reduce-ruby (2.0.0)
       json
       lazy_priority_queue
 
@@ -9,41 +9,42 @@ GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
-    diff-lcs (1.4.4)
-    json (2.5.1)
+    diff-lcs (1.5.0)
+    json (2.6.2)
     lazy_priority_queue (0.1.1)
-    parallel (1.20.1)
-    parser (3.0.0.0)
+    parallel (1.22.1)
+    parser (3.1.2.1)
       ast (~> 2.4.1)
-    rainbow (3.0.0)
-    regexp_parser (2.0.3)
-    rexml (3.2.4)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    rainbow (3.1.1)
+    regexp_parser (2.5.0)
+    rexml (3.2.5)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.1)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.1)
-    rubocop (0.93.1)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.1)
+    rubocop (1.36.0)
+      json (~> 2.3)
       parallel (~> 1.10)
-      parser (>= 2.7.1.5)
+      parser (>= 3.1.2.1)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8)
-      rexml
-      rubocop-ast (>= 0.6.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.20.1, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (1.4.1)
-      parser (>= 2.7.1.5)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.21.0)
+      parser (>= 3.1.1.0)
     ruby-progressbar (1.11.0)
-    unicode-display_width (1.7.0)
+    unicode-display_width (2.3.0)
 
 PLATFORMS
   ruby

data/README.md

@@ -120,6 +120,26 @@ mappers are finished.
 
 That's it.
 
+## Limitations for Keys
+
+You have to make sure that your keys are properly sortable in ruby. Please
+note:
+
+```ruby
+"key" < nil # comparison of String with nil failed (ArgumentError)
+
+false < true # undefined method `<' for false:FalseClass (NoMethodError)
+
+1 > "key" # comparison of Integer with String failed (ArgumentError
+
+{ "key" => "value1" } < { "key" => "value2" } #=> false
+{ "key" => "value1" } > { "key" => "value2" } #=> false
+{ "key" => "value1" } <=> { "key" => "value2" } #=> nil
+```
+
+For those reasons, it is recommended to only use strings, numbers and arrays or
+a combination of those.
+
 ## Internals
 
 To fully understand the performance details, the following outlines the inner

data/lib/map_reduce/mapper.rb

@@ -96,7 +96,7 @@ module MapReduce
     def write_chunk
       tempfile = Tempfile.new
 
-      @buffer.sort_by! { |item| JSON.generate(item.first) }
+      @buffer.sort_by!(&:first)
 
       reduce_chunk(@buffer, @implementation).each do |pair|
         tempfile.puts JSON.generate(pair)

data/lib/map_reduce/mergeable.rb

@@ -29,7 +29,7 @@ module MapReduce
 
         key, value = JSON.parse(line)
 
-        queue.push([key, value, index], JSON.generate(key))
+        queue.push([key, value, index], key)
       end
 
       loop do
@@ -45,7 +45,7 @@ module MapReduce
 
         key, value = JSON.parse(line)
 
-        queue.push([key, value, index], JSON.generate(key))
+        queue.push([key, value, index], key)
       end
 
       files.each(&:rewind)

data/lib/map_reduce/priority_queue.rb

@@ -1,4 +1,26 @@
 module MapReduce
+  # Since LazyPriorityQueue is using <= and >=, but not <=>, it does not
+  # support sorting array keys. Therefore we wrap the keys in SortKey, which
+  # provides those operators. See https://bugs.ruby-lang.org/issues/5574
+
+  class SortKey
+    include Comparable
+
+    attr_reader :object
+
+    def initialize(object)
+      @object = object
+    end
+
+    def <=>(other)
+      res = object <=> other.object
+
+      raise(ArgumentError, "Unable to compare #{@object.inspect} with #{other.object.inspect}") if res.nil?
+
+      res
+    end
+  end
+
   # The MapReduce::PriorityQueue implements a min priority queue using a
   # binomial heap.
 
@@ -25,7 +47,7 @@ module MapReduce
     #   priority_queue.push("some object", "some key")
 
     def push(object, key)
-      @queue.push([@sequence_number, object], key)
+      @queue.push([@sequence_number, object], SortKey.new(key))
 
       @sequence_number += 1
     end

data/lib/map_reduce/version.rb

@@ -1,3 +1,3 @@
 module MapReduce
-  VERSION = "1.0.0"
+  VERSION = "2.0.0"
 end

data/map-reduce-ruby.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |spec|
   spec.email         = ["vetter@flakks.com"]
 
   spec.summary       = "The easiest way to write distributed, larger than memory map-reduce jobs"
-  spec.description   = "The MapReduce gem is the easiest way to write custom, distributed, larger "\
+  spec.description   = "The MapReduce gem is the easiest way to write custom, distributed, larger " \
                        "than memory map-reduce jobs"
   spec.homepage      = "https://github.com/mrkamel/map-reduce-ruby"
   spec.license       = "MIT"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: map-reduce-ruby
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Benjamin Vetter
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-07-05 00:00:00.000000000 Z
+date: 2022-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -104,7 +104,7 @@ metadata:
   homepage_uri: https://github.com/mrkamel/map-reduce-ruby
   source_code_uri: https://github.com/mrkamel/map-reduce-ruby
   changelog_uri: https://github.com/mrkamel/map-reduce/blob/master/CHANGELOG.md
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -119,8 +119,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.3.3
+signing_key:
 specification_version: 4
 summary: The easiest way to write distributed, larger than memory map-reduce jobs
 test_files: []