map-reduce-ruby 2.0.0 → 2.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 68967e02da6776738d27e48228374396dc613f4ee4a48f08268b71c92764eaa0
-  data.tar.gz: 0f0633adb6f2c51617ea3234284b30433da4a9ceb25d167ce8eaef2527dc09db
+  metadata.gz: 5545309a188291db41e8f5fc24af45a8d983c5084f2233d735cab309921c928c
+  data.tar.gz: 779a839704ace3780a304bc7295c8b1f27e834253ee0544e9ff6ae21eda93753
 SHA512:
-  metadata.gz: d27e08793121dd81d4f8cc887f350d30ec9ea4039dad3490a4c7969164413cf575bdba0aa4c767097fc4af124aa3bb17b55f6781ff03919ef7d36164b3b14fa6
-  data.tar.gz: 71165ebbab647370de51c709b34f8e5cc8348300a41809354472ac125ef542255ec767a4613046cb964443b698fa7ec5dc8334d40df2a64728822aab0b170b43
+  metadata.gz: eb577347b7e5c09dd34166e814fc0c50180b6f036fad84de3857dc93d28242be81637d5b0c7f19ea7a846c659eca0c12fcbae01cdac37c4f2bf50c6d9f8f27f6
+  data.tar.gz: 704b5d6a140583099c53902ceaab5af7b45c41c004f159fc215a942a4749063bf0990c2d60cde15af9663da726bab03a1258e085c0e789470150ab96caf895f7

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # CHANGELOG
 
+## v2.1.1
+
+* Fix in `MapReduce::Mapper` when no `reduce` implementation is given
+
+## v2.1.0
+
+* Do not reduce in `MapReduce::Mapper` when no `reduce` implementation is given
+
 ## v2.0.0
 
 * [BREAKING] Keys are no longer automatically converted to json before using

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    map-reduce-ruby (2.0.0)
+    map-reduce-ruby (2.1.1)
       json
       lazy_priority_queue
 

data/README.md

@@ -7,8 +7,7 @@ than memory map-reduce jobs by using your local disk and some arbitrary storage
 layer like s3. You can specify how much memory you are willing to offer and
 MapReduce will use its buffers accordingly. Finally, you can use your already
 existing background job system like `sidekiq` or one of its various
-alternatives. Finally, your keys and values can be everything that can be
-serialized as json.
+alternatives.
 
 ## Installation
 
@@ -30,9 +29,7 @@ Or install it yourself as:
 
 Any map-reduce job consists of an implementation of your `map` function, your
 `reduce` function and worker code. So let's start with an implementation for a
-word count map-reduce task which fetches txt documents from the web. Please
-note that your keys and values can be everything that can be serialized as
-json, but nothing else.
+word count map-reduce task which fetches txt documents from the web.
 
 ```ruby
 class WordCounter
@@ -68,8 +65,8 @@ class WordCountMapper
 end
 ```
 
-Please note that `MapReduce::HashPartitioner.new(16)` states that we want split
-the dataset into 16 partitions (i.e. 0, 1, ... 15). Finally, we need some
+Please note that `MapReduce::HashPartitioner.new(16)` states that we want to
+split the dataset into 16 partitions (i.e. 0, 1, ... 15). Finally, we need some
 worker code to run the reduce part:
 
 ```ruby

data/lib/map_reduce/mapper.rb

@@ -71,7 +71,10 @@ module MapReduce
 
       partitions = {}
 
-      reduce_chunk(k_way_merge(@chunks), @implementation).each do |pair|
+      chunk = k_way_merge(@chunks)
+      chunk = reduce_chunk(chunk, @implementation) if @implementation.respond_to?(:reduce)
+
+      chunk.each do |pair|
         partition = @partitioner.call(pair[0])
 
         (partitions[partition] ||= Tempfile.new).puts(JSON.generate(pair))
@@ -98,7 +101,10 @@ module MapReduce
 
       @buffer.sort_by!(&:first)
 
-      reduce_chunk(@buffer, @implementation).each do |pair|
+      chunk = @buffer
+      chunk = reduce_chunk(chunk, @implementation) if @implementation.respond_to?(:reduce)
+
+      chunk.each do |pair|
         tempfile.puts JSON.generate(pair)
       end
 

data/lib/map_reduce/mergeable.rb

@@ -20,6 +20,16 @@ module MapReduce
     def k_way_merge(files)
       return enum_for(:k_way_merge, files) unless block_given?
 
+      if files.size == 1
+        files.first.each_line do |line|
+          yield(JSON.parse(line))
+        end
+
+        files.each(&:rewind)
+
+        return
+      end
+
       queue = PriorityQueue.new
 
       files.each_with_index do |file, index|

data/lib/map_reduce/version.rb

@@ -1,3 +1,3 @@
 module MapReduce
-  VERSION = "2.0.0"
+  VERSION = "2.1.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: map-reduce-ruby
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.1
 platform: ruby
 authors:
 - Benjamin Vetter
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-09-22 00:00:00.000000000 Z
+date: 2022-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec