map_reduce 0.0.2 → 0.0.3

data/lib/map_reduce/map_log.rb

@@ -2,15 +2,17 @@ module MapReduce
   class MapLog
     MAX_BUFFER_SIZE = 2 ** 21 # 2 MB
 
+    attr_reader :log_folder
+
     def initialize(log_folder, task)
-      @log_folder = log_folder
-      @task = task
+      @task = task || "default"
+      @log_folder = File.join(log_folder, @task)
       @log = ""
       @log_size = 0
     end
 
     def <<(str)
-      @log_size += str.size
+      @log_size += str.bytesize
       @log << str << "\n"
       flush  if @log_size >= MAX_BUFFER_SIZE
     end
@@ -26,17 +28,19 @@ module MapReduce
     def reset
       flush
       if @log_file
-        fn = File.path(@log_file)
+        map_fn = File.path(@log_file)
         @log_file.close
+        reduce_fn = File.join(@log_folder, "reducer.log")
+        File.rename(map_fn, reduce_fn)
         @log_file = nil
-        fn
+        reduce_fn
       end
     end
 
     def log_file
       @log_file ||= begin
         begin
-          fn = File.join(@log_folder, "map_#{@task}_#{Time.now.to_i}_#{Process.pid}_#{rand(1000)}.log")
+          fn = File.join(@log_folder, "mapper.log")
         end while File.exist?(fn)
         FileUtils.mkdir_p(@log_folder)
         File.open(fn, "a")

data/lib/map_reduce/reduce_log.rb

@@ -23,7 +23,7 @@ module MapReduce
             end
           rescue StopIteration => e
             @file.close
-            FileUtils.rm(File.path(@file))
+            File.unlink(File.path(@file))
             @lines = nil
             break
           end
@@ -43,11 +43,15 @@ module MapReduce
     end
 
     def log_file
-      fn = @map_log.reset
-      if fn
-        @more = true
-        sort(fn)
-        fn
+      reduce_fn = File.join(@map_log.log_folder, "reducer.log")
+      if File.exist? reduce_fn
+        reduce_fn
+      else
+        reduce_fn = @map_log.reset
+        if reduce_fn
+          sort(reduce_fn)
+          reduce_fn
+        end
       end
     end
 

data/lib/map_reduce/version.rb

@@ -1,3 +1,3 @@
 module MapReduce
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/spec/map_reduce/map_reduce_spec.rb

@@ -4,7 +4,8 @@ describe "MapReduce stack" do
   describe "single master" do
     before do
       @pid1 = fork do
-        master = MapReduce::Master.new socket: "tcp://127.0.0.1:15555"
+        master = MapReduce::Master.new socket: "tcp://127.0.0.1:15555", 
+                                       log_folder: "/tmp/map_reduce/master1"
         trap("SIGINT") do
           master.stop
           exit
@@ -12,7 +13,8 @@ describe "MapReduce stack" do
         master.run
       end
       @pid2 = fork do
-        master = MapReduce::Master.new socket: "tcp://127.0.0.1:15556"
+        master = MapReduce::Master.new socket: "tcp://127.0.0.1:15556", 
+                                       log_folder: "/tmp/map_reduce/master2"
         trap("SIGINT") do
           master.stop
           exit
@@ -94,7 +96,7 @@ describe "MapReduce stack" do
               end
             end
             data.sort.must_equal (0...100).to_a.map(&:to_s).sort
-            
+
             EM.stop
           end.resume
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: map_reduce
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-04 00:00:00.000000000 Z
+date: 2013-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -117,7 +117,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1253589631458738880
+      hash: 3976831796496443677
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -126,7 +126,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1253589631458738880
+      hash: 3976831796496443677
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25