trafficbroker-mandy 0.1.6 → 0.1.7

data/lib/mappers/base_mapper.rb

@@ -27,7 +27,7 @@ module Mandy
     
       def emit(key, value=nil)
         key = 'nil' if key.nil?
-        @output.puts(value.nil? ? key.to_s : "#{key}\t#{value}")
+        @output.puts(value.nil? ? key.to_s : "#{serialize(key)}\t#{serialize(value)}")
       end
     
       private
@@ -35,6 +35,11 @@ module Mandy
       def mapper(key,value)
         #nil
       end
+      
+      def serialize(value)
+        value = ArraySerializer.new(value) if value.is_a?(Array)
+        value.to_s
+      end
     end
   end
 end

data/lib/reducers/base_reducer.rb

@@ -32,7 +32,7 @@ module Mandy
     
       def emit(key, value=nil)
         key = 'nil' if key.nil?
-        @output.puts(value.nil? ? key.to_s : "#{key}\t#{value}")
+        @output.puts(value.nil? ? key.to_s : "#{serialize(key)}\t#{serialize(value)}")
       end
     
       private
@@ -40,6 +40,11 @@ module Mandy
       def reducer(key,values)
         #nil
       end
+      
+      def serialize(value)
+        value = ArraySerializer.new(value) if value.is_a?(Array)
+        value.to_s
+      end
     end
   end
 end

data/lib/support/array_serializer.rb

@@ -3,6 +3,8 @@ module Mandy
     
     SEPERATOR = '|' unless defined?(SEPERATOR)
     
+    attr_reader :items
+    
     def initialize(items)
       @items = items || []
     end
@@ -11,6 +13,14 @@ module Mandy
       @items.join(SEPERATOR)
     end
     
+    def ==(other)
+      (self.class == other.class && self.items == other.items) || (other.is_a?(Array) && self.items == other)
+    end
+    
+    def to_a
+      @items
+    end
+    
     def self.from_s(str)
       str.split(SEPERATOR)
     end

data/lib/support/tuple.rb

@@ -5,8 +5,10 @@ module Mandy
     
     attr_accessor :name, :value
     
-    def initialize(name, value)
+    def initialize(name, value, name_accessor = nil, value_accessor = nil)
       @name, @value = name, value
+      alias_accessor(name_accessor, :name) unless name_accessor.nil?
+      alias_accessor(value_accessor, :value) unless value_accessor.nil?
     end
     
     def to_s
@@ -14,7 +16,9 @@ module Mandy
     end
     
     def self.from_s(str)
-      new(*str.split(SEPERATOR))
+      parts = str.split(SEPERATOR)
+      raise "Can't create tuple from #{str.inspect}. Format should be 'A#{SEPERATOR}B'" unless parts.size==2
+      new(*parts)
     end
     
     def inspect
@@ -25,5 +29,12 @@ module Mandy
       return false unless self.class == other.class
       self.name == other.name && self.value == other.value
     end
+    
+    private
+    
+    def alias_accessor(new_accessor, old_accessor)
+      self.class.send(:alias_method, new_accessor, old_accessor)
+      self.class.send(:alias_method, :"#{new_accessor}=", :"#{old_accessor}=")
+    end
   end
 end

data/lib/test_runner.rb

@@ -1,25 +1,31 @@
 module Mandy
   class TestRunner
+    attr_reader :job
+    
     def initialize(job=Mandy::Job.jobs.first.name)
       @job = Mandy::Job.find_by_name(job)
     end
     
-    def map(input, output_stream=StringIO.new(''), &blk)
-      input = input_from_array(input) if input.is_a?(Array)
-      input_stream = StringIO.new(input.to_s)
+    def map(input_stream, output_stream=StringIO.new(''), &blk)
+      input_stream = input_from_array(input_stream) if input_stream.is_a?(Array)
+      input_stream = StringIO.new(input_stream.to_s) unless input_stream.is_a?(StringIO)
       @job.run_map(input_stream, output_stream, &blk)
       output_stream.rewind
       output_stream
     end
     
-    def reduce(input, output_stream=StringIO.new(''), &blk)
-      input = input_from_hash(input) if input.is_a?(Hash)
-      input_stream = StringIO.new(input.to_s)
+    def reduce(input_stream, output_stream=StringIO.new(''), &blk)
+      input_stream = input_from_hash(input_stream) if input_stream.is_a?(Hash)
+      input_stream = StringIO.new(input_stream.to_s) unless input_stream.is_a?(StringIO)
       @job.run_reduce(input_stream, output_stream, &blk)
       output_stream.rewind
       output_stream
     end
     
+    def self.end_to_end(verbose=false)
+      CompositeJobRunner.new(Mandy::Job.jobs,verbose)
+    end
+    
     private
     
     def input_from_array(input)
@@ -34,5 +40,35 @@ module Mandy
       end
       input_from_array(output.sort)
     end
+    
+    class CompositeJobRunner
+      def initialize(jobs, verbose=false)
+        @jobs = jobs
+        @verbose = verbose
+        @job_runners = @jobs.map { |job| Mandy::TestRunner.new(job.name) }
+      end
+      
+      def execute(input_stream, output_stream=StringIO.new(''))
+        map_temp = StringIO.new('')
+        reduce_temp = StringIO.new('')
+        @job_runners.each_with_index do |runner, index|
+          runner.map(input_stream, map_temp)
+          if @verbose
+            puts "#{runner.job.name} [MAP] #{map_temp.readlines.inspect}"
+            map_temp.rewind
+          end
+          reduce_input = StringIO.new(map_temp.readlines.sort.join(''))
+          runner.reduce(reduce_input, (index==@job_runners.size-1 ? output_stream : reduce_temp))
+          if @verbose
+            puts "#{runner.job.name} [RED] #{reduce_temp.readlines.inspect}"
+            reduce_temp.rewind
+          end
+          input_stream = reduce_temp
+          map_temp = StringIO.new('')
+          reduce_temp = StringIO.new('')
+        end
+        output_stream
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: trafficbroker-mandy
 version: !ruby/object:Gem::Version 
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors: 
 - Andy Kent