trafficbroker-mandy 0.2.12 → 0.2.13

data/lib/job.rb

@@ -26,6 +26,15 @@ module Mandy
     def mixin(*modules)
       modules.each {|m| @modules << m}
     end
+    alias_method :serialize, :mixin
+    
+    def input_format(format)
+      @input_format = format
+    end
+
+    def output_format(format)
+      @output_format = format
+    end
     
     def set(key, value)
       @settings[key.to_s] = value.to_s
@@ -61,15 +70,23 @@ module Mandy
     end
     
     def run_map(input=STDIN, output=STDOUT, &blk)
-      mapper = @mapper_class.new(input, output)
+      @mapper_class.send(:include, Mandy::IO::OutputFormatting) unless reducer_defined?
+      mapper = @mapper_class.new(input, output, @input_format, @output_format)
       yield(mapper) if blk
       mapper.execute
     end
     
     def run_reduce(input=STDIN, output=STDOUT, &blk)
-      reducer = @reducer_class.new(input, output)
+      reducer = @reducer_class.new(input, output, @input_format, @output_format)
       yield(reducer) if blk
       reducer.execute
     end
+    
+    private
+    
+    def reducer_defined?
+      @reducer_class != Mandy::Reducers::PassThroughReducer
+    end
+    
   end
 end

data/lib/mandy.rb

@@ -1,4 +1,9 @@
+require "rubygems"
+require "json"
+require "uri"
+
 %w(
+  support/formatting
   task
   dsl 
   job 
@@ -14,16 +19,13 @@
   reducers/max_reducer 
   reducers/min_reducer
   reducers/transpose_reducer
+  serializers/json
   stores/hbase
   stores/in_memory
   test_runner
   ruby-hbase
 ).each {|file| require File.join(File.dirname(__FILE__), file) }
 
-require "rubygems"
-require "json"
-require "uri"
-
 module Mandy
   class << self
     def stores

data/lib/mappers/base_mapper.rb

@@ -1,6 +1,8 @@
 module Mandy
   module Mappers
     class Base < Mandy::Task
+      include Mandy::IO::InputFormatting
+
       def self.compile(&blk)
         Class.new(Mandy::Mappers::Base) do 
           self.class_eval do
@@ -14,10 +16,10 @@ module Mandy
            key, value = line.split(KEY_VALUE_SEPERATOR, 2)
            key, value = nil, key if value.nil?
            value.chomp!
-           mapper(key, value)
+           mapper(input_deserialize_key(key), input_deserialize_value(value))
         end
       end
-      
+
       private
     
       def mapper(key,value)

data/lib/reducers/base_reducer.rb

@@ -1,6 +1,8 @@
 module Mandy
   module Reducers
     class Base < Mandy::Task
+      include Mandy::IO::OutputFormatting
+
       def self.compile(&blk)
         Class.new(Mandy::Reducers::Base) do 
           self.class_eval do
@@ -21,11 +23,11 @@ module Mandy
            end
            values << value
         end
-        reducer(last_key, values)
+        reducer(deserialize_key(last_key), values.map {|v| deserialize_value(v) })
       end
     
       private
-    
+      
       def reducer(key,values)
         #nil
       end

data/lib/serializers/json.rb

@@ -0,0 +1,13 @@
+module Mandy
+  module Serializers
+    module Json
+      def serialize_value(value)
+        value.to_json
+      end
+      
+      def deserialize_value(value)
+        JSON.parse(value)
+      end
+    end
+  end
+end

data/lib/support/formatting.rb

@@ -0,0 +1,27 @@
+module Mandy
+  module IO
+    module InputFormatting
+      def input_deserialize_key(key)
+        return key if input_format && input_format == :plain
+        deserialize_key(key)
+      end
+      
+      def input_deserialize_value(value)
+        return value if input_format && input_format == :plain
+        deserialize_value(value)
+      end
+    end
+    
+    module OutputFormatting
+      def output_serialize_key(key)
+        return key if output_format && output_format == :plain
+        serialize_key(key)
+      end
+      
+      def output_serialize_value(value)
+        return value if output_format && output_format == :plain
+        serialize_value(value)
+      end
+    end
+  end
+end

data/lib/task.rb

@@ -3,18 +3,19 @@ module Mandy
     JSON_PAYLOAD_KEY = "json"
     KEY_VALUE_SEPERATOR = "\t" unless defined?(KEY_VALUE_SEPERATOR)
     NUMERIC_PADDING = 16
-
-    def initialize(input=STDIN, output=STDOUT)
+    
+    attr_reader :input_format, :output_format
+    
+    def initialize(input=STDIN, output=STDOUT, input_format = nil, output_format = nil)
       @input, @output = input, output
+      @input_format, @output_format = input_format, output_format
     end
 
     def emit(key, value=nil)
       key = 'nil' if key.nil?
-      key = pad(key) if key.is_a?(Numeric) && key.to_s.length < NUMERIC_PADDING
-      
-      @output.puts(value.nil? ? key.to_s : "#{serialize(key)}\t#{serialize(value)}")
+      @output.puts(value.nil? ? key.to_s : "#{output_serialize_key(key)}\t#{output_serialize_value(value)}")
     end
-
+    
     def get(store, key)
       Mandy.stores[store].get(key)
     end
@@ -43,10 +44,32 @@ module Mandy
     def json_provided?
       !ENV[JSON_PAYLOAD_KEY].nil?
     end
+    
+    def deserialize_key(key)
+      key
+    end
+    
+    def deserialize_value(value)
+      value
+    end
+    
+    def serialize_key(key)
+      key = pad(key) if key.is_a?(Numeric) && key.to_s.length < NUMERIC_PADDING
+      key
+    end
 
-    def serialize(value)
+    def serialize_value(value)
       value = ArraySerializer.new(value) if value.is_a?(Array)
       value.to_s
     end
+    
+    def output_serialize_key(key)
+      serialize_key(key)
+    end
+
+    def output_serialize_value(value)
+      serialize_value(value)
+    end
+    
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: trafficbroker-mandy
 version: !ruby/object:Gem::Version 
-  version: 0.2.12
+  version: 0.2.13
 platform: ruby
 authors: 
 - Andy Kent
@@ -40,6 +40,7 @@ files:
 - Rakefile
 - lib/mandy.rb
 - lib/support/tuple.rb
+- lib/support/formatting.rb
 - lib/support/array_serializer.rb
 - lib/task.rb
 - lib/dsl.rb
@@ -54,6 +55,7 @@ files:
 - lib/reducers/sum_reducer.rb
 - lib/reducers/max_reducer.rb
 - lib/reducers/min_reducer.rb
+- lib/serializers/json.rb
 - lib/stores/hbase.rb
 - lib/stores/in_memory.rb
 - lib/ruby-hbase.rb