trafficbroker-mandy 0.1.7 → 0.1.8

data/lib/job.rb

@@ -34,6 +34,15 @@ module Mandy
       set('mapred.reduce.tasks', count)
     end
     
+    def store(type, name, options={})
+      Mandy.stores[name] = case type
+      when :hbase
+        Stores::HBase.new(options)
+      else
+        raise "Unknown store type #{type}"
+      end
+    end
+    
     def map(klass=nil, &blk)
       @mapper_class = klass || Mandy::Mappers::Base.compile(&blk)
     end

data/lib/mandy.rb

@@ -1,4 +1,7 @@
 %w(
+  task
+  dsl 
+  job 
   support/tuple 
   support/array_serializer 
   mappers/base_mapper 
@@ -7,8 +10,17 @@
   reducers/pass_through_reducer 
   reducers/sum_reducer 
   reducers/max_reducer 
-  reducers/min_reducer 
-  dsl 
-  job 
+  reducers/min_reducer
+  stores/hbase
+  stores/in_memory
   test_runner
-).each {|file| require File.join(File.dirname(__FILE__), file) }
+  ruby-hbase
+).each {|file| require File.join(File.dirname(__FILE__), file) }
+
+module Mandy
+  class << self
+    def stores
+      @stores||={}
+    end
+  end
+end

data/lib/mappers/base_mapper.rb

@@ -1,13 +1,6 @@
 module Mandy
   module Mappers
-    class Base
-      
-      KEY_VALUE_SEPERATOR = "\t" unless defined?(KEY_VALUE_SEPERATOR)
-    
-      def initialize(input=STDIN, output=STDOUT)
-        @input, @output = input, output
-      end
-    
+    class Base < Mandy::Task
       def self.compile(&blk)
         Class.new(Mandy::Mappers::Base) do 
           self.class_eval do
@@ -15,7 +8,7 @@ module Mandy
           end
         end
       end
-    
+      
       def execute
         @input.each_line do |line|
            key, value = line.split(KEY_VALUE_SEPERATOR)
@@ -24,22 +17,12 @@ module Mandy
            mapper(key, value)
         end
       end
-    
-      def emit(key, value=nil)
-        key = 'nil' if key.nil?
-        @output.puts(value.nil? ? key.to_s : "#{serialize(key)}\t#{serialize(value)}")
-      end
-    
+      
       private
     
       def mapper(key,value)
         #nil
       end
-      
-      def serialize(value)
-        value = ArraySerializer.new(value) if value.is_a?(Array)
-        value.to_s
-      end
     end
   end
 end

data/lib/reducers/base_reducer.rb

@@ -1,12 +1,6 @@
 module Mandy
   module Reducers
-    class Base
-      KEY_VALUE_SEPERATOR = "\t" unless defined?(KEY_VALUE_SEPERATOR)
-    
-      def initialize(input=STDIN, output=STDOUT)
-        @input, @output = input, output
-      end
-    
+    class Base < Mandy::Task
       def self.compile(&blk)
         Class.new(Mandy::Reducers::Base) do 
           self.class_eval do
@@ -30,21 +24,11 @@ module Mandy
         reducer(last_key, values)
       end
     
-      def emit(key, value=nil)
-        key = 'nil' if key.nil?
-        @output.puts(value.nil? ? key.to_s : "#{serialize(key)}\t#{serialize(value)}")
-      end
-    
       private
     
       def reducer(key,values)
         #nil
       end
-      
-      def serialize(value)
-        value = ArraySerializer.new(value) if value.is_a?(Array)
-        value.to_s
-      end
     end
   end
 end

data/lib/ruby-hbase/hbase_table.rb

@@ -0,0 +1,166 @@
+module HBase
+  class RowNotFound < Exception
+    def initialize(msg=nil)
+      super
+    end
+  end
+  
+  class HTable    
+    include XmlDecoder
+    
+    def initialize(table_uri)
+      @table_uri = table_uri
+
+      @uri = URI.parse(table_uri)
+      
+      @host, @table_name = @uri.host, @uri.path.split("/").last
+    end
+
+    def name
+      @table_name
+    end
+
+    ######################
+    # Meta-type requests
+
+    def start_keys
+      raise NotImplementedError
+    end
+
+
+    def column_descriptors
+      column_families = []
+      
+      # get the xml for the column descriptors
+      response = Net::HTTP.get_response(@uri.host, "/api/#{@table_name}", @uri.port)
+      body = response.body
+      
+      # parse the xml into a document
+      doc = XML::Parser.string(body).parse
+      
+      doc.find("/table/columnfamilies/columnfamily").each do |node|
+        colfam = {}
+        colfam[:name] = node.find_first("name").content.strip.chop
+        column_families << colfam
+      end
+      column_families
+    end    
+
+    
+    #####################
+    # Standard CRUD ops
+    
+    DEFAULT_GET_OPTIONS = {:timestamp => nil, :columns => nil}
+    
+    def get(key, options = {})
+      opts = DEFAULT_GET_OPTIONS.merge(options)
+      
+      columns = Array(opts.delete(:columns)).compact
+      timestamp = opts.delete(:timestamp)
+      timestamp = (timestamp.to_f * 1000).to_i.to_s if timestamp
+      
+      Net::HTTP.start(@uri.host, @uri.port) do |session|
+        columns_query = columns.map{ |name| "column=#{name}" }.join("&")
+
+        ts_section = timestamp ? "/#{timestamp}" : ""
+
+        query_string = "?" + columns_query
+                      
+        query = "/api/#{@table_name}/row/#{url_encode(key)}#{ts_section}#{query_string}"
+        response = session.get(query, {"Accept" => "*/*"})
+
+        case response.code.to_i
+          when 200 #success!
+            body = response.body
+            parse_row_result(body).last
+          when 204 #no data - probably an incorrect colname
+            raise "Didn't get any data back - check your column names!"
+          when 404
+            raise RowNotFound, "Could not find row '#{key}'"
+          else
+            nil
+        end
+      end
+    end
+    
+    def put(key, keys_and_values, timestamp = nil)
+      Net::HTTP.start(@uri.host, @uri.port) do |session|
+        xml = "<columns>"
+        
+        ts_section = timestamp ? "/#{(timestamp.to_f * 1000).to_i}" : ""
+        
+        keys_and_values.each do |name, value|
+          xml << "<column><name>#{name}</name><value>#{[value.to_s].pack("m")}</value></column>"          
+        end
+        
+        xml << "</columns>"
+        
+        query = "/api/#{@table_name}/row/#{url_encode(key)}#{ts_section}"
+        response = session.post(query, xml, {"Content-type" => "text/xml"})
+        
+        case response.code.to_i
+          when 200
+            true
+          else
+            unexpected_response(response)
+        end
+      end
+    end
+        
+    def delete(row, columns = nil, timestamp = nil)
+      Net::HTTP.start(@uri.host, @uri.port) do |session|
+        columns_query = Array(columns).compact.map{ |name| "column=#{name}" }.join("&")
+        
+        response = session.delete("/api/#{@table_name}/row/#{row}?#{columns_query}")  
+        case response.code.to_i
+          when 202
+            return true
+          else
+            unexpected_response(response)
+        end
+
+      end
+    end
+    
+    #######################
+    # Scanning interface
+    
+    def get_scanner(start_row, end_row, timestamp = nil, columns = nil)
+      start_row_query = start_row ? "start_row=#{start_row}" : nil
+      end_row_query = end_row ? "end_row=#{end_row}" : nil      
+      timestamp_section = timestamp ? "/#{(timestamp.to_f * 1000).to_i}" : nil
+      columns_section = columns ? columns.map{ |col| "column=#{col}" }.join("&") : nil
+
+      query_string = [start_row_query, end_row_query, 
+                      timestamp_section, columns_section].compact.join("&")
+
+      path = ""
+
+      # open the scanner
+      Net::HTTP.start(@uri.host, @uri.port) do |session|
+        response = session.post("/api/#{@table_name}/scanner?#{query_string}", 
+          "", {"Accept" => "text/xml"}
+        )
+        
+        case response.code.to_i
+          when 201
+            # redirect - grab the path and send
+            Scanner.new(self, "http://#{@uri.host}:#{@uri.port}" + response["Location"])
+          else
+            unexpected_response(response)
+        end
+      end      
+    end
+    
+    
+    private
+      
+    def url_encode(str)
+      ERB::Util.url_encode(str)
+    end
+    
+    def unexpected_response(response)
+      raise "Unexpected response code #{response.code.to_i}:\n#{response.body}"
+    end
+  end
+end

data/lib/ruby-hbase/scanner.rb

@@ -0,0 +1,55 @@
+module HBase
+  class Scanner
+    include XmlDecoder
+    
+    def initialize(table, scanner_uri)
+      @table, @scanner_uri = table, scanner_uri
+    end
+    
+    def close
+      
+    end
+    
+    def next
+      
+    end
+    
+    def each
+      parsed_uri = URI.parse(@scanner_uri)
+      Net::HTTP.start(parsed_uri.host, parsed_uri.port) do |session|
+        while true
+          response = session.post(@scanner_uri, "")
+
+          case response.code.to_i
+            when 404
+              # over
+              break
+            when 200
+              # item
+              yield *parse_row_result(response.body)
+            else
+              # error
+              raise "Unexpected response code #{response.code}, body:\n#{response.body}"
+          end
+        end
+      end
+    end
+
+    private
+    
+    # def parse_row(xml)
+    #   doc = REXML::Document.new(xml)
+    # 
+    #   result = {}
+    # 
+    #   doc.root.each_element("/row/column") do |column|
+    #     name = column.get_elements("name")[0].text.strip
+    #     value = column.get_elements("value")[0].text.strip.unpack("m").first
+    #     result[name] = value
+    #   end
+    #   
+    #   [doc.root.get_elements("name")[0].text.strip, result]
+    # end
+    
+  end
+end

data/lib/ruby-hbase/version.rb

@@ -0,0 +1,9 @@
+module RubyHbase #:nodoc:
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 0
+    TINY  = 4
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

data/lib/ruby-hbase/xml_decoder.rb

@@ -0,0 +1,18 @@
+module HBase
+  module XmlDecoder
+    def parse_row_result(xml)
+      doc = XML::Parser.string(xml).parse
+
+      name_node = doc.root.find_first("/row/name")
+      name = name_node ? name_node.content.strip : nil
+      
+      values = {}
+      
+      doc.find("/row/columns/column").each do |node|
+        values[node.find_first("name").content.strip.unpack('m').first] = node.find_first("value").content.strip.unpack("m").first
+      end
+      
+      [name, values]
+    end
+  end
+end

data/lib/ruby-hbase.rb

@@ -0,0 +1,10 @@
+$:.unshift File.dirname(__FILE__)
+
+require "rubygems"
+require "net/http"
+require "erb"
+require "xml/libxml"
+
+require "ruby-hbase/xml_decoder"
+require "ruby-hbase/hbase_table"
+require "ruby-hbase/scanner"

data/lib/stores/hbase.rb

@@ -0,0 +1,24 @@
+module Mandy
+  module Stores
+    class HBase
+      attr_reader :options
+      
+      def initialize(options)
+        @options = options
+        @table = ::HBase::HTable.new(options[:url])
+      end
+      
+      def get(key)
+        @table.get(key)
+      end
+      
+      def put(key, values)
+        @table.put(key, values)
+      end
+      
+      def ==(other)
+        self.class == other.class && self.options == other.options
+      end
+    end
+  end
+end

data/lib/stores/in_memory.rb

@@ -0,0 +1,24 @@
+module Mandy
+  module Stores
+    class InMemory
+      attr_reader :options
+      
+      def initialize(options={})
+        @options = options
+        @table = {}
+      end
+      
+      def get(key)
+        @table[key.to_s]
+      end
+      
+      def put(key, values)
+        @table[key.to_s] = values
+      end
+      
+      def ==(other)
+        self.class == other.class && self.options == other.options
+      end
+    end
+  end
+end

data/lib/task.rb

@@ -0,0 +1,30 @@
+module Mandy
+  class Task
+    KEY_VALUE_SEPERATOR = "\t" unless defined?(KEY_VALUE_SEPERATOR)
+  
+    def initialize(input=STDIN, output=STDOUT)
+      @input, @output = input, output
+    end
+  
+    def emit(key, value=nil)
+      key = 'nil' if key.nil?
+      @output.puts(value.nil? ? key.to_s : "#{serialize(key)}\t#{serialize(value)}")
+    end
+    
+    def get(store, key)
+      Mandy.stores[store].get(key)
+    end
+    
+    def put(store, key, values)
+      Mandy.stores[store].put(key, values)
+    end
+  
+    private
+
+    
+    def serialize(value)
+      value = ArraySerializer.new(value) if value.is_a?(Array)
+      value.to_s
+    end
+  end
+end

data/lib/test_runner.rb

@@ -8,7 +8,7 @@ module Mandy
     
     def map(input_stream, output_stream=StringIO.new(''), &blk)
       input_stream = input_from_array(input_stream) if input_stream.is_a?(Array)
-      input_stream = StringIO.new(input_stream.to_s) unless input_stream.is_a?(StringIO)
+      input_stream = StringIO.new(input_stream) if input_stream.is_a?(String)
       @job.run_map(input_stream, output_stream, &blk)
       output_stream.rewind
       output_stream
@@ -16,7 +16,7 @@ module Mandy
     
     def reduce(input_stream, output_stream=StringIO.new(''), &blk)
       input_stream = input_from_hash(input_stream) if input_stream.is_a?(Hash)
-      input_stream = StringIO.new(input_stream.to_s) unless input_stream.is_a?(StringIO)
+      input_stream = StringIO.new(input_stream) if input_stream.is_a?(String)
       @job.run_reduce(input_stream, output_stream, &blk)
       output_stream.rewind
       output_stream

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: trafficbroker-mandy
 version: !ruby/object:Gem::Version 
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors: 
 - Andy Kent
@@ -39,6 +39,7 @@ files:
 - lib/mandy.rb
 - lib/support/tuple.rb
 - lib/support/array_serializer.rb
+- lib/task.rb
 - lib/dsl.rb
 - lib/job.rb
 - lib/mappers/base_mapper.rb
@@ -48,6 +49,13 @@ files:
 - lib/reducers/sum_reducer.rb
 - lib/reducers/max_reducer.rb
 - lib/reducers/min_reducer.rb
+- lib/stores/hbase.rb
+- lib/stores/in_memory.rb
+- lib/ruby-hbase.rb
+- lib/ruby-hbase/hbase_table.rb
+- lib/ruby-hbase/scanner.rb
+- lib/ruby-hbase/version.rb
+- lib/ruby-hbase/xml_decoder.rb
 - lib/test_runner.rb
 has_rdoc: false
 homepage: