massive_record 0.1.0

data/lib/massive_record/thrift/hbase_constants.rb

@@ -0,0 +1,14 @@
+#
+# Autogenerated by Thrift
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+module Apache
+  module Hadoop
+    module Hbase
+      module Thrift
+      end
+    end
+  end
+end

data/lib/massive_record/thrift/hbase_types.rb

@@ -0,0 +1,225 @@
+#
+# Autogenerated by Thrift
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+module Apache
+  module Hadoop
+    module Hbase
+      module Thrift
+        # TCell - Used to transport a cell value (byte[]) and the timestamp it was
+        # stored with together as a result for get and getRow methods. This promotes
+        # the timestamp of a cell to a first-class value, making it easy to take
+        # note of temporal data. Cell is used all the way from HStore up to HTable.
+        class TCell
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          VALUE = 1
+          TIMESTAMP = 2
+
+          FIELDS = {
+            VALUE => {:type => ::Thrift::Types::STRING, :name => 'value', :binary => true},
+            TIMESTAMP => {:type => ::Thrift::Types::I64, :name => 'timestamp'}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # An HColumnDescriptor contains information about a column family
+        # such as the number of versions, compression settings, etc. It is
+        # used as input when creating a table or adding a column.
+        class ColumnDescriptor
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          NAME = 1
+          MAXVERSIONS = 2
+          COMPRESSION = 3
+          INMEMORY = 4
+          BLOOMFILTERTYPE = 5
+          BLOOMFILTERVECTORSIZE = 6
+          BLOOMFILTERNBHASHES = 7
+          BLOCKCACHEENABLED = 8
+          TIMETOLIVE = 9
+
+          FIELDS = {
+            NAME => {:type => ::Thrift::Types::STRING, :name => 'name', :binary => true},
+            MAXVERSIONS => {:type => ::Thrift::Types::I32, :name => 'maxVersions', :default => 3},
+            COMPRESSION => {:type => ::Thrift::Types::STRING, :name => 'compression', :default => %q"NONE"},
+            INMEMORY => {:type => ::Thrift::Types::BOOL, :name => 'inMemory', :default => false},
+            BLOOMFILTERTYPE => {:type => ::Thrift::Types::STRING, :name => 'bloomFilterType', :default => %q"NONE"},
+            BLOOMFILTERVECTORSIZE => {:type => ::Thrift::Types::I32, :name => 'bloomFilterVectorSize', :default => 0},
+            BLOOMFILTERNBHASHES => {:type => ::Thrift::Types::I32, :name => 'bloomFilterNbHashes', :default => 0},
+            BLOCKCACHEENABLED => {:type => ::Thrift::Types::BOOL, :name => 'blockCacheEnabled', :default => false},
+            TIMETOLIVE => {:type => ::Thrift::Types::I32, :name => 'timeToLive', :default => -1}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # A TRegionInfo contains information about an HTable region.
+        class TRegionInfo
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          STARTKEY = 1
+          ENDKEY = 2
+          ID = 3
+          NAME = 4
+          VERSION = 5
+
+          FIELDS = {
+            STARTKEY => {:type => ::Thrift::Types::STRING, :name => 'startKey', :binary => true},
+            ENDKEY => {:type => ::Thrift::Types::STRING, :name => 'endKey', :binary => true},
+            ID => {:type => ::Thrift::Types::I64, :name => 'id'},
+            NAME => {:type => ::Thrift::Types::STRING, :name => 'name', :binary => true},
+            VERSION => {:type => ::Thrift::Types::BYTE, :name => 'version'}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # A Mutation object is used to either update or delete a column-value.
+        class Mutation
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          ISDELETE = 1
+          COLUMN = 2
+          VALUE = 3
+
+          FIELDS = {
+            ISDELETE => {:type => ::Thrift::Types::BOOL, :name => 'isDelete', :default => false},
+            COLUMN => {:type => ::Thrift::Types::STRING, :name => 'column', :binary => true},
+            VALUE => {:type => ::Thrift::Types::STRING, :name => 'value', :binary => true}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # A BatchMutation object is used to apply a number of Mutations to a single row.
+        class BatchMutation
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          ROW = 1
+          MUTATIONS = 2
+
+          FIELDS = {
+            ROW => {:type => ::Thrift::Types::STRING, :name => 'row', :binary => true},
+            MUTATIONS => {:type => ::Thrift::Types::LIST, :name => 'mutations', :element => {:type => ::Thrift::Types::STRUCT, :class => Apache::Hadoop::Hbase::Thrift::Mutation}}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # Holds row name and then a map of columns to cells.
+        class TRowResult
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          ROW = 1
+          COLUMNS = 2
+
+          FIELDS = {
+            ROW => {:type => ::Thrift::Types::STRING, :name => 'row', :binary => true},
+            COLUMNS => {:type => ::Thrift::Types::MAP, :name => 'columns', :key => {:type => ::Thrift::Types::STRING, :binary => true}, :value => {:type => ::Thrift::Types::STRUCT, :class => Apache::Hadoop::Hbase::Thrift::TCell}}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # An IOError exception signals that an error occurred communicating
+        # to the Hbase master or an Hbase region server.  Also used to return
+        # more general Hbase error conditions.
+        class IOError < ::Thrift::Exception
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          def initialize(message=nil)
+            super()
+            self.message = message
+          end
+
+          MESSAGE = 1
+
+          FIELDS = {
+            MESSAGE => {:type => ::Thrift::Types::STRING, :name => 'message'}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # An IllegalArgument exception indicates an illegal or invalid
+        # argument was passed into a procedure.
+        class IllegalArgument < ::Thrift::Exception
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          def initialize(message=nil)
+            super()
+            self.message = message
+          end
+
+          MESSAGE = 1
+
+          FIELDS = {
+            MESSAGE => {:type => ::Thrift::Types::STRING, :name => 'message'}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # An AlreadyExists exceptions signals that a table with the specified
+        # name already exists
+        class AlreadyExists < ::Thrift::Exception
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          def initialize(message=nil)
+            super()
+            self.message = message
+          end
+
+          MESSAGE = 1
+
+          FIELDS = {
+            MESSAGE => {:type => ::Thrift::Types::STRING, :name => 'message'}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+      end
+    end
+  end
+end

data/lib/massive_record/version.rb

@@ -0,0 +1,3 @@
+module MassiveRecord
+  VERSION = "0.1.0"
+end

data/lib/massive_record/wrapper/base.rb

@@ -0,0 +1,28 @@
+require 'yaml'
+require 'massive_record/wrapper/connection'
+require 'massive_record/wrapper/tables_collection'
+require 'massive_record/wrapper/table'
+require 'massive_record/wrapper/row'
+require 'massive_record/wrapper/column_families_collection'
+require 'massive_record/wrapper/column_family'
+require 'massive_record/wrapper/cell'
+require 'massive_record/wrapper/scanner'
+
+module MassiveRecord
+  module Wrapper
+    class Base
+      
+      def self.config
+        config = YAML.load_file(Rails.root.join('config', 'hbase.yml'))[Rails.env]
+        { :host => config['host'], :port => config['port'] }        
+      end
+
+      def self.connection(opts = {})
+        conn = Connection.new(opts.empty? ? config : opts)
+        conn.open
+        conn
+      end
+      
+    end  
+  end
+end

data/lib/massive_record/wrapper/cell.rb

@@ -0,0 +1,45 @@
+module MassiveRecord
+  module Wrapper
+    class Cell
+      attr_writer :value
+      attr_accessor :created_at
+
+      class << self
+        def serialize_value(v)
+          serialize?(v) ? v.to_yaml : v.to_s.force_encoding(Encoding::BINARY)
+        end
+
+        private
+
+        def serialize?(v)
+          [Hash, Array, NilClass].include?(v.class)
+        end
+      end
+
+      def initialize(opts = {})
+        @value = opts[:value]
+        @created_at = opts[:created_at]
+      end
+    
+      def value
+        @value.is_a?(String) ? @value.to_s.force_encoding(Encoding::UTF_8) : @value
+      end
+    
+      def deserialize_value
+        is_yaml? ? YAML.load(@value) : value
+      end
+    
+      def serialize_value(v)
+        @value = self.class.serialize_value(v)
+      end
+    
+      def serialized_value
+        self.class.serialize_value(@value)
+      end
+    
+      def is_yaml?
+        @value =~ /^--- \n/ || @value =~ /^--- {}/ || @value =~ /^--- \[\]/
+      end
+    end
+  end
+end

data/lib/massive_record/wrapper/column_families_collection.rb

@@ -0,0 +1,19 @@
+module MassiveRecord
+  module Wrapper
+    class ColumnFamiliesCollection < Array
+  
+      attr_accessor :table
+  
+      def create(column_family, opts = {})
+        if column_family.is_a?(MassiveRecord::Wrapper::ColumnFamily)
+          self.push(column_family)
+        else
+          self.push(MassiveRecord::Wrapper::ColumnFamily.new(column_family, opts))
+        end
+    
+        true
+      end
+  
+    end
+  end
+end

data/lib/massive_record/wrapper/column_family.rb

@@ -0,0 +1,22 @@
+module MassiveRecord
+  module Wrapper
+    class ColumnFamily
+    
+      attr_accessor :name, :max_versions, :columns
+    
+      def initialize(column_name, opts = {})
+        @name = column_name
+        @max_versions = opts[:max_versions] || 10
+        @columns = opts[:columns] || []
+      end
+    
+      def descriptor
+        Apache::Hadoop::Hbase::Thrift::ColumnDescriptor.new do |col|
+          col.name = "#{name}:"
+          col.maxVersions = max_versions
+        end
+      end
+    
+    end
+  end
+end

data/lib/massive_record/wrapper/connection.rb

@@ -0,0 +1,71 @@
+module MassiveRecord
+  module Wrapper
+    class Connection
+  
+      attr_accessor :host, :port, :timeout
+    
+      def initialize(opts = {})
+        @timeout = 4000
+        @host    = opts[:host]
+        @port    = opts[:port] || 9090
+      end
+      
+      def transport
+        @transport ||= Thrift::BufferedTransport.new(Thrift::Socket.new(@host, @port, @timeout))
+      end
+      
+      def open
+        protocol = Thrift::BinaryProtocol.new(transport)
+        @client = Apache::Hadoop::Hbase::Thrift::Hbase::Client.new(protocol)
+        
+        begin
+          transport.open()
+          true
+        rescue
+          raise MassiveRecord::ConnectionException.new, "Unable to connect to HBase on #{@host}, port #{@port}"
+        end
+      end
+      
+      def close
+        @transport.close.nil?
+      end
+          
+      def client
+        @client
+      end
+      
+      def active?
+        @transport.open?
+      end
+      
+      def tables
+        collection = TablesCollection.new
+        collection.connection = self
+        getTableNames().each{|table_name| collection.push(table_name)}
+        collection
+      end
+    
+      def load_table(table_name)
+        MassiveRecord::Wrapper::Table.new(self, table_name)
+      end
+    
+      # Wrapp HBase API to be able to catch errors and try reconnect
+      def method_missing(method, *args)
+        begin
+          open if not @client
+          client.send(method, *args) if @client
+        rescue IOError
+          @client = nil
+          open
+          client.send(method, *args) if @client
+        rescue Thrift::TransportException
+          @transport = nil
+          @client = nil
+          open
+          client.send(method, *args) if @client
+        end
+      end
+    
+    end
+  end
+end

data/lib/massive_record/wrapper/row.rb

@@ -0,0 +1,170 @@
+require 'json'
+
+module MassiveRecord
+  module Wrapper  
+    class Row
+    
+      attr_accessor :id, :column_families, :columns, :new_record, :table
+    
+      def initialize(opts = {})
+        @id              = opts[:id]
+        self.values      = opts[:values] || {}
+        @table           = opts[:table]
+        @column_families = opts[:column_families] || []
+        @columns         = opts[:columns] || {}
+        @new_record      = true
+      end
+      
+      def column_names
+        columns.keys
+      end
+    
+      def fetch_all_column_families
+        @table.fetch_column_family
+        fetch_column_families(@table.column_family_names)
+      end
+    
+      def fetch_column_families(list)
+        @column_families = table.column_families.collect do |column_name, description|
+          ColumnFamily.new(column_name.split(":").first, {
+            :row          => self,
+            :name         => description.name,
+            :max_versions => description.maxVersions,
+            :compression  => description.compression,
+            :in_memory    => description.inMemory
+            # bloomFilterType, bloomFilterVectorSize, bloomFilterNbHashes, blockCacheEnabled, timeToLive
+          })
+        end
+      end
+    
+      # = Parse columns / cells and create a Hash from them
+      def values
+        @columns.inject({"id" => id}) {|h, (column_name, cell)| h[column_name] = cell.deserialize_value; h}
+      end
+    
+      def values=(data)
+        @values = {}
+        update_columns(data)
+      end
+    
+      def update_columns(data = {})
+        data.each do |column_family_name, columns|
+          columns.each do |column_name, values|
+            update_column(column_family_name, column_name, values)
+          end
+        end
+      end
+    
+      def update_column(column_family_name, column_name, value)
+        column = "#{column_family_name}:#{column_name}"
+      
+        if @columns[column].nil?
+          @columns[column] = Cell.new({ :value => Cell.serialize_value(value), :created_at => Time.now })
+        else
+          @columns[column].serialize_value(value)
+        end
+      end
+    
+      # = Merge column values with new data : it implies that column values is a JSON encoded string
+      def merge_columns(data)
+        data.each do |column_family_name, columns|
+          columns.each do |column_name, values|
+            if values.is_a?(Hash)
+              unless @columns["#{column_family_name}:#{column_name}"].nil?
+                column_value = @columns["#{column_family_name}:#{column_name}"].deserialize_value.merge(values)
+              else
+                column_value = values
+              end            
+            elsif values.is_a?(Array)
+              unless @columns["#{column_family_name}:#{column_name}"].nil?
+                column_value = @columns["#{column_family_name}:#{column_name}"].deserialize_value | values
+              else
+                column_value = values
+              end            
+            else
+              column_value = values
+            end
+            update_column(column_family_name, column_name, column_value)
+          end
+        end
+      end
+    
+      # = Parse columns cells and save them
+      def save
+        mutations = []
+      
+        @columns.each do |column_name, cell|
+          m        = Apache::Hadoop::Hbase::Thrift::Mutation.new
+          m.column = column_name
+          m.value  = cell.serialized_value
+        
+          mutations.push(m)
+        end
+
+        @table.client.mutateRow(@table.name, id.to_s, mutations).nil?
+      end
+
+
+      #
+      # FIXME
+      # 
+      # The thrift wrapper is only working with strings as far as I can see,
+      # and the atomicIncrement call on strings kinda doesn't make sense on strings
+      #
+      # For now I'll implement this without atomicIncrement, to get the behaviour we want.
+      # Guess this in time will either be fixed or raised an not-supported-error. If the
+      # latter is the case I guess we'll need to shift over to a jruby adapter and use the
+      # java api instead of thrift.
+      #
+      def atomic_increment(column_name, by = 1)
+        # @table.client.atomicIncrement(@table.name, id.to_s, column_name, by) 
+        value_to_increment = @columns[column_name.to_s].value
+
+        raise "Value to increment (#{value_to_increment}) doesnt seem to be a number!" unless value_to_increment =~ /^\d+$/
+        raise "Argument by must be an integer" unless by.is_a? Fixnum
+
+        value_to_increment = value_to_increment.to_i
+        value_to_increment += by
+        value_to_increment = value_to_increment.to_s
+
+        mutation = Apache::Hadoop::Hbase::Thrift::Mutation.new
+        mutation.column = column_name
+        mutation.value = value_to_increment
+
+        if @table.client.mutateRow(@table.name, id.to_s, [mutation]).nil?
+          value_to_increment
+        end
+      end
+    
+      def self.populate_from_trow_result(result, connection, table_name, column_families = [])
+        row                 = self.new
+        row.id              = result.row
+        row.new_record      = false
+        row.table           = Table.new(connection, table_name)
+        row.column_families = column_families
+
+        result.columns.each do |name, value|
+          row.columns[name] = Cell.new({
+            :value      => value.value,
+            :created_at => Time.at(value.timestamp / 1000, (value.timestamp % 1000) * 1000)
+          })
+        end
+      
+        row
+      end
+    
+      def destroy
+        @table.client.deleteAllRow(@table.name, @id).nil?
+      end
+    
+      def new_record?
+        @new_record
+      end
+    
+      def prev
+        self
+      end
+    
+    end  
+  end
+end