massive_record 0.1.1 → 0.2.0.beta

data/TODO.md

@@ -1,8 +0,0 @@
-# TODO
-
-  * code documentation
-  * Rails ORM (ActiveModel etc.)
-  * write tests
-  * add generators for hbase.yml etc.
-  * add migration tools
-  * ...

data/lib/massive_record/exceptions.rb

@@ -1,11 +0,0 @@
-module MassiveRecord
-  
-  class ConnectionException < Exception
-    
-    def initialize
-      
-    end
-    
-  end
-  
-end

data/lib/massive_record/wrapper/column_family.rb

@@ -1,22 +0,0 @@
-module MassiveRecord
-  module Wrapper
-    class ColumnFamily
-    
-      attr_accessor :name, :max_versions, :columns
-    
-      def initialize(column_name, opts = {})
-        @name = column_name
-        @max_versions = opts[:max_versions] || 10
-        @columns = opts[:columns] || []
-      end
-    
-      def descriptor
-        Apache::Hadoop::Hbase::Thrift::ColumnDescriptor.new do |col|
-          col.name = "#{name}:"
-          col.maxVersions = max_versions
-        end
-      end
-    
-    end
-  end
-end

data/lib/massive_record/wrapper/connection.rb

@@ -1,71 +0,0 @@
-module MassiveRecord
-  module Wrapper
-    class Connection
-  
-      attr_accessor :host, :port, :timeout
-    
-      def initialize(opts = {})
-        @timeout = 4000
-        @host    = opts[:host]
-        @port    = opts[:port] || 9090
-      end
-      
-      def transport
-        @transport ||= Thrift::BufferedTransport.new(Thrift::Socket.new(@host, @port, @timeout))
-      end
-      
-      def open
-        protocol = Thrift::BinaryProtocol.new(transport)
-        @client = Apache::Hadoop::Hbase::Thrift::Hbase::Client.new(protocol)
-        
-        begin
-          transport.open()
-          true
-        rescue
-          raise MassiveRecord::ConnectionException.new, "Unable to connect to HBase on #{@host}, port #{@port}"
-        end
-      end
-      
-      def close
-        @transport.close.nil?
-      end
-          
-      def client
-        @client
-      end
-      
-      def active?
-        @transport.open?
-      end
-      
-      def tables
-        collection = TablesCollection.new
-        collection.connection = self
-        getTableNames().each{|table_name| collection.push(table_name)}
-        collection
-      end
-    
-      def load_table(table_name)
-        MassiveRecord::Wrapper::Table.new(self, table_name)
-      end
-    
-      # Wrapp HBase API to be able to catch errors and try reconnect
-      def method_missing(method, *args)
-        begin
-          open if not @client
-          client.send(method, *args) if @client
-        rescue IOError
-          @client = nil
-          open
-          client.send(method, *args) if @client
-        rescue Thrift::TransportException
-          @transport = nil
-          @client = nil
-          open
-          client.send(method, *args) if @client
-        end
-      end
-    
-    end
-  end
-end

data/lib/massive_record/wrapper/row.rb

@@ -1,173 +0,0 @@
-require 'json'
-
-module MassiveRecord
-  module Wrapper  
-    class Row
-    
-      attr_accessor :id, :column_families, :columns, :new_record, :table
-    
-      def initialize(opts = {})
-        @id              = opts[:id]
-        self.values      = opts[:values] || {}
-        @table           = opts[:table]
-        @column_families = opts[:column_families] || []
-        @columns         = opts[:columns] || {}
-        @new_record      = true
-      end
-      
-      def column_names
-        columns.keys
-      end
-    
-      def fetch_all_column_families
-        @table.fetch_column_family
-        fetch_column_families(@table.column_family_names)
-      end
-    
-      def fetch_column_families(list)
-        @column_families = table.column_families.collect do |column_name, description|
-          ColumnFamily.new(column_name.split(":").first, {
-            :row          => self,
-            :name         => description.name,
-            :max_versions => description.maxVersions,
-            :compression  => description.compression,
-            :in_memory    => description.inMemory
-            # bloomFilterType, bloomFilterVectorSize, bloomFilterNbHashes, blockCacheEnabled, timeToLive
-          })
-        end
-      end
-    
-      # = Parse columns / cells and create a Hash from them
-      def values
-        @columns.inject({"id" => id}) {|h, (column_name, cell)| h[column_name] = cell.deserialize_value; h}
-      end
-
-      def values=(data)
-        @values = {}
-        update_columns(data)
-      end
-    
-      def update_columns(data = {})
-        data.each do |column_family_name, columns|
-          columns.each do |column_name, values|
-            update_column(column_family_name, column_name, values)
-          end
-        end
-      end
-    
-      def update_column(column_family_name, column_name, value)
-        column = "#{column_family_name}:#{column_name}"
-      
-        if @columns[column].nil?
-          @columns[column] = Cell.new({ :value => Cell.serialize_value(value), :created_at => Time.now })
-        else
-          @columns[column].serialize_value(value)
-        end
-      end
-    
-      # = Merge column values with new data : it implies that column values is a JSON encoded string
-      def merge_columns(data)
-        data.each do |column_family_name, columns|
-          columns.each do |column_name, values|
-            if values.is_a?(Hash)
-              unless @columns["#{column_family_name}:#{column_name}"].nil?
-                column_value = @columns["#{column_family_name}:#{column_name}"].deserialize_value.merge(values)
-              else
-                column_value = values
-              end            
-            elsif values.is_a?(Array)
-              unless @columns["#{column_family_name}:#{column_name}"].nil?
-                column_value = @columns["#{column_family_name}:#{column_name}"].deserialize_value | values
-              else
-                column_value = values
-              end            
-            else
-              column_value = values
-            end
-            update_column(column_family_name, column_name, column_value)
-          end
-        end
-      end
-    
-      # = Parse columns cells and save them
-      def save
-        mutations = []
-      
-        @columns.each do |column_name, cell|
-          m        = Apache::Hadoop::Hbase::Thrift::Mutation.new
-          m.column = column_name
-          m.value  = cell.serialized_value
-        
-          mutations.push(m)
-        end
-
-        @table.client.mutateRow(@table.name, id.to_s, mutations).nil?
-      end
-
-
-      #
-      # FIXME
-      # 
-      # The thrift wrapper is only working with strings as far as I can see,
-      # and the atomicIncrement call on strings kinda doesn't make sense on strings
-      #
-      # For now I'll implement this without atomicIncrement, to get the behaviour we want.
-      # Guess this in time will either be fixed or raised an not-supported-error. If the
-      # latter is the case I guess we'll need to shift over to a jruby adapter and use the
-      # java api instead of thrift.
-      #
-      def atomic_increment(column_name, by = 1)
-        # @table.client.atomicIncrement(@table.name, id.to_s, column_name, by) 
-        value_to_increment = @columns[column_name.to_s].value
-
-        raise "Value to increment (#{value_to_increment}) doesnt seem to be a number!" unless value_to_increment =~ /^\d+$/
-        raise "Argument by must be an integer" unless by.is_a? Fixnum
-
-        value_to_increment = value_to_increment.to_i
-        value_to_increment += by
-        value_to_increment = value_to_increment.to_s
-
-        mutation = Apache::Hadoop::Hbase::Thrift::Mutation.new
-        mutation.column = column_name
-        mutation.value = value_to_increment
-
-        if @table.client.mutateRow(@table.name, id.to_s, [mutation]).nil?
-          value_to_increment
-        end
-      end
-    
-      def self.populate_from_trow_result(result, connection, table_name, column_families = [])
-        row                 = self.new
-        row.id              = result.row
-        row.new_record      = false
-        row.table           = Table.new(connection, table_name)
-        row.column_families = column_families
-
-        result.columns.each do |name, value|
-          row.columns[name] = Cell.new({
-            :value      => value.value,
-            :created_at => Time.at(value.timestamp / 1000, (value.timestamp % 1000) * 1000)
-          })
-        end
-      
-        row
-      end
-    
-      def destroy
-        @table.client.deleteAllRow(@table.name, @id).nil?
-      end
-    
-      def new_record?
-        @new_record
-      end
-    
-      def prev
-        self
-      end
-    
-      def updated_at
-        columns.values.collect(&:created_at).max
-      end
-    end  
-  end
-end

data/lib/massive_record/wrapper/scanner.rb

@@ -1,61 +0,0 @@
-module MassiveRecord
-  module Wrapper
-    class Scanner
-      
-      attr_accessor :connection, :table_name, :column_family_names, :opened_scanner
-      attr_accessor :start_key, :offset_key, :created_at, :limit
-      attr_accessor :formatted_column_family_names, :column_family_names
-      
-      def initialize(connection, table_name, column_family_names, opts = {})
-        @connection = connection
-        @table_name = table_name
-        @column_family_names = column_family_names.collect{|n| n.split(":").first}
-        @column_family_names = opts[:columns] unless opts[:columns].nil?
-        @formatted_column_family_names = column_family_names.collect{|n| "#{n.split(":").first}:"}
-        @start_key = opts[:start_key].to_s
-        @offset_key = opts[:offset_key].to_s
-        @created_at = opts[:created_at].to_s
-        @limit = opts[:limit] || 10
-      end
-      
-      def key
-        start_key.empty? ? offset_key : start_key
-      end
-      
-      def open
-        if created_at.empty?
-          self.opened_scanner = connection.scannerOpen(table_name, key, formatted_column_family_names)
-        else
-          self.opened_scanner = connection.scannerOpenTs(table_name, key, formatted_column_family_names, created_at)
-        end
-      end
-      
-      def close
-        connection.scannerClose(opened_scanner)
-      end
-      
-      def fetch_trows(opts = {})
-        connection.scannerGetList(opened_scanner, limit)
-      end
-      
-      def fetch_rows(opts = {})
-        populate_rows(fetch_trows(opts))
-      end
-      
-      def populate_rows(results)
-        results.collect do |result|
-          if offset_key.empty?
-            populate_row(result) unless result.row.match(/^#{start_key}/).nil?
-          else
-            populate_row(result)            
-          end
-        end.select{|r| !r.nil?}
-      end
-      
-      def populate_row(result)
-        Row.populate_from_trow_result(result, connection, table_name, column_family_names)
-      end
-      
-    end
-  end
-end

data/lib/massive_record/wrapper/table.rb

@@ -1,149 +0,0 @@
-module MassiveRecord
-  module Wrapper
-    class Table
-        
-      attr_accessor :connection, :name, :column_families
-    
-      def initialize(connection, table_name)
-        @connection = connection
-        @name = table_name.to_s
-        init_column_families
-      end
-    
-      def init_column_families      
-        @column_families = ColumnFamiliesCollection.new
-        @column_families.table = self
-      end
-    
-      def self.create(connection, table_name, column_families = [])
-        table = self.new(connection, table_name)
-        table.column_families = column_families
-        table.save
-      end
-    
-      def save
-        begin
-          client.createTable(name, @column_families.collect{|cf| cf.descriptor}).nil?
-        rescue Apache::Hadoop::Hbase::Thrift::AlreadyExists => ex
-          "The table already exists."
-        rescue => ex
-          raise ex
-        end
-      end
-    
-      def client
-        connection
-      end    
-    
-      def disable
-        client.disableTable(name).nil?
-      end
-    
-      def destroy
-        disable
-        client.deleteTable(name).nil?
-      end
-    
-      def create_column_families(column_family_names)
-        column_family_names.each{|name| @column_families.push(ColumnFamily.new(name))}
-      end
-    
-      def fetch_column_families
-        @column_families.clear
-        client.getColumnDescriptors(name).each do |column_name, description| 
-          @column_families.push(ColumnFamily.new(column_name.split(":").first))
-        end
-        @column_families
-      end
-    
-      def column_family_names
-        @column_families.collect{|column_family| column_family.name.to_s}
-      end
-    
-      def fetch_column_family_names
-        fetch_column_families
-        column_family_names
-      end
-      
-      def column_names
-        first.column_names
-      end
-    
-      def scanner(opts = {})
-        scanner = Scanner.new(connection, name, column_family_names, format_options_for_scanner(opts))
-        
-        if block_given?
-          begin
-            scanner.open
-            yield scanner
-          ensure
-            scanner.close
-          end
-        else
-          scanner
-        end
-      end
-      
-      def format_options_for_scanner(opts = {})
-        {
-          :start_key  => opts[:start],
-          :offset_key => opts[:offset],
-          :created_at => opts[:created_at],
-          :columns    => opts[:select], # list of column families to fetch from hbase
-          :limit      => opts[:limit] || opts[:batch_size]
-        }
-      end
-      
-      def all(opts = {})
-        scanner(opts) do |s|
-          s.fetch_rows(opts)
-        end
-      end
-      
-      def first(opts = {})
-        all(opts.merge(:limit => 1)).first
-      end
-      
-      def find(*args)
-        arg  = args[0]
-        opts = args[1] || {}
-        arg.is_a?(Array) ? arg.collect{|id| first(opts.merge(:start => id))} : first(opts.merge(:start => arg))
-      end
-    
-      def find_in_batches(opts = {})        
-        results_limit = opts.delete(:limit)
-        results_found = 0
-        
-        scanner(opts) do |s|
-          while (true) do
-            s.limit = results_limit - results_found if !results_limit.nil? && results_limit <= results_found + s.limit
-            rows = s.fetch_rows
-            if rows.empty?
-              break
-            else
-              results_found += rows.size
-              yield rows
-            end
-          end
-        end
-      end
-    
-      def exists?
-        connection.tables.include?(name)
-      end
-    
-      def regions
-        connection.getTableRegions(name).collect do |r|
-          {
-            :start_key => r.startKey,
-            :end_key => r.endKey,
-            :id => r.id,
-            :name => r.name,
-            :version => r.version
-          }
-        end
-      end
-
-    end
-  end
-end