bigrecord 0.0.5

data/lib/big_record/connection_adapters/hbase_adapter.rb

@@ -0,0 +1,432 @@
+require 'rubygems'
+require 'set'
+require 'drb'
+
+unless defined?(BigRecordDriver)
+  begin
+    # Bigrecord's source is included with Bigrecord-Driver, that's we check for this first
+    require File.join(File.dirname(__FILE__), "..", "..", "..", "..", "bigrecord-driver", "lib", "big_record_driver")
+  rescue LoadError
+    begin
+      gem 'bigrecord-driver'
+      require 'bigrecord_driver'
+    rescue Gem::LoadError
+      puts "bigrecord-driver not available. Install it with: sudo gem install bigrecord-driver -s http://gemcutter.org"
+    end
+  end
+end
+
+module BigRecord
+  class Base
+    # Establishes a connection to the database that's used by all Active Record objects.
+    def self.hbase_connection(config) # :nodoc:
+      config = config.symbolize_keys
+
+      zookeeper_host          = config[:zookeeper_host]
+      zookeeper_client_port   = config[:zookeeper_client_port]
+      drb_host                = config[:drb_host]
+      drb_port                = config[:drb_port]
+
+      hbase = BigRecordDriver::Client.new(config)
+
+      ConnectionAdapters::HbaseAdapter.new(hbase, logger, [zookeeper_host, zookeeper_client_port], config)
+    end
+  end
+
+  module ConnectionAdapters
+    class HbaseAdapter < AbstractAdapter
+      @@emulate_booleans = true
+      cattr_accessor :emulate_booleans
+
+      LOST_CONNECTION_ERROR_MESSAGES = [
+        "Server shutdown in progress",
+        "Broken pipe",
+        "Lost connection to HBase server during query",
+        "HBase server has gone away"
+      ]
+
+      # data types
+      TYPE_NULL     = 0x00;
+      TYPE_STRING   = 0x01; # utf-8 strings
+#      TYPE_INTEGER  = 0x02; # delegate to YAML
+#      TYPE_FLOAT    = 0x03; # fixed 1 byte
+      TYPE_BOOLEAN  = 0x04; # delegate to YAML
+#      TYPE_MAP      = 0x05; # delegate to YAML
+#      TYPE_DATETIME = 0x06; # delegate to YAML
+      TYPE_BINARY   = 0x07; # byte[] => no conversion
+
+      # string charset
+      CHARSET = "utf-8"
+
+      # utility constants
+      NULL = "\000"
+#      TRUE = "\001"
+#      FALSE = "\000"
+
+      def initialize(connection, logger, connection_options, config)
+        super(connection, logger)
+        @connection_options, @config = connection_options, config
+
+        connect
+      end
+
+      def configuration
+        @config.clone
+      end
+
+      def adapter_name #:nodoc:
+        'HBase'
+      end
+
+      def supports_migrations? #:nodoc:
+        true
+      end
+
+      # CONNECTION MANAGEMENT ====================================
+
+      def active?
+        @connection.ping
+      rescue BigRecordError
+        false
+      end
+
+      def reconnect!
+        disconnect!
+        connect
+      end
+
+      def disconnect!
+        @connection.close rescue nil
+      end
+
+
+      # DATABASE STATEMENTS ======================================
+
+      def update_raw(table_name, row, values, timestamp)
+        result = nil
+        log "UPDATE #{table_name} SET #{values.inspect if values} WHERE ROW=#{row};" do
+          result = @connection.update(table_name, row, values, timestamp)
+        end
+        result
+      end
+
+      def update(table_name, row, values, timestamp)
+        serialized_collection = {}
+        values.each do |column, value|
+          serialized_collection[column] = serialize(value)
+        end
+        update_raw(table_name, row, serialized_collection, timestamp)
+      end
+
+      def get_raw(table_name, row, column, options={})
+        result = nil
+        log "SELECT (#{column}) FROM #{table_name} WHERE ROW=#{row};" do
+          result = @connection.get(table_name, row, column, options)
+        end
+        result
+      end
+
+      def get(table_name, row, column, options={})
+        serialized_result = get_raw(table_name, row, column, options)
+        result = nil
+        if serialized_result.is_a?(Array)
+          result = serialized_result.collect{|e| deserialize(e)}
+        else
+          result = deserialize(serialized_result)
+        end
+        result
+      end
+
+      def get_columns_raw(table_name, row, columns, options={})
+        result = {}
+        log "SELECT (#{columns.join(", ")}) FROM #{table_name} WHERE ROW=#{row};" do
+          result = @connection.get_columns(table_name, row, columns, options)
+        end
+        result
+      end
+
+      def get_columns(table_name, row, columns, options={})
+        row_cols = get_columns_raw(table_name, row, columns, options)
+        result = {}
+        return nil unless row_cols
+
+        row_cols.each do |key, col|
+          result[key] =
+          if key == 'id'
+            col
+          else
+            deserialize(col)
+          end
+        end
+        result
+      end
+
+      def get_consecutive_rows_raw(table_name, start_row, limit, columns, stop_row = nil)
+        result = nil
+        log "SCAN (#{columns.join(", ")}) FROM #{table_name} WHERE START_ROW=#{start_row} AND STOP_ROW=#{stop_row} LIMIT=#{limit};" do
+          result = @connection.get_consecutive_rows(table_name, start_row, limit, columns, stop_row)
+        end
+        result
+      end
+
+      def get_consecutive_rows(table_name, start_row, limit, columns, stop_row = nil)
+        rows = get_consecutive_rows_raw(table_name, start_row, limit, columns, stop_row)
+        result = rows.collect do |row_cols|
+          cols = {}
+          row_cols.each do |key, col|
+            begin
+              cols[key] =
+              if key == 'id'
+                col
+              else
+                deserialize(col)
+              end
+            rescue Exception => e
+              puts "Could not load column value #{key} for row=#{row_cols['id']}"
+            end
+          end
+          cols
+        end
+        result
+      end
+
+      def delete(table_name, row, timestamp = nil)
+        timestamp ||= Time.now.to_bigrecord_timestamp
+        result = nil
+        log "DELETE FROM #{table_name} WHERE ROW=#{row};" do
+          result = @connection.delete(table_name, row, timestamp)
+        end
+        result
+      end
+
+      def truncate_table(table_name)
+        result = nil
+        log "TRUNCATE TABLE #{table_name}" do
+          result = @connection.truncate_table(table_name)
+        end
+        result
+      end
+
+
+      # SCHEMA STATEMENTS ========================================
+
+      def initialize_schema_migrations_table
+        sm_table = BigRecord::Migrator.schema_migrations_table_name
+
+        unless table_exists?(sm_table)
+          create_table(sm_table) do |t|
+            t.family :attribute, :versions => 1
+          end
+        end
+      end
+
+      def get_all_schema_versions
+        sm_table = BigRecord::Migrator.schema_migrations_table_name
+
+        get_consecutive_rows(sm_table, nil, nil, ["attribute:version"]).map{|version| version["attribute:version"]}
+      end
+
+      def table_exists?(table_name)
+        log "TABLE EXISTS? #{table_name};" do
+          @connection.table_exists?(table_name)
+        end
+      end
+
+      def create_table(table_name, options = {})
+        table_definition = TableDefinition.new
+
+        yield table_definition if block_given?
+
+        if options[:force] && table_exists?(table_name)
+          drop_table(table_name)
+        end
+
+        result = nil
+        log "CREATE TABLE #{table_name} (#{table_definition.column_families_list});" do
+          result = @connection.create_table(table_name, table_definition.to_adapter_format)
+        end
+        result
+      end
+
+      def drop_table(table_name)
+        result = nil
+        log "DROP TABLE #{table_name};" do
+          result = @connection.drop_table(table_name)
+        end
+        result
+      end
+
+      def add_column_family(table_name, column_name, options = {})
+        column = BigRecordDriver::ColumnDescriptor.new(column_name.to_s, options)
+
+        result = nil
+        log "ADD COLUMN TABLE #{table_name} COLUMN #{column_name} (#{options.inspect});" do
+          result = @connection.add_column(table_name, column)
+        end
+        result
+      end
+
+      alias :add_family :add_column_family
+
+      def remove_column_family(table_name, column_name)
+        result = nil
+        log "REMOVE COLUMN TABLE #{table_name} COLUMN #{column_name};" do
+          result = @connection.remove_column(table_name, column_name)
+        end
+        result
+      end
+
+      alias :remove_family :remove_column_family
+
+      def modify_column_family(table_name, column_name, options = {})
+        column = BigRecordDriver::ColumnDescriptor.new(column_name.to_s, options)
+
+        result = nil
+        log "MODIFY COLUMN TABLE #{table_name} COLUMN #{column_name} (#{options.inspect});" do
+          result = @connection.modify_column(table_name, column)
+        end
+        result
+      end
+
+      alias :modify_family :modify_column_family
+
+      # Serialize the given value
+      def serialize(value)
+        case value
+        when NilClass then NULL
+        when String then build_serialized_value(TYPE_STRING, value)
+        else value.to_yaml
+        end
+      end
+
+      # Serialize an object in a given type
+      def build_serialized_value(type, value)
+        type.chr + value
+      end
+
+      # Deserialize the given string. This method supports both the pure YAML format and
+      # the type header format.
+      def deserialize(str)
+        return unless str
+
+        #	stay compatible with the old serialization code
+        #	YAML documents start with "--- " so if we find that sequence at the beginning we
+        #	consider it as a serialized YAML value, else it's the new format with the type header
+        if str[0..3] == "--- "
+          YAML::load(str) if str
+        else
+          deserialize_with_header(str)
+        end
+      end
+
+      # Deserialize the given string assumed to be in the type header format.
+      def deserialize_with_header(data)
+        return unless data and data.size >= 2
+
+        # the type of the data is encoded in the first byte
+        type = data[0];
+
+        case type
+        when TYPE_NULL then nil
+        when TYPE_STRING then data[1..-1]
+        when TYPE_BINARY then data[1..-1]
+        else nil
+        end
+      end
+
+      private
+        def connect
+          @connection.configure(@config)
+        rescue DRb::DRbConnError
+          raise BigRecord::ConnectionFailed, "Failed to connect to the DRb server (jruby) " +
+                                                "at #{@config[:drb_host]}:#{@config[:drb_port]}."
+        end
+
+      protected
+        def log(str, name = nil)
+          if block_given?
+            if @logger and @logger.level <= Logger::INFO
+              result = nil
+              seconds = Benchmark.realtime { result = yield }
+              @runtime += seconds
+              log_info(str, name, seconds)
+              result
+            else
+              yield
+            end
+          else
+            log_info(str, name, 0)
+            nil
+          end
+        rescue Exception => e
+          # Log message and raise exception.
+          # Set last_verfication to 0, so that connection gets verified
+          # upon reentering the request loop
+          @last_verification = 0
+          message = "#{e.class.name}: #{e.message}: #{str}"
+          log_info(message, name, 0)
+          raise e
+        end
+
+        def log_info(str, name, runtime)
+          return unless @logger
+
+          @logger.debug(
+            format_log_entry(
+              "#{name.nil? ? "HBASE" : name} (#{sprintf("%f", runtime)})",
+              str.gsub(/ +/, " ")
+            )
+          )
+        end
+
+        def format_log_entry(message, dump = nil)
+          if BigRecord::Base.colorize_logging
+            if @@row_even
+              @@row_even = false
+              message_color, dump_color = "4;36;1", "0;1"
+            else
+              @@row_even = true
+              message_color, dump_color = "4;35;1", "0"
+            end
+
+            log_entry = "  \e[#{message_color}m#{message}\e[0m   "
+            log_entry << "\e[#{dump_color}m%#{String === dump ? 's' : 'p'}\e[0m" % dump if dump
+            log_entry
+          else
+            "%s  %s" % [message, dump]
+          end
+        end
+    end
+
+    class TableDefinition
+
+      def initialize
+        @column_families = []
+      end
+
+      # Returns a column family for the column with name +name+.
+      def [](name)
+        @column_families.find {|column| column.name.to_s == name.to_s}
+      end
+
+      def column_family(name, options = {})
+        column = self[name] || BigRecordDriver::ColumnDescriptor.new(name.to_s, options)
+
+        @column_families << column unless @column_families.include? column
+        self
+      end
+
+      alias :family :column_family
+
+      def to_adapter_format
+        @column_families
+      end
+
+      def column_families_list
+        @column_families.map(&:name).join(", ")
+      end
+
+    end
+
+  end
+end

data/lib/big_record/connection_adapters/view.rb

@@ -0,0 +1,27 @@
+module BigRecord
+  module ConnectionAdapters
+    class View
+      attr_reader :name, :owner
+
+      def initialize(name, column_names, owner)
+        @name = name.to_s
+        @column_names = column_names ? column_names.collect{|c| c.to_s} : nil
+        @owner = owner
+      end
+
+      # Return the column objects associated with this view. By default the views 'all' and 'default' return every column.
+      def columns
+        if @column_names
+          @column_names.collect{|cn| owner.columns_hash[cn]}
+        else
+          owner.columns
+        end
+      end
+
+      # Return the name of the column objects associated with this view. By default the views 'all' and 'default' return every column.
+      def column_names
+        @column_names || owner.column_names
+      end
+    end
+  end
+end

data/lib/big_record/connection_adapters.rb

@@ -0,0 +1,10 @@
+dir = File.expand_path(File.join(File.dirname(__FILE__), "connection_adapters"))
+
+require dir + '/column'
+require dir + '/view'
+require dir + '/abstract/database_statements'
+require dir + '/abstract/quoting'
+require dir + '/abstract/connection_specification'
+
+require dir + '/abstract_adapter'
+require dir + '/hbase_adapter'

data/lib/big_record/deletion.rb

@@ -0,0 +1,73 @@
+module BigRecord
+  module Deletion
+    def self.included(base) #:nodoc:
+      base.alias_method_chain :destroy_without_callbacks, :flag_deleted
+      base.extend ClassMethods
+
+      base.class_eval do
+        class << self
+          alias_method_chain :find_one, :flag_deleted
+          alias_method_chain :find_every, :flag_deleted
+        end
+      end
+
+    end
+
+    # Flag the record as "deleted" if it responds to "deleted", else destroy it
+    def destroy_without_callbacks_with_flag_deleted #:nodoc:
+      if self.respond_to?(:deleted)
+        # mark as deleted
+        self.deleted = true
+
+        # set the timestamp
+        if record_timestamps
+          t = self.class.default_timezone == :utc ? Time.now.utc : Time.now
+          self.send(:updated_at=, t) if respond_to?(:updated_at)
+          self.send(:updated_on=, t) if respond_to?(:updated_on)
+        end
+
+        self.update_without_callbacks
+      else
+        destroy_without_callbacks_without_flag_deleted
+      end
+    end
+
+    module ClassMethods
+      def find_one_with_flag_deleted(*args)
+        options = args.last.is_a?(Hash) ? args.last : {}
+        records = find_one_without_flag_deleted(*args)
+        unless options[:include_deleted]
+          if records.is_a?(Array)
+            records.each{|record| check_not_deleted(record)}
+          else
+            check_not_deleted(records)
+          end
+        end
+        records
+      end
+
+      def find_every_with_flag_deleted(*args)
+        options = args.last.is_a?(Hash) ? args.last : {}
+        records = find_every_without_flag_deleted(*args)
+
+        unless options[:include_deleted]
+          records.select do |record|
+            begin
+              check_not_deleted(record)
+              true
+            rescue
+              false
+            end
+          end
+        else
+          records
+        end
+      end
+
+      def check_not_deleted(record)
+        raise BigRecord::RecordNotFound, "The record (id=#{record.id}) is marked as deleted." if record.respond_to?(:deleted) and record.deleted
+      end
+    end
+
+  end
+end

data/lib/big_record/dynamic_schema.rb

@@ -0,0 +1,92 @@
+# Replace the anonymous classes
+module BigRecord
+  module DynamicSchema
+
+    def self.included(base) #:nodoc:
+      super
+
+      base.alias_method_chain :column_for_attribute, :dynamic_schema
+      base.alias_method_chain :attributes_from_column_definition, :dynamic_schema
+      base.alias_method_chain :inspect, :dynamic_schema
+      base.alias_method_chain :define_read_methods, :dynamic_schema
+
+    end
+
+    # Stub of the callback for setting the dynamic columns. Override this to add dynamic columns
+    def initialize_columns(options={})
+
+    end
+
+    # Create and add a dynamic column to this record
+    def dynamic_column(name, type, options={})
+      add_dynamic_column ConnectionAdapters::Column.new(name.to_s, type, options)
+    end
+
+    # Add an existing dynamic column to this record
+    def add_dynamic_column(c)
+      columns_hash[c.name] = c
+      @columns_name= nil; @columns= nil #reset
+      c
+    end
+
+    def columns_hash
+      unless @columns_hash
+        @columns_hash = self.class.columns_hash.dup
+        initialize_columns
+      end
+      @columns_hash
+    end
+
+    def columns
+      @columns ||= columns_hash.values
+    end
+
+    def column_names
+      @column_names ||= columns_hash.keys
+    end
+
+    # Returns the column object for the named attribute.
+    def column_for_attribute_with_dynamic_schema(name)
+      self.columns_hash[name.to_s]
+    end
+
+    # Initializes the attributes array with keys matching the columns from the linked table and
+    # the values matching the corresponding default value of that column, so
+    # that a new instance, or one populated from a passed-in Hash, still has all the attributes
+    # that instances loaded from the database would.
+    def attributes_from_column_definition_with_dynamic_schema
+      self.columns.inject({}) do |attributes, column|
+        unless column.name == self.class.primary_key
+          attributes[column.name] = column.default
+        end
+        attributes
+      end
+    end
+
+    # Returns the contents of the record as a nicely formatted string.
+    def inspect_with_dynamic_schema
+      attributes_as_nice_string = self.column_names.collect { |name|
+        if has_attribute?(name) || new_record?
+          "#{name}: #{attribute_for_inspect(name)}"
+        end
+      }.compact.join(", ")
+      "#<#{self.class} #{attributes_as_nice_string}>"
+    end
+
+    # Called on first read access to any given column and generates reader
+    # methods for all columns in the columns_hash if
+    # ActiveRecord::Base.generate_read_methods is set to true.
+    def define_read_methods_with_dynamic_schema
+      columns_hash.each do |name, column|
+        unless respond_to_without_attributes?(name)
+          define_read_method(name.to_sym, name, column)
+        end
+
+        unless respond_to_without_attributes?("#{name}?")
+          define_question_method(name)
+        end
+      end
+    end
+
+  end
+end

data/lib/big_record/embedded.rb

@@ -0,0 +1,71 @@
+module BigRecord
+  class Embedded < AbstractBase
+
+    def initialize(attrs = nil)
+      super
+      # Regenerate the id unless it's already there (i.e. we're instantiating an existing property)
+      @attributes["id"] ||= generate_id
+    end
+
+    def connection
+      self.class.connection
+    end
+
+    def id
+      super || (self.id = generate_id)
+    end
+
+  protected
+    def generate_id
+      UUIDTools::UUID.random_create.to_s
+    end
+
+  public
+    class << self
+      def store_primary_key?
+        true
+      end
+
+      def primary_key
+        "id"
+      end
+
+      # Borrow the default connection of BigRecord
+      def connection
+        BigRecord::Base.connection
+      end
+
+      def base_class
+        (superclass == BigRecord::Embedded) ? self : superclass.base_class
+      end
+
+      # Class attribute that holds the name of the embedded type for dispaly
+      def pretty_name
+        @pretty_name || self.to_s
+      end
+
+      def set_pretty_name new_name
+        @pretty_name = new_name
+      end
+
+      def hide_to_users
+        @hide_to_user = true
+      end
+
+      def show_to_users?
+        !@hide_to_user
+      end
+
+      def inherited(child) #:nodoc:
+        child.set_pretty_name child.name.split("::").last
+        super
+      end
+
+      def default_columns
+        {primary_key => ConnectionAdapters::Column.new(primary_key, 'string')}
+      end
+
+    end
+
+  end
+end