bigrecord 0.0.7 → 0.0.8

data/lib/big_record/connection_adapters/abstract/quoting.rb

@@ -2,7 +2,7 @@ module BigRecord
   module ConnectionAdapters # :nodoc:
     module Quoting
       # Quotes the column value to help prevent
-      # {SQL injection attacks}[http://en.wikipedia.org/wiki/SQL_injection].
+      # SQL injection attacks [http://en.wikipedia.org/wiki/SQL_injection].
       def quote(value, column = nil)
         # records are quoted as their primary key
         return value.quoted_id if value.respond_to?(:quoted_id)

data/lib/big_record/connection_adapters/column.rb

@@ -4,6 +4,36 @@ require 'bigdecimal/util'
 
 module BigRecord
   module ConnectionAdapters #:nodoc:
+
+    # = Column/Attribute Definition
+    #
+    # As long as a model has at least one column family set up for it, then
+    # columns (a.k.a. model attributes) can then be defined for the model.
+    #
+    # The following is an example of a model named book.rb that has a
+    # column family called "attribute" set up for it:
+    #
+    #   class Book < BigRecord::Base
+    #     column 'attribute:title',   :string
+    #     column :author,             :string
+    #     column :description,        :string
+    #     column :links,              :string,  :collection => true
+    #   end
+    #
+    # This simple model defines 4 columns of type string. An important thing
+    # to notice here is that the first column 'attribute:title' has the column
+    # family prepended to it. This is identical to just passing the symbol
+    # :title to the column method, and the default behaviour is to prepend
+    # the column family (attribute) automatically if one is not defined.
+    #
+    # Furthermore, in HBase, there's the option of storing collections for a
+    # given column. This will return an array for the links attribute on a
+    # Book record.
+    #
+    # == Types and Options
+    #
+    # @see BigRecord::AbstractBase.column
+    #
     class Column
       module Format
         ISO_DATE = /\A(\d{4})-(\d\d)-(\d\d)\z/

data/lib/big_record/connection_adapters/hbase_rest_adapter.rb

@@ -0,0 +1,424 @@
+require 'set'
+require 'hbase'
+
+module BigRecord
+  class Base
+    # Establishes a connection to the database that's used by all Active Record objects.
+    def self.hbase_rest_connection(config) # :nodoc:
+      config = config.symbolize_keys
+
+      api_address = config[:api_address]
+
+      hbase = HBase::Client.new(api_address)
+
+      ConnectionAdapters::HbaseRestAdapter.new(hbase, logger, [], config)
+    end
+  end
+
+  module ConnectionAdapters
+    class HbaseRestAdapter < AbstractAdapter
+      @@emulate_booleans = true
+      cattr_accessor :emulate_booleans
+
+      LOST_CONNECTION_ERROR_MESSAGES = [
+        "Server shutdown in progress",
+        "Broken pipe",
+        "Lost connection to HBase server during query",
+        "HBase server has gone away"
+      ]
+
+      # data types
+      TYPE_NULL     = 0x00;
+      TYPE_STRING   = 0x01; # utf-8 strings
+      TYPE_BOOLEAN  = 0x04; # delegate to YAML
+      TYPE_BINARY   = 0x07; # byte[] => no conversion
+
+      # string charset
+      CHARSET = "utf-8"
+
+      # utility constants
+      NULL = "\000"
+
+      def initialize(connection, logger, connection_options, config)
+        super(connection, logger)
+        @connection_options, @config = connection_options, config
+
+        connect
+      end
+
+      def configuration
+        @config.clone
+      end
+
+      def adapter_name #:nodoc:
+        'HBase-Rest'
+      end
+
+      def supports_migrations? #:nodoc:
+        true
+      end
+
+      # CONNECTION MANAGEMENT ====================================
+
+      def active?
+        true
+      end
+
+      def reconnect!
+      end
+
+      def disconnect!
+      end
+
+
+      # DATABASE STATEMENTS ======================================
+
+      def columns_to_hbase_format(data = {})
+        # Convert it to the hbase-ruby format
+        # TODO: Add this function to hbase-ruby instead.
+        data.map{|col, content| {:name => col.to_s, :value => content}}.compact
+      end
+
+      def update_raw(table_name, row, values, timestamp)
+        result = nil
+
+        columns = columns_to_hbase_format(values)
+        timestamp = Time.now.to_bigrecord_timestamp
+
+        log "UPDATE #{table_name} SET #{values.inspect if values} WHERE ROW=#{row};" do
+          @connection.create_row(table_name, row, timestamp, columns)
+        end
+        result
+      end
+
+      def update(table_name, row, values, timestamp)
+        serialized_collection = {}
+        values.each do |column, value|
+          serialized_collection[column] = serialize(value)
+        end
+        update_raw(table_name, row, serialized_collection, timestamp)
+      end
+
+      def get_raw(table_name, row, column, options={})
+        result = nil
+        timestamp = options[:timestamp] || nil
+        log "SELECT (#{column}) FROM #{table_name} WHERE ROW=#{row};" do
+          columns = @connection.show_row(table_name, row, timestamp, column, options).columns
+
+          result = (columns.size == 1) ? columns.first.value : columns.map(&:value)
+        end
+        result
+      end
+
+      def get(table_name, row, column, options={})
+        serialized_result = get_raw(table_name, row, column, options)
+        result = nil
+        if serialized_result.is_a?(Array)
+          result = serialized_result.collect{|e| deserialize(e)}
+        else
+          result = deserialize(serialized_result)
+        end
+        result
+      end
+
+      def get_columns_raw(table_name, row, columns = nil, options={})
+        result = {}
+
+        timestamp = options[:timestamp] || nil
+
+        log "SELECT (#{columns.join(", ")}) FROM #{table_name} WHERE ROW=#{row};" do
+          row = @connection.show_row(table_name, row, timestamp, columns, options)
+          result.merge!({'id' => row.name})
+          columns = row.columns
+          columns.each{ |col| result.merge!({col.name => col.value}) }
+        end
+        result
+      end
+
+      def get_columns(table_name, row, columns, options={})
+        row_cols = get_columns_raw(table_name, row, columns, options)
+        return nil unless row_cols
+
+        result = {}
+        row_cols.each do |key,value|
+          begin
+            result[key] =
+            if key == 'id'
+              value
+            else
+              deserialize(value)
+            end
+          rescue Exception => e
+            puts "Could not load column value #{key} for row=#{row.name}"
+          end
+        end
+        result
+      end
+
+      def get_consecutive_rows_raw(table_name, start_row, limit, columns, stop_row = nil)
+        result = nil
+        log "SCAN (#{columns.join(", ")}) FROM #{table_name} WHERE START_ROW=#{start_row} AND STOP_ROW=#{stop_row} LIMIT=#{limit};" do
+          options = {:start_row => start_row, :end_row => stop_row, :columns => columns}
+          scanner = @connection.open_scanner(table_name, options)
+          result = @connection.get_rows(scanner, limit)
+          @connection.close_scanner(scanner)
+        end
+        result
+      end
+
+      def get_consecutive_rows(table_name, start_row, limit, columns, stop_row = nil)
+        rows = get_consecutive_rows_raw(table_name, start_row, limit, columns, stop_row)
+
+        result = rows.collect do |row|
+          cols = {}
+          row.columns.each do |col|
+            key = col.name
+            value = col.value
+            begin
+              cols[key] =
+              if key == 'id'
+                value
+              else
+                deserialize(value)
+              end
+            rescue Exception => e
+              puts "Could not load column value #{key} for row=#{row.name}"
+            end
+          end
+          cols
+        end
+        result
+      end
+
+      def delete(table_name, row, timestamp = nil)
+        timestamp ||= Time.now.to_bigrecord_timestamp
+        result = nil
+        log "DELETE FROM #{table_name} WHERE ROW=#{row};" do
+          result = @connection.delete_row(table_name, row, timestamp)
+        end
+        result
+      end
+
+      def truncate_table(table_name)
+      end
+
+
+      # SCHEMA STATEMENTS ========================================
+
+      def initialize_schema_migrations_table
+        sm_table = BigRecord::Migrator.schema_migrations_table_name
+
+        unless table_exists?(sm_table)
+          create_table(sm_table) do |t|
+            t.family :attribute, :versions => 1
+          end
+        end
+      end
+
+      def get_all_schema_versions
+        sm_table = BigRecord::Migrator.schema_migrations_table_name
+
+        get_consecutive_rows(sm_table, nil, nil, ["attribute:version"]).map{|version| version["attribute:version"]}
+      end
+
+      def table_exists?(table_name)
+        log "TABLE EXISTS? #{table_name};" do
+          @connection.list_tables.map(&:name).include?(table_name)
+        end
+      end
+
+      def create_table(table_name, options = {})
+        table_definition = TableDefinition.new
+
+        yield table_definition if block_given?
+
+        if options[:force] && table_exists?(table_name)
+          drop_table(table_name)
+        end
+
+        result = nil
+        log "CREATE TABLE #{table_name} (#{table_definition.column_families_list});" do
+          result = @connection.create_table(table_name, *table_definition.to_adapter_format)
+        end
+        result
+      end
+
+      def drop_table(table_name)
+        result = nil
+        log "DROP TABLE #{table_name};" do
+          result = @connection.destroy_table(table_name)
+        end
+        result
+      end
+
+      def add_column_family(table_name, column_name, options = {})
+      end
+
+      alias :add_family :add_column_family
+
+      def remove_column_family(table_name, column_name)
+      end
+
+      alias :remove_family :remove_column_family
+
+      def modify_column_family(table_name, column_name, options = {})
+      end
+
+      alias :modify_family :modify_column_family
+
+      # Serialize the given value
+      def serialize(value)
+        case value
+        when NilClass then NULL
+        when String then build_serialized_value(TYPE_STRING, value)
+        else value.to_yaml
+        end
+      end
+
+      # Serialize an object in a given type
+      def build_serialized_value(type, value)
+        type.chr + value
+      end
+
+      # Deserialize the given string. This method supports both the pure YAML format and
+      # the type header format.
+      def deserialize(str)
+        return unless str
+
+        #	stay compatible with the old serialization code
+        #	YAML documents start with "--- " so if we find that sequence at the beginning we
+        #	consider it as a serialized YAML value, else it's the new format with the type header
+        if str[0..3] == "--- "
+          YAML::load(str) if str
+        else
+          deserialize_with_header(str)
+        end
+      end
+
+      # Deserialize the given string assumed to be in the type header format.
+      def deserialize_with_header(data)
+        return unless data and data.size >= 2
+
+        # the type of the data is encoded in the first byte
+        type = data[0];
+
+        case type
+        when TYPE_NULL then nil
+        when TYPE_STRING then data[1..-1]
+        when TYPE_BINARY then data[1..-1]
+        else nil
+        end
+      end
+
+      private
+
+        def connect
+        end
+
+      protected
+        def log(str, name = nil)
+          if block_given?
+            if @logger and @logger.level <= Logger::INFO
+              result = nil
+              seconds = Benchmark.realtime { result = yield }
+              @runtime += seconds
+              log_info(str, name, seconds)
+              result
+            else
+              yield
+            end
+          else
+            log_info(str, name, 0)
+            nil
+          end
+        rescue Exception => e
+          # Log message and raise exception.
+          # Set last_verfication to 0, so that connection gets verified
+          # upon reentering the request loop
+          @last_verification = 0
+          message = "#{e.class.name}: #{e.message}: #{str}"
+          log_info(message, name, 0)
+          raise e
+        end
+
+        def log_info(str, name, runtime)
+          return unless @logger
+
+          @logger.debug(
+            format_log_entry(
+              "#{name.nil? ? "HBASE" : name} (#{sprintf("%f", runtime)})",
+              str.gsub(/ +/, " ")
+            )
+          )
+        end
+
+        def format_log_entry(message, dump = nil)
+          if BigRecord::Base.colorize_logging
+            if @@row_even
+              @@row_even = false
+              message_color, dump_color = "4;36;1", "0;1"
+            else
+              @@row_even = true
+              message_color, dump_color = "4;35;1", "0"
+            end
+
+            log_entry = "  \e[#{message_color}m#{message}\e[0m   "
+            log_entry << "\e[#{dump_color}m%#{String === dump ? 's' : 'p'}\e[0m" % dump if dump
+            log_entry
+          else
+            "%s  %s" % [message, dump]
+          end
+        end
+    end
+
+    class TableDefinition
+
+      ##
+      # Given an column descriptor's options hash from Bigrecord, translate it into
+      # the format for this adapter's ColumnDescriptor.
+      #
+      def self.translate_to_adapter_format(options)
+        # Translating to the hbase-ruby column descriptor
+        # TODO: Refactor this
+        hbase_params = {}
+
+        hbase_params[:name] = options[:name].to_s if options[:name]
+        hbase_params[:compression] = options[:compression] if options[:compression]
+        hbase_params[:max_versions] = options[:versions] if options[:versions]
+        hbase_params[:bloomfilter] = options[:bloom_filter] if options[:bloom_filter]
+
+        hbase_params
+      end
+
+      def initialize
+        @column_families = []
+      end
+
+      # Returns a column family for the column with name +name+.
+      def [](name)
+        @column_families.find {|column| column[:name].to_s == name.to_s}
+      end
+
+      def column_family(name, options = {})
+        hbase_params = self.class.translate_to_adapter_format(options.merge({:name => name}))
+
+        column = self[name] || hbase_params
+
+        @column_families << column unless @column_families.include? column
+        self
+      end
+
+      alias :family :column_family
+
+      def to_adapter_format
+        @column_families
+      end
+
+      def column_families_list
+        @column_families.map{|x| x[:name]}.join(", ")
+      end
+
+    end
+
+  end
+end

data/lib/big_record/connection_adapters/view.rb

@@ -1,5 +1,68 @@
 module BigRecord
   module ConnectionAdapters
+
+    # = Named Views
+    #
+    # To use column-oriented databases more efficiently, it helps to know
+    # exactly which specific columns are required for a query.
+    #
+    # For example, supposing we're designing two queries:
+    # (A) used on the front page of a website.
+    # (B) used on the summary page for a particular item.
+    #
+    # Since we know these two queries will be serving up different information,
+    # we can also figure out which columns are needed by each. In general,
+    # query (A) would only require a subset of columns of (B). This
+    # functionality is analogous to the SQL command "SELECT column1,column2 ..."
+    # for defining specific columns to query for.
+    #
+    # Named views are defined in models by using the {BigRecord::Base.view}
+    # macro like in the following example:
+    #
+    #   class Book < BigRecord::Base
+    #     column :title,   :string
+    #     column :author,             :string
+    #     column :description,        :string
+    #     column :links,              :string,  :collection => true
+    #
+    #     view :front_page, :title, :author
+    #     view :summary_page, :title, :author, :description
+    #
+    #     # Override default if you don't want all columns returned
+    #     view :default, :title, :author, :description
+    #   end
+    #
+    # Now, whenever you work with a Book record, it will only returned the
+    # columns you specify according to the view option you pass. i.e.
+    #
+    #   >> Book.find(:first, :view => :front_page)
+    #   => #<Book id: "2e13f182-1085-495e-9841-fe5c84ae9992", attribute:title: "Hello Thar", attribute:author: "Greg">
+    #
+    #   >> Book.find(:first, :view => :summary_page)
+    #   => #<Book id: "2e13f182-1085-495e-9841-fe5c84ae9992", attribute:description: "Masterpiece!", attribute:title: "Hello Thar", attribute:author: "Greg">
+    #
+    #   >> Book.find(:first, :view => :default)
+    #   => #<Book id: "2e13f182-1085-495e-9841-fe5c84ae9992", attribute:description: "Masterpiece!", attribute:title: "Hello Thar", attribute:links: ["link1", "link2", "link3", "link4"], attribute:author: "Greg">
+    #
+    # Any attributes that were not loaded in as part of the view will be
+    # lazy-loaded on request. This is very inefficient and should be avoided
+    # at all costs! It's better to load more columns in, than to have
+    # any attributes lazy-loaded afterwards.
+    #
+    # Note: A Bigrecord model will return all the columns within the default
+    # column family (when :view option is left blank, for example).
+    # You can override the :default name view to change this behaviour.
+    #
+    # It's also possible to define specific columns to load at query time.
+    # To do this, you can just pass an array of columns to the :columns
+    # option of the find method and it will return only those attributes:
+    #
+    #   >> Book.find(:first, :columns => [:author, :description])
+    #   => #<Book id: "2e13f182-1085-495e-9841-fe5c84ae9992", attribute:description: "Masterpiece!", attribute:author: "Greg">
+    #
+    # Once again, any attributes not loaded during query time will be
+    # lazy-loaded on request.
+    #
     class View
       attr_reader :name, :owner
 

data/lib/big_record/connection_adapters.rb

@@ -8,3 +8,4 @@ require dir + '/abstract/connection_specification'
 
 require dir + '/abstract_adapter'
 require dir + '/hbase_adapter'
+require dir + '/hbase_rest_adapter'

data/lib/big_record/embedded.rb

@@ -1,9 +1,134 @@
 module BigRecord
+
+  # = Embedded Records
+  #
+  # Since a single column in a column-oriented database is perfectly suited
+  # to handle large amounts of data, BigRecord gives you the option to store
+  # entire records within a single column of another {BigRecord::Base} record.
+  #
+  # These are known as Embedded records, and they behave similarly to
+  # {BigRecord::Base} objects, except that their data is physically stored
+  # within another {BigRecord::Base} record, and they don't exist unless
+  # associated with one. Furthermore, they don't possess any find or querying
+  # functionality.
+  #
+  # So what are the benefits of Embedded records?
+  # * Cleaner organization of models
+  # * Avoids the need to create entire tables and associations for models that
+  #   exist only in the context of another model.
+  # * Allows more complicated functionality to be encompassed within an
+  #   embedded record, instead of in a parent model.
+  #
+  # All of this has been very abstract so far, therefore examples are in order.
+  #
+  # == Examples
+  #
+  # Let's say we start off with the following arbitrarily created models:
+  #
+  # app/model/book.rb:
+  #   class Book < BigRecord::Base
+  #     column :title,       :string
+  #     column :author,      :string
+  #     column :description, :string
+  #   end
+  #
+  # app/model/company.rb:
+  #   class Company < BigRecord::Base
+  #     column :name,        :string
+  #     column :address,     :string
+  #     column :description, :string
+  #   end
+  #
+  # Now, let's say we want the ability to create and associate weblinks
+  # with each of these models. This is a trivial modification if all we do is
+  # create a new column in each model called "weblink" (or something similar),
+  # and have a string that stores a URL.
+  #
+  # However, what if we wanted these weblinks to have more metadata attached
+  # and more complex functionality added to it? Then our only choice is to
+  # create a new model called WebLink (for example), with its own table, set of
+  # attributes and methods. Then we have our Book and Company models associate
+  # to these newly created WebLink models, giving us something like this:
+  #
+  # app/model/web_link.rb:
+  #   class WebLink < BigRecord::Base
+  #     column :name,         :string
+  #     column :url,          :string
+  #     column :description,  :string
+  #     column :submitted_by, :string
+  #     column :book_id,      :string
+  #     column :company_id,   :string
+  #
+  #     # Could use a polymorphic association here, of course.
+  #     belongs_to_bigrecord :book, :foreign_key => "attribute:book_id"
+  #     belongs_to_bigrecord :company, :foreign_key => "attribute:company_id"
+  #
+  #     # other methods ...
+  #   end
+  #
+  # and likewise an association to WebLink from the Book and Company models.
+  #
+  # Now notice the problem here? A simple concept like adding a WebLink with
+  # some metadata has increased the model logic and created some unnecessary
+  # associations. In this situation, a WebLink doesn't need to exist except
+  # when associated to a certain model. Therefore this association should be
+  # implicit somehow.
+  #
+  # Enter Embedded records. We will now instead define WebLink as follows:
+  #
+  # app/model/web_link.rb:
+  #   class WebLink < BigRecord::Embedded
+  #     column :name,         :string
+  #     column :url,          :string
+  #     column :description,  :string
+  #     column :submitted_by, :string
+  #
+  #     # other methods ...
+  #   end
+  #
+  # And modify our Base records like so:
+  #
+  # app/model/book.rb:
+  #   class Book < BigRecord::Base
+  #     column :title,       :string
+  #     column :author,      :string
+  #     column :description, :string
+  #     column :web_link,    'WebLink'
+  #   end
+  #
+  # app/model/company.rb:
+  #   class Company < BigRecord::Base
+  #     column :name,        :string
+  #     column :address,     :string
+  #     column :description, :string
+  #     column :web_link,    'WebLink'
+  #   end
+  #
+  # Now we can encompass any WebLink specific attributes and methods within
+  # the WebLink embedded class, and use them with any other Base model.
+  #
+  # To use WebLink now, we treat it as though it were a regular model, except
+  # that we don't execute saves on it. For example:
+  #
+  #   >> amazon_link = WebLink.new(:name => "Amazon Link to Book", :url => "http://amazon.com/some/book", :description => "Amazon sells this book for cheap!")
+  #   => #<WebLink created_at: "2009-11-12 17:11:57", name: "Amazon Link to Book", url: "http://amazon.com/some/book", description: "Amazon sells this book for cheap!", submitted_by: nil, id: "2b619a68-e462-475d-8e04-01ba2aace11a">
+  #   >> amazon_link.save
+  #   BigRecord::NotImplemented: BigRecord::NotImplemented
+  #   # => [...]
+  #   >> book = Book.find(:first)
+  #   # => [...]
+  #   >> book.web_link = amazon_link
+  #   >> book.save
+  #
+  # Now any subsequent access to that book object we just saved to will have
+  # a WebLink record available with it.
+  #
   class Embedded < AbstractBase
 
     def initialize(attrs = nil)
       super
-      # Regenerate the id unless it's already there (i.e. we're instantiating an existing property)
+      # Regenerate the id unless it's already there
+      # (i.e. we're instantiating an existing property)
       @attributes["id"] ||= generate_id
     end
 
@@ -16,11 +141,13 @@ module BigRecord
     end
 
   protected
+
     def generate_id
       UUIDTools::UUID.random_create.to_s
     end
 
   public
+
     class << self
       def store_primary_key?
         true