massive_record 0.2.0 → 0.2.1.rc1

data/lib/massive_record/orm/attribute_methods/write.rb

@@ -9,6 +9,24 @@ module MassiveRecord
         end
         
 
+        module ClassMethods
+          protected
+
+          def define_method_attribute=(attr_name)
+            if attr_name =~ ActiveModel::AttributeMethods::COMPILABLE_REGEXP
+              generated_attribute_methods.module_eval <<-RUBY, __FILE__, __LINE__
+                def #{attr_name}=(value)
+                  write_attribute('#{attr_name}', value)
+                end
+              RUBY
+            else
+              generated_attribute_methods.send(:define_method, "#{attr_name}=") do |value|
+                write_attribute(attr_name, value)
+              end
+            end
+          end
+        end
+
         def write_attribute(attr_name, value)
           @attributes[attr_name.to_s] = value
         end

data/lib/massive_record/orm/base.rb

@@ -4,6 +4,7 @@ require 'active_support/core_ext/class/attribute'
 require 'active_support/core_ext/class/subclasses'
 require 'active_support/core_ext/module'
 require 'active_support/core_ext/string'
+require 'active_support/core_ext/array'
 require 'active_support/memoizable'
 
 require 'massive_record/orm/schema'
@@ -13,7 +14,9 @@ require 'massive_record/orm/config'
 require 'massive_record/orm/relations'
 require 'massive_record/orm/finders'
 require 'massive_record/orm/finders/scope'
+require 'massive_record/orm/finders/rescue_missing_table_on_find'
 require 'massive_record/orm/attribute_methods'
+require 'massive_record/orm/attribute_methods/time_zone_conversion'
 require 'massive_record/orm/attribute_methods/write'
 require 'massive_record/orm/attribute_methods/read'
 require 'massive_record/orm/attribute_methods/dirty'
@@ -22,6 +25,8 @@ require 'massive_record/orm/validations'
 require 'massive_record/orm/callbacks'
 require 'massive_record/orm/timestamps'
 require 'massive_record/orm/persistence'
+require 'massive_record/orm/default_id'
+require 'massive_record/orm/query_instrumentation'
 
 
 module MassiveRecord
@@ -35,6 +40,19 @@ module MassiveRecord
       # Accepts a logger conforming to the interface of Log4r or the default Ruby 1.8+ Logger class,
       cattr_accessor :logger, :instance_writer => false
 
+
+      #
+      # Integers are now persisted as a hax representation in hbase, not as
+      # a string any more. This makes for instance atomic_increment!(:int_attr) work
+      # as expected.
+      #
+      # The problem is that if you have old data in your database, you need to handle
+      # this. Every new integer will still be written as hex though.
+      #
+      cattr_accessor :backward_compatibility_integers_might_be_persisted_as_strings, :instance_writer => false
+      self.backward_compatibility_integers_might_be_persisted_as_strings = false
+
+
       # Add a prefix or a suffix to the table name
       # example:
       #
@@ -47,6 +65,18 @@ module MassiveRecord
       
       class_attribute :table_name_suffix, :instance_writer => false
       self.table_name_suffix = ""
+
+      #
+      # Will do a simple exists?(id) check before create as a simple (and
+      # kinda insecure) sanity check on if that ID exists or not. If it do
+      # exists a RecordNotUnique will be raised. This is done from the ORM
+      # layer, so obviously there is a speed cost on create.
+      #
+      class_attribute :check_record_uniqueness_on_create, :instance_writer => false
+      self.check_record_uniqueness_on_create = false
+
+      class_attribute :auto_increment_id, :instance_writer => false
+      self.auto_increment_id = true
      
       class << self
         def table_name
@@ -104,15 +134,20 @@ module MassiveRecord
       # and assign to instance variables. How read- and write 
       # methods are defined might change over time when the DSL
       # for describing column families and fields are in place
+      # You can call initialize in multiple ways:
+      #   ORMClass.new(attr_one: value, attr_two: value)
+      #   ORMClass.new("the-id-of-the-new-record")
+      #   ORMClass.new("the-id-of-the-new-record", attr_one: value, attr_two: value)
       #
-      def initialize(attributes = {})
+      def initialize(*args)
+        attributes = args.extract_options!
+        id = args.first
+
         @new_record = true
         @destroyed = @readonly = false
         @relation_proxy_cache = {}
 
-        attributes = {} if attributes.nil?
-
-        self.attributes_raw = attributes_from_field_definition
+        self.attributes_raw = attributes_from_field_definition.merge('id' => id)
         self.attributes = attributes
 
         clear_dirty_states!
@@ -186,6 +221,11 @@ module MassiveRecord
         read_attribute(:id)
       end
 
+      def id=(id)
+        id = id.to_s unless id.blank?
+        write_attribute(:id, id)
+      end
+
 
 
       def readonly?
@@ -238,17 +278,20 @@ module MassiveRecord
 
     Base.class_eval do
       include Config
-      include Relations::Interface
       include Persistence
+      include Relations::Interface
       include Finders
-      include ActiveModel::Translation
+      extend  RescueMissingTableOnFind
       include AttributeMethods
       include AttributeMethods::Write, AttributeMethods::Read
+      include AttributeMethods::TimeZoneConversion
       include AttributeMethods::Dirty
       include Validations
       include Callbacks
       include Timestamps
       include SingleTableInheritance
+      include DefaultId
+      include QueryInstrumentation
 
 
       alias [] read_attribute
@@ -257,10 +300,9 @@ module MassiveRecord
   end
 end
 
-
-
-
-
 require 'massive_record/orm/table'
 require 'massive_record/orm/column'
 require 'massive_record/orm/id_factory'
+require 'massive_record/orm/log_subscriber'
+
+ActiveSupport.run_load_hooks(:massive_record, MassiveRecord::ORM::Base)

data/lib/massive_record/orm/callbacks.rb

@@ -44,7 +44,7 @@ module MassiveRecord
         _run_create_callbacks { super }
       end 
 
-      def update
+      def update(*)
         _run_update_callbacks { super }
       end
     end

data/lib/massive_record/orm/default_id.rb

@@ -0,0 +1,20 @@
+module MassiveRecord
+  module ORM
+    module DefaultId
+      extend ActiveSupport::Concern
+
+      included do
+        before_create :ensure_record_has_id, :if => :auto_increment_id
+      end
+
+
+      module InstanceMethods
+        private
+
+        def ensure_record_has_id
+          self.id = next_id if id.blank?
+        end
+      end
+    end
+  end
+end

data/lib/massive_record/orm/errors.rb

@@ -20,6 +20,10 @@ module MassiveRecord
     class RecordNotFound < MassiveRecordError
     end
 
+    # Raised when we try to create a new record with an id which exists.
+    class RecordNotUnique < MassiveRecordError
+    end
+
     # Raised if an attribute is unkown
     class UnknownAttributeError < MassiveRecordError
     end

data/lib/massive_record/orm/finders.rb

@@ -12,6 +12,79 @@ module MassiveRecord
       end
 
       module ClassMethods
+        #
+        # Find records in batches. Makes it easier to work with
+        # big data sets where you don't want to load every record up front.
+        #
+        def find_in_batches(*args)
+          table.find_in_batches(*args) do |rows|
+            records = rows.collect do |row|
+              instantiate(transpose_hbase_columns_to_record_attributes(row))
+            end    
+            yield records
+          end
+        end
+        
+        #
+        # Similar to all, except that this will use find_in_batches
+        # behind the scene.
+        #
+        def find_each(*args)
+          find_in_batches(*args) do |rows|
+            rows.each do |row|
+              yield row
+            end
+          end
+        end
+
+
+        #
+        # Returns true if a record do exist
+        #
+        def exists?(id)
+          !!find(id) rescue false
+        end
+
+
+
+        #
+        # Entry point for method delegation like find, first, all etc.
+        #
+        def finder_scope
+          default_scoping || unscoped
+        end
+
+
+        #
+        # Sets a default scope which will be used for calls like find, first, all etc.
+        # Makes it possible to for instance set default column families to load on all
+        # calls to the database.
+        #
+        def default_scope(scope)
+          self.default_scoping =  case scope
+                                    when Scope, nil
+                                      scope
+                                    when Hash
+                                      Scope.new(self, :find_options => scope)
+                                    else
+                                      raise "Don't know how to set scope with #{scope.class}."
+                                    end
+        end
+
+        #
+        # Returns an fresh scope object with no limitations set by
+        # for instance the default scope
+        #
+        def unscoped
+          Scope.new(self)
+        end
+
+
+
+
+        #
+        # This do_find method is not very nice it's logic should be re-factored at some point.
+        #
         def do_find(*args) # :nodoc:
           options = args.extract_options!.to_options
           raise ArgumentError.new("At least one argument required!") if args.empty?
@@ -25,26 +98,10 @@ module MassiveRecord
           type = args.shift if args.first.is_a? Symbol
           find_many = type == :all
           expected_result_size = nil
-
-          return (find_many ? [] : raise(RecordNotFound.new("Could not find #{model_name} with id=#{args.first}"))) unless table.exists?
+          what_to_find = []
+          result_from_table = []
           
-          result_from_table = if type
-                                table.send(type, *args) # first() / all()
-                              else
-                                options = args.extract_options!
-                                what_to_find = args.first
-                                expected_result_size = 1
-
-                                if args.first.kind_of?(Array)
-                                  find_many = true
-                                elsif args.length > 1
-                                  find_many = true
-                                  what_to_find = args
-                                end
-
-                                expected_result_size = what_to_find.length if what_to_find.is_a? Array
-                                table.find(what_to_find, options)
-                              end
+          find_many, expected_result_size, what_to_find, result_from_table = query_hbase(type, args, find_many)
 
           # Filter out unexpected IDs (unless type is set (all/first), in that case
           # we have no expectations on the returned rows' ids)
@@ -71,54 +128,42 @@ module MassiveRecord
           find_many ? records : records.first
         end
 
-        def find_in_batches(*args)
-          return unless table.exists?
 
-          table.find_in_batches(*args) do |rows|
-            records = rows.collect do |row|
-              instantiate(transpose_hbase_columns_to_record_attributes(row))
-            end    
-            yield records
-          end
-        end
-        
-        def find_each(*args)
-          find_in_batches(*args) do |rows|
-            rows.each do |row|
-              yield row
-            end
-          end
-        end
 
 
-        def exists?(id)
-          !!find(id) rescue false
-        end
+        private
 
+        def query_hbase(type, args, find_many) # :nodoc:
+          result_from_table = if type
+                                hbase_query_all_first(type, args)
+                              else
+                                options = args.extract_options!
+                                what_to_find = args.first
+                                expected_result_size = 1
 
-        def finder_scope
-          default_scoping || unscoped
-        end
+                                if args.first.kind_of?(Array)
+                                  find_many = true
+                                elsif args.length > 1
+                                  find_many = true
+                                  what_to_find = args
+                                end
 
-        def default_scope(scope)
-          self.default_scoping =  case scope
-                                    when Scope, nil
-                                      scope
-                                    when Hash
-                                      Scope.new(self, :find_options => scope)
-                                    else
-                                      raise "Don't know how to set scope with #{scope.class}."
-                                    end
-        end
+                                expected_result_size = what_to_find.length if what_to_find.is_a? Array
+                                hbase_query_find(what_to_find, options)
+                              end
 
-        def unscoped
-          Scope.new(self)
+          [find_many, expected_result_size, what_to_find, result_from_table]
         end
 
+        def hbase_query_all_first(type, args)
+          table.send(type, *args) # first() / all()
+        end
 
-        private
+        def hbase_query_find(what_to_find, options)
+          table.find(what_to_find, options)
+        end
 
-        def transpose_hbase_columns_to_record_attributes(row)
+        def transpose_hbase_columns_to_record_attributes(row) #: nodoc:
           attributes = {:id => row.id}
           
           autoload_column_families_and_fields_with(row.columns.keys)
@@ -131,7 +176,7 @@ module MassiveRecord
           attributes
         end
 
-        def instantiate(record)
+        def instantiate(record) # :nodoc:
           model = if klass = record[inheritance_attribute] and klass.present?
                     klass.constantize.allocate
                   else

data/lib/massive_record/orm/finders/rescue_missing_table_on_find.rb

@@ -0,0 +1,45 @@
+module MassiveRecord
+  module ORM
+    #
+    # Module which adds functionality so we rescue errors which might occur on
+    # find calls when we are querying tables which does not exist.
+    # Small problem with this, which will need to look into.
+    #
+    module RescueMissingTableOnFind
+      def do_find(*args)
+        create_table_and_retry_if_table_missing { super }
+      end
+
+      def find_in_batches(*args) 
+        create_table_and_retry_if_table_missing { super }
+      end
+
+
+
+      private
+
+
+      #
+      # Yields the block and if any errors occur we will check if table does exist or not.
+      # Create it if it's missing and try again.
+      #
+      # Errors which we'll retry on are:
+      #   Apache::Hadoop::Hbase::Thrift::IOError          -> Raised on simple find(id) calls
+      #   Apache::Hadoop::Hbase::Thrift::IllegalArgument  -> Raised when a scanner is used
+      #
+      def create_table_and_retry_if_table_missing # :nodoc:
+        begin
+          yield
+        rescue Apache::Hadoop::Hbase::Thrift::IOError, Apache::Hadoop::Hbase::Thrift::IllegalArgument => error
+          if table.exists?
+            raise error
+          else
+            logger.try :info, "*** TABLE MISSING: Table '#{table_name}' seems to be missing. Will create it, then retry call to find()."
+            hbase_create_table!
+            yield
+          end
+        end
+      end
+    end
+  end
+end

data/lib/massive_record/orm/id_factory.rb

@@ -38,7 +38,7 @@ module MassiveRecord
         if table_exists?
           begin
             if @instance
-              @instance.reload
+              @instance.reload # If, for some reason, the record has been removed. Will be rescued and set to nil
             else
               @instance = find(ID)
             end