massive_record 0.2.1 → 0.2.2.rc1

data/lib/massive_record/orm/column.rb

@@ -1,18 +1,14 @@
-require 'massive_record/orm/schema/column_interface'
-
 module MassiveRecord
   module ORM
     class Column < Base
-      include Schema::ColumnInterface
-
-      # TODO  Column does not support these kind of methods
-      class << self
-        undef_method :first, :last, :all, :exists?, :destroy_all
+      def self.inherited(by_class)
+        raise(<<-TXT
+            #{by_class} inherits from MassiveRecord::ORM::Column which has been renamed to
+            MassiveRecord::ORM::Embedded. Please inherit from the Embedded class instead as
+            Column will be removed in the an upcomming of MassiveRecord.
+          TXT
+        )
       end
-
-      undef_method :create, :reload, :save, :save!, :update_attribute, :update_attributes,
-        :update_attributes!, :touch, :destroy, :increment!, :atomic_increment!,
-        :decrement!, :delete
     end
   end
 end

data/lib/massive_record/orm/default_id.rb

@@ -4,7 +4,7 @@ module MassiveRecord
       extend ActiveSupport::Concern
 
       included do
-        before_create :ensure_record_has_id, :if => :auto_increment_id
+        before_create :ensure_record_has_id, :if => :set_id_from_factory_before_create
       end
 
 

data/lib/massive_record/orm/embedded.rb

@@ -0,0 +1,66 @@
+require 'massive_record/orm/schema/embedded_interface'
+
+module MassiveRecord
+  module ORM
+    class Embedded < Base
+      include Schema::EmbeddedInterface
+
+      DATABASE_ID_SEPARATOR = '|'
+
+
+      # TODO  Embedded does not support these kind of methods
+      class << self
+        undef_method :first, :last, :all, :exists?, :destroy_all
+      end
+
+
+      def self.parse_database_id(database_id)
+        if splitted = database_id.split(DATABASE_ID_SEPARATOR) and splitted.length == 2
+          splitted
+        else
+          fail InvalidEmbeddedDatabaseId.new(
+            <<-TXT
+              Expected database id '#{database_id}' to be on a format like
+              base_class_here#{DATABASE_ID_SEPARATOR}record_id_here
+            TXT
+          )
+        end
+      end
+
+      def self.database_id(klass, id)
+        [klass.base_class.to_s.underscore, id].join(DATABASE_ID_SEPARATOR)
+      end
+
+      #
+      # Database id is base_class plus the record's id.
+      # It is given, as we might want to embed records in an existing
+      # column family, or share a family for multiple types. In which case,
+      # we'll end up with a column family like this:
+      #
+      # | key           | attributes                                            |
+      # --------------------------------------------------------------------------
+      # | "address|123" | { :street => "Askerveien", :number => "12", etc... }  |
+      # | "address|124" | { :street => "Askerveien", :number => "12", etc... }  |
+      # | "name"        | "Thorbjorn Hermansen"                                 |
+      # | "age"         | "30"                                                  |
+      #
+      # ..in this case we fetch embedded records to collection addresses by scoping
+      # on keys which starts with base_class name. The records itself will only have
+      # id equal to 123 and 124.
+      #
+      def database_id # :nodoc:
+        if id
+          self.class.database_id(self.class, id)
+        end
+      end
+
+      #
+      # Writer for database id. Used when loading records to easily set record's id.
+      # Should not have the need to be used in other situations.
+      #
+      def database_id=(database_id) # :nodoc:
+        self.id = self.class.parse_database_id(database_id)[1]
+      end
+    end
+  end
+end

data/lib/massive_record/orm/errors.rb

@@ -66,8 +66,25 @@ module MassiveRecord
     class RelationTypeMismatch < MassiveRecordError
     end
 
+    # Used if we are dependent on a relation, but it does not seem to exist. For instance,
+    # embedded_in is dependent of it's inverse in the owner object.
+    class RelationMissing < MassiveRecordError
+    end
+
     # Raised when an attribute is decoded from the database, but the type returned does not match what is expected
     class SerializationTypeMismatch < MassiveRecordError
     end
+
+    # Raised when an ORM::Embedded is asked to save itself without being assigned a collection to be embedded in
+    class NotAssignedToEmbeddedCollection < MassiveRecordError
+      attr_reader :embedded_in_missing_values
+      def initialize(record, embedded_in_missing_values)
+        @embedded_in_missing_values = embedded_in_missing_values
+        super("#{record.inspect} needs to be embedded in collection before save. Embedded-in-attribute(s) missing assignment(s): #{embedded_in_missing_values.join(', ')}")
+      end
+    end
+
+    class InvalidEmbeddedDatabaseId < MassiveRecordError
+    end
   end
 end

data/lib/massive_record/orm/finders.rb

@@ -5,13 +5,17 @@ module MassiveRecord
 
       included do
         class << self
-          delegate :find, :first, :last, :all, :select, :limit, :to => :finder_scope
+          delegate :find, :last, :all, :select, :limit, :starts_with, :offset, :to => :finder_scope
         end
 
         class_attribute :default_scoping, :instance_writer => false
       end
 
       module ClassMethods
+        def first(*args)
+          finder_scope.first(*args)
+        end
+
         #
         # Find records in batches. Makes it easier to work with
         # big data sets where you don't want to load every record up front.
@@ -19,7 +23,7 @@ module MassiveRecord
         def find_in_batches(*args)
           table.find_in_batches(*args) do |rows|
             records = rows.collect do |row|
-              instantiate(transpose_hbase_columns_to_record_attributes(row))
+              instantiate(*transpose_hbase_row_to_record_attributes_and_raw_data(row))
             end    
             yield records
           end
@@ -51,7 +55,11 @@ module MassiveRecord
         # Entry point for method delegation like find, first, all etc.
         #
         def finder_scope
-          default_scoping || unscoped
+          if default_scoping
+            default_scoping.dup
+          else
+            unscoped
+          end
         end
 
 
@@ -65,7 +73,7 @@ module MassiveRecord
                                     when Scope, nil
                                       scope
                                     when Hash
-                                      Scope.new(self, :find_options => scope)
+                                      Scope.new(self).apply_finder_options(scope)
                                     else
                                       raise "Don't know how to set scope with #{scope.class}."
                                     end
@@ -83,107 +91,152 @@ module MassiveRecord
 
 
         #
-        # This do_find method is not very nice it's logic should be re-factored at some point.
+        # Method actually doing the find operation. It handles first, last (not supported though), all
+        # and find records by id(s). It simply delegates to more spesific methods.
         #
         def do_find(*args) # :nodoc:
           options = args.extract_options!.to_options
-          raise ArgumentError.new("At least one argument required!") if args.empty?
-          raise RecordNotFound.new("Can't find a #{model_name.human} without an ID.") if args.first.nil?
+
           raise ArgumentError.new("Sorry, conditions are not supported!") if options.has_key? :conditions
 
-          skip_expected_result_check = options.delete(:skip_expected_result_check)
+          case args.first
+          when :first, :last
+            send(args.first, options)
+          when :all
+            find_all(options)
+          else
+            find_by_ids(*args, options)
+          end
+        end
 
-          args << options
 
-          type = args.shift if args.first.is_a? Symbol
-          find_many = type == :all
-          expected_result_size = nil
-          what_to_find = []
-          result_from_table = []
-          
-          find_many, expected_result_size, what_to_find, result_from_table = query_hbase(type, args, find_many)
-
-          # Filter out unexpected IDs (unless type is set (all/first), in that case
-          # we have no expectations on the returned rows' ids)
-          unless type || result_from_table.blank?
-            if find_many
-              result_from_table.select! { |result| what_to_find.include? result.try(:id) }
-            else 
-              if result_from_table.id != what_to_find
-                result_from_table = nil
-              end
-            end
+
+
+
+        private
+
+
+
+        def find_by_ids(*ids, options) # :nodoc:
+          raise ArgumentError.new("At least one argument required!") if ids.empty?
+
+          find_many = ids.first.is_a? Array
+          ids = ids.flatten.compact.uniq
+
+          case ids.length
+          when 0
+            raise RecordNotFound.new("Can't find a #{model_name.human} without an ID.")
+          when 1
+            record = find_one(ids.first, options)
+            find_many ? [record] : record
+          else
+            find_some(ids, options)
           end
+        end
 
-          raise RecordNotFound.new("Could not find #{model_name} with id=#{what_to_find}") if result_from_table.blank? && type.nil?
-          
-          if find_many && !skip_expected_result_check && expected_result_size && expected_result_size != result_from_table.length
-            raise RecordNotFound.new("Expected to find #{expected_result_size} records, but found only #{result_from_table.length}")
+        def find_one(id, options) # :nodoc:
+          query_hbase(id, options).first.tap do |record|
+            raise RecordNotFound.new("Could not find #{model_name} with id=#{id}") if record.nil? || record.id != id
           end
-          
-          records = [result_from_table].compact.flatten.collect do |row|
-            instantiate(transpose_hbase_columns_to_record_attributes(row))
+        end
+
+        def find_some(ids, options) # :nodoc:
+          expected_result_size = ids.length 
+
+          query_hbase(ids, options).tap do |records|
+            records.select! { |record| ids.include?(record.id) }
+
+            if !options[:skip_expected_result_check] && records.length != expected_result_size
+              raise RecordNotFound.new("Expected to find #{expected_result_size} records, but found only #{records.length}")
+            end
           end
+        end
 
-          find_many ? records : records.first
+        def find_all(options) # :nodoc:
+          select_known_column_families_if_no_selections_are_added(options)
+          query_hbase { table.all(options) }
         end
 
 
+        def select_known_column_families_if_no_selections_are_added(options)
+          unless options.has_key? :select
+            default_selection = known_column_family_names
+            options[:select] = default_selection if default_selection.any?
+          end
+        end
 
 
-        private
 
-        def query_hbase(type, args, find_many) # :nodoc:
-          result_from_table = if type
-                                hbase_query_all_first(type, args)
-                              else
-                                options = args.extract_options!
-                                what_to_find = args.first
-                                expected_result_size = 1
+        #
+        # Queries hbase. Either looks for what to find with given options
+        # or yields the block and uses that as result when instantiate records from rows
+        #
+        def query_hbase(what_to_find = nil, options = nil) # :nodoc:
+          result =  if block_given?
+                      yield
+                    else
+                      select_known_column_families_if_no_selections_are_added(options)
+                      table.find(what_to_find, options)
+                    end
+
+          ensure_id_is_utf8_encoded(Array(result).compact).collect do |row|
+            instantiate_row_from_hbase(row)
+          end
+        rescue => e
+          if e.is_a?(Apache::Hadoop::Hbase::Thrift::IOError) && e.message =~ /NoSuchColumnFamilyException/
+            raise ColumnFamiliesMissingError.new(self, Persistence::Operations::TableOperationHelpers.calculate_missing_family_names(self))
+          else
+            raise e
+          end
+        end
 
-                                if args.first.kind_of?(Array)
-                                  find_many = true
-                                elsif args.length > 1
-                                  find_many = true
-                                  what_to_find = args
-                                end
+        def instantiate_row_from_hbase(row)
+          instantiate(*transpose_hbase_row_to_record_attributes_and_raw_data(row)) # :nodoc:
+        end
 
-                                expected_result_size = what_to_find.length if what_to_find.is_a? Array
-                                hbase_query_find(what_to_find, options)
-                              end
 
-          [find_many, expected_result_size, what_to_find, result_from_table]
-        end
 
-        def hbase_query_all_first(type, args)
-          table.send(type, *args) # first() / all()
+        def instantiate(record, raw_data) # :nodoc:
+          model = if record.has_key?(inheritance_attribute)
+                    if klass = record[inheritance_attribute] and klass.present?
+                      klass.constantize.allocate
+                    else
+                      base_class.allocate
+                    end
+                  else
+                    allocate
+                  end
+
+          model.init_with('attributes' => record, 'raw_data' => raw_data)
         end
 
-        def hbase_query_find(what_to_find, options)
-          table.find(what_to_find, options)
+
+
+        def ensure_id_is_utf8_encoded(result_from_table) # :nodoc
+          return nil if result_from_table.nil?
+
+          if result_from_table.respond_to? :id
+            result_from_table.id.force_encoding(Encoding::UTF_8) if result_from_table.id.respond_to? :force_encoding
+          elsif result_from_table.respond_to? :each
+            result_from_table.collect! { |result| ensure_id_is_utf8_encoded(result) }
+          end
+
+          result_from_table
         end
 
-        def transpose_hbase_columns_to_record_attributes(row) #: nodoc:
+        def transpose_hbase_row_to_record_attributes_and_raw_data(row) # :nodoc:
           attributes = {:id => row.id}
+          raw_data = row.values_raw_data_hash
           
           autoload_column_families_and_fields_with(row.columns.keys)
 
           # Parse the schema to populate the instance attributes
           attributes_schema.each do |key, field|
-            cell = row.columns[field.unique_name]
-            attributes[field.name] = cell.nil? ? nil : field.decode(cell.value)
+            data = raw_data.has_key?(field.column_family.name) ? raw_data[field.column_family.name][field.column] : nil
+            attributes[field.name] = data.nil? ? nil : field.decode(data.value)
           end
-          attributes
-        end
-
-        def instantiate(record) # :nodoc:
-          model = if klass = record[inheritance_attribute] and klass.present?
-                    klass.constantize.allocate
-                  else
-                    allocate
-                  end
 
-          model.init_with('attributes' => record)
+          [attributes, raw_data]
         end
       end
     end

data/lib/massive_record/orm/finders/rescue_missing_table_on_find.rb

@@ -35,7 +35,7 @@ module MassiveRecord
             raise error
           else
             logger.try :info, "*** TABLE MISSING: Table '#{table_name}' seems to be missing. Will create it, then retry call to find()."
-            hbase_create_table!
+            Persistence::Operations::TableOperationHelpers.hbase_create_table!(self)
             yield
           end
         end

data/lib/massive_record/orm/finders/scope.rb

@@ -16,31 +16,34 @@ module MassiveRecord
       #
       class Scope
         MULTI_VALUE_METHODS = %w(select)
-        SINGLE_VALUE_METHODS = %w(limit)
+        SINGLE_VALUE_METHODS = %w(limit starts_with offset)
         
         attr_accessor *MULTI_VALUE_METHODS.collect { |m| m + "_values" }
         attr_accessor *SINGLE_VALUE_METHODS.collect { |m| m + "_value" }
-        attr_accessor :loaded, :klass
+        attr_accessor :loaded, :klass, :extra_finder_options
         alias :loaded? :loaded
 
 
-        delegate :to_xml, :to_yaml, :length, :collect, :map, :each, :all?, :include?, :to => :to_a
+        delegate :to_xml, :to_yaml, :length, :size, :collect, :map, :each, :all?, :include?, :to => :to_a
         
 
-        def initialize(klass, options = {})
+        def initialize(klass)
           @klass = klass
           @extra_finder_options = {}
 
           reset
           reset_single_values_options
           reset_multi_values_options
+        end
 
-          apply_finder_options(options[:find_options]) if options.has_key? :find_options
+        def initialize_copy(old)
+          reset
         end
         
-        
+
         def reset
           @loaded = false
+          @records = []
         end
 
 
@@ -50,8 +53,7 @@ module MassiveRecord
         #
 
         def select(*select)
-          self.select_values |= select.flatten.compact.collect(&:to_s)
-          self
+          cloned_version_with { self.select_values |= select.flatten.compact.collect(&:to_s) }
         end
 
 
@@ -60,8 +62,15 @@ module MassiveRecord
         #
 
         def limit(limit)
-          self.limit_value = limit
-          self
+          cloned_version_with { self.limit_value = limit }
+        end
+
+        def starts_with(starts_with)
+          cloned_version_with { self.starts_with_value = starts_with }
+        end
+
+        def offset(offset)
+          cloned_version_with { self.offset_value = offset }
         end
 
 
@@ -83,24 +92,32 @@ module MassiveRecord
 
         def find(*args)
           options = args.extract_options!.to_options
-          apply_finder_options(options)
-          args << options.merge(find_options)
-
-          klass.do_find(*args)
+          
+          if options.any?
+            apply_finder_options(options).find(*args)
+          else
+            klass.do_find(*args << find_options)
+          end
         end
 
         def all(options = {})
-          apply_finder_options(options)
-          to_a
+          if options.empty?
+            to_a
+          else
+            apply_finder_options(options).to_a
+          end
         end
 
         def first(options = {})
-          apply_finder_options(options)
-          limit(1).to_a.first
+          if loaded? && options.empty?
+            @records.first
+          else
+            apply_finder_options(options).limit(1).to_a.first
+          end
         end
 
         def last(*args)
-          raise "Sorry, not implemented!"
+          raise "Sorry, but query last requires all records to be fetched. If you really want to do this, do an scope.all.last instead."
         end
 
 
@@ -112,11 +129,32 @@ module MassiveRecord
           @records
         end
 
+        
+        #
+        # Takes a hash of finder options, applies them to
+        # a new scope and returns a that scope.
+        #
+        def apply_finder_options(options)
+          scope = clone
+          return scope if options.empty?
+
+          options.each do |scope_method, arguments|
+            if respond_to? scope_method
+              scope = scope.send(scope_method, arguments)
+            else
+              scope.extra_finder_options[scope_method] = arguments              
+            end
+          end
 
+          scope
+        end
 
 
         private
 
+        def cloned_version_with(&block)
+          clone.tap { |scope| scope.instance_eval(&block) }
+        end
 
         def load_records
           @records = klass.do_find(:all, find_options)
@@ -124,6 +162,7 @@ module MassiveRecord
           @records
         end
 
+        # Returns find options which adapter's find understands.
         def find_options
           options = {}
 
@@ -138,21 +177,6 @@ module MassiveRecord
           options.merge(@extra_finder_options)
         end
 
-
-
-        def apply_finder_options(options)
-          options.each do |scope_method, arguments|
-            if respond_to? scope_method
-              send(scope_method, arguments)
-            else
-              @extra_finder_options[scope_method] = arguments              
-            end
-          end
-        end
-
-
-
-
         def reset_single_values_options
           SINGLE_VALUE_METHODS.each { |m| instance_variable_set("@#{m}_value", nil) } 
         end