massive_record 0.1.0

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+require 'rubygems'
+require 'bundler'
+require 'yaml'
+
+Bundler.require :default, :development
+
+SPEC_DIR = File.dirname(__FILE__) unless defined? SPEC_DIR
+MR_CONFIG = YAML.load_file(File.join(SPEC_DIR, 'config.yml')) unless defined? MR_CONFIG
+
+Rspec.configure do |c|
+  #c.fail_fast = true 
+end
+
+Dir["#{SPEC_DIR}/orm/models/*.rb"].each { |f| require f }
+Dir["#{SPEC_DIR}/support/**/*.rb"].each { |f| require f }

data/spec/support/connection_helpers.rb

@@ -0,0 +1,76 @@
+require 'massive_record/spec/support/simple_database_cleaner'
+
+module SetUpHbaseConnectionBeforeAll
+  extend ActiveSupport::Concern
+
+  included do
+    before(:all) do
+      unless @connection
+        @connection_configuration = {:host => MR_CONFIG['host'], :port => MR_CONFIG['port']}
+        MassiveRecord::ORM::Base.connection_configuration = @connection_configuration
+        @connection = MassiveRecord::Wrapper::Connection.new(@connection_configuration)
+        @connection.open
+      end
+    end
+  end
+end
+
+module SetTableNamesToTestTable
+  extend ActiveSupport::Concern
+
+  included do
+    include MassiveRecord::Rspec::SimpleDatabaseCleaner
+
+    after do
+      MassiveRecord::ORM::Base.reset_connection!
+      MassiveRecord::ORM::Base.descendants.each { |klass| klass.unmemoize_all }
+    end
+  end
+end
+
+
+module CreatePersonBeforeEach
+  extend ActiveSupport::Concern
+
+  included do
+    include SetUpHbaseConnectionBeforeAll
+    include SetTableNamesToTestTable
+
+    before do
+      @table = MassiveRecord::Wrapper::Table.new(@connection, Person.table_name)
+      @table.column_families.create(:info)
+      @table.save
+      
+      @row = MassiveRecord::Wrapper::Row.new
+      @row.id = "ID1"
+      @row.values = {:info => {:name => "John Doe", :email => "john@base.com", :age => "20"}}
+      @row.table = @table
+      @row.save
+    end
+  end
+end
+
+module CreatePeopleBeforeEach
+  extend ActiveSupport::Concern
+
+  included do
+    include SetUpHbaseConnectionBeforeAll
+    include SetTableNamesToTestTable
+
+    before do
+      @table = MassiveRecord::Wrapper::Table.new(@connection, Person.table_name)
+      @table.column_families.create(:info)
+      @table.save
+      
+      @table_size = 9
+      
+      @table_size.times.each do |id|
+        @row = MassiveRecord::Wrapper::Row.new
+        @row.id = id + 1
+        @row.values = {:info => {:name => "John Doe", :email => "john@doe.com", :age => "20"}}
+        @row.table = @table
+        @row.save
+      end
+    end
+  end
+end

data/spec/support/mock_massive_record_connection.rb

@@ -0,0 +1,80 @@
+#
+# Set up mock MassiveRecord connection to speed things up and
+# skip the actual database when it's not needed.
+#
+# ...now, the more I work on this thing, the more hackish it gets.
+# So I guess we really should reconsider what we are doing here, and
+# instead only do tests against a real connection. 
+#
+module MockMassiveRecordConnection
+  extend ActiveSupport::Concern
+
+  included do
+    before do
+      #
+      # The following is needed to make all READ from the DB to go through
+      #
+
+      # Setting up expected connection configuration, or else an error will be raised
+      MassiveRecord::ORM::Base.connection_configuration = {:host => "foo", :port => 9001}
+
+      # Setting up a mock connection when asked for new
+      mock_connection = mock(MassiveRecord::Wrapper::Connection,
+        :open => true,
+        :tables => MassiveRecord::ORM::Table.descendants.collect(&:table_name)
+      )
+      MassiveRecord::Wrapper::Connection.stub(:new).and_return(mock_connection)
+
+      # Inject find method on tables so that we don't need to go through with
+      # the actual call to the database.
+      new_table_method = MassiveRecord::Wrapper::Table.method(:new)
+      MassiveRecord::Wrapper::Table.stub!(:new) do |*args|
+        table = new_table_method.call(*args)
+        # Defines a dummy find method which simply returns a hash where id is set to the first
+        # argument (Like Person.find(1)).
+        def table.find(*args)
+          row = MassiveRecord::Wrapper::Row.new
+          row.id = args[0]
+          row.values = {}
+          row
+        end
+        
+        # Simply returning all known column families across all tables to make the need
+        # for creating new one on create disappear.
+        def table.fetch_column_families
+          MassiveRecord::ORM::Table.descendants.collect(&:column_families).compact.collect(&:to_a).flatten
+        end
+        
+        table
+      end
+
+
+
+
+      #
+      # The following is needed to make all WRITE to the DB to go through
+      #
+      new_row_method = MassiveRecord::Wrapper::Row.method(:new)
+      MassiveRecord::Wrapper::Row.stub!(:new) do |*args|
+        row = new_row_method.call(*args)
+
+        def row.save
+          true
+        end
+
+        def row.destroy
+          true
+        end
+        row
+      end
+    end
+
+
+
+
+    after do
+      MassiveRecord::ORM::Base.descendants.each { |klass| klass.unmemoize_all }
+      MassiveRecord::ORM::Base.reset_connection!
+    end
+  end
+end

data/spec/thrift/cases/encoding_spec.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe "encoding" do
+  before :all do
+    @table_name = "encoding_test" + ActiveSupport::SecureRandom.hex(3)
+  end
+
+  before do
+    transport = Thrift::BufferedTransport.new(Thrift::Socket.new(MR_CONFIG['host'], 9090))
+    protocol  = Thrift::BinaryProtocol.new(transport)
+    @client   = Apache::Hadoop::Hbase::Thrift::Hbase::Client.new(protocol)
+    
+    transport.open()
+    
+    @column_family = "info:"
+  end
+  
+  it "should create a new table" do
+    column = Apache::Hadoop::Hbase::Thrift::ColumnDescriptor.new{|c| c.name = @column_family}
+    @client.createTable(@table_name, [column]).should be_nil
+  end
+  
+  it "should save standard caracteres" do
+    m        = Apache::Hadoop::Hbase::Thrift::Mutation.new
+    m.column = "info:first_name"
+    m.value  = "Vincent"
+    
+    m.value.encoding.should == Encoding::UTF_8
+    @client.mutateRow(@table_name, "ID1", [m]).should be_nil    
+  end
+  
+  it "should save UTF8 caracteres" do
+    pending "UTF8 enconding need to be fixed!"
+    
+    m        = Apache::Hadoop::Hbase::Thrift::Mutation.new
+    m.column = "info:first_name"
+    m.value  = "Thorbjørn"
+    
+    m.value.encoding.should == Encoding::UTF_8
+    @client.mutateRow(@table_name, "ID1", [m]).should be_nil
+  end
+  
+  it "should destroy the table" do
+    @client.disableTable(@table_name).should be_nil
+    @client.deleteTable(@table_name).should be_nil
+  end
+end

data/spec/wrapper/cases/connection_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+describe MassiveRecord::Wrapper::Connection do
+  
+  before do
+    @connection = MassiveRecord::Wrapper::Connection.new(:host => MR_CONFIG['host'], :port => MR_CONFIG['port'])
+  end
+  
+  it "should have a host and port attributes" do
+    connections = [@connection, MassiveRecord::Wrapper::Connection.new(:host => "somewhere")]
+    
+    connections.each do |conn|
+      conn.host.to_s.should_not be_empty
+      conn.port.to_s.should_not be_empty
+    end
+  end 
+  
+  it "should not be active" do
+    pending "should we implement this, Vincent? :-)"
+    @connection.active?.should be_false
+  end
+   
+  it "should not be able to open a new connection with a wrong configuration and Raise an error" do
+    @connection.port = 1234
+    lambda{@connection.open}.should raise_error(MassiveRecord::ConnectionException)
+  end
+  
+  it "should be able to open a new connection with a good configuration" do
+    @connection.open.should be_true
+  end
+  
+  it "should not be active if it is closed" do 
+    @connection.open
+    @connection.active?.should be_true
+    @connection.close.should be_true
+    @connection.active?.should be_false
+  end
+  
+  it "should have a collection of tables" do
+    @connection.open
+    @connection.tables.should be_a_kind_of(MassiveRecord::Wrapper::TablesCollection)
+  end
+  
+  it "should reconnect on IOError" do
+    @connection.open
+    @connection.transport.open?.should be_true
+    @connection.getTableNames().should be_a_kind_of(Array)
+    
+    @connection.close
+    @connection.transport.open?.should be_false
+    @connection.getTableNames().should be_a_kind_of(Array)
+  end
+end

data/spec/wrapper/cases/table_spec.rb

@@ -0,0 +1,231 @@
+require 'spec_helper'
+
+describe MassiveRecord::Wrapper::Table do
+  
+  describe "with a new connection" do
+
+    before do
+      @connection = MassiveRecord::Wrapper::Connection.new(:host => MR_CONFIG['host'], :port => MR_CONFIG['port'])
+      @connection.open
+      
+      @table = MassiveRecord::Wrapper::Table.new(@connection, MR_CONFIG['table'])
+    end
+  
+    describe "and a new initialized table" do
+    
+      it "should not exists is the database" do
+        @connection.tables.should_not include(MR_CONFIG['table'])
+      end
+    
+      it "should not have any column families" do
+        @table.column_families.should be_empty
+      end
+
+    end
+    
+    describe "and a new initialized table with column families" do 
+    
+      before do
+        @table.column_families.create(MassiveRecord::Wrapper::ColumnFamily.new(:info, :max_versions => 3))
+        @table.column_families.create(:misc)
+      end
+  
+      it "should contains two column families" do
+        @table.column_families.size.should == 2
+      end
+  
+      it "should create a test table" do
+        @table.save.should be_true
+      end
+    
+      it "should load a table" do
+        @connection.load_table(MR_CONFIG['table']).class.should eql(MassiveRecord::Wrapper::Table)
+        @connection.tables.load(MR_CONFIG['table']).class.should eql(MassiveRecord::Wrapper::Table)
+      end
+      
+      it "should fetch column families from the database" do
+        @table.fetch_column_families.size.should == 2
+      end
+    
+      it "should add a row" do
+        row = MassiveRecord::Wrapper::Row.new
+        row.id = "ID1"
+        row.values = { 
+          :info => { :first_name => "John", :last_name => "Doe", :email => "john@base.com" },
+          :misc => { 
+            :like => ["Eating", "Sleeping", "Coding"], 
+            :dislike => {
+              "Washing" => "Boring 6/10",
+              "Ironing" => "Boring 8/10"
+            },
+            :empty => {},
+            :value_to_increment => "1"
+          }
+        }
+        row.table = @table
+        row.save
+      end
+      
+      it "should contains one row" do
+        @table.all.size.should == 1
+      end
+      
+      it "should load the first row" do
+        @table.first.should be_a_kind_of(MassiveRecord::Wrapper::Row)
+      end
+        
+      it "should list 5 column names" do
+        @table.column_names.size.should == 7
+      end
+      
+      it "should only load one column family" do
+        @table.first(:select => ["info"]).column_families.should == ["info"]
+        @table.all(:limit => 1, :select => ["info"]).first.column_families.should == ["info"]
+        @table.find("ID1", :select => ["info"]).column_families.should == ["info"]
+      end
+      
+      it "should update row values" do
+        row = @table.first
+        row.values["info:first_name"].should eql("John")
+        
+        row.update_columns({ :info =>  { :first_name => "Bob" } })
+        row.values["info:first_name"].should eql("Bob")
+        
+        row.update_column(:info, :email, "bob@base.com")
+        row.values["info:email"].should eql("bob@base.com")
+      end
+      
+      it "should save row changes" do
+        row = @table.first
+        row.update_columns({ :info => { :first_name => "Bob" } })
+        row.save.should be_true
+      end
+      
+      it "should merge data" do
+        row = @table.first
+        row.update_columns({ :misc => { :super_power => "Eating"} })
+        row.columns.collect{|k, v| k if k.include?("misc:")}.delete_if{|v| v.nil?}.sort.should(
+          eql(["misc:value_to_increment", "misc:like", "misc:empty", "misc:dislike", "misc:super_power"].sort)
+        )
+      end
+      
+      it "should merge array data" do
+        row = @table.first
+        row.merge_columns({ :misc => { :like => ["Playing"] } })
+        row.columns["misc:like"].deserialize_value.should =~ ["Eating", "Sleeping", "Coding", "Playing"]
+      end
+      
+      it "should merge hash data" do
+        row = @table.first
+        row.merge_columns({ :misc => { :dislike => { "Ironing" => "Boring 10/10", "Running" => "Boring 5/10" } } })
+        row.columns["misc:dislike"].deserialize_value["Ironing"].should eql("Boring 10/10") # Check updated value
+        row.columns["misc:dislike"].deserialize_value.keys.should =~ ["Washing", "Ironing", "Running"] # Check new value
+      end
+      
+      it "should deserialize Array / Hash values from YAML automatically" do
+        row = @table.first
+        row.values["misc:like"].class.should eql(Array)
+        row.values["misc:dislike"].class.should eql(Hash)
+        row.values["misc:empty"].class.should eql(Hash)
+      end
+      
+      it "should display the previous value (versioning) of the column 'info:first_name'" do
+        pending "should we implement this, Vincent? :-)"
+      
+        row = @table.first
+        row.values["info:first_name"].should eql("Bob")
+        
+        prev_row = row.prev
+        prev_row.values["info:first_name"].should eql("John")
+      end
+      
+      it "should be able to perform partial updates" do
+        row = @table.first(:select => ["misc"])
+        row.update_columns({ :misc => { :genre => "M" } })
+        row.save
+        
+        row = @table.first
+        row.values["info:first_name"].should == "Bob"
+        row.values["misc:genre"].should == "M"
+      end
+
+      it "should be able to do atomic increment call on values" do
+        row = @table.first
+        row.values["misc:value_to_increment"].should == "1"
+
+        result = row.atomic_increment("misc:value_to_increment")
+        result.should == "2"
+      end
+
+      it "should be able to pass inn what to incremet by" do
+        row = @table.first
+        row.values["misc:value_to_increment"].should == "2"
+        row.atomic_increment("misc:value_to_increment", 2)
+
+        row = @table.first
+        row.values["misc:value_to_increment"].should == "4"
+      end
+      
+      it "should delete a row" do
+        @table.first.destroy.should be_true
+      end
+      
+      it "should not contains any row" do
+        @table.first.should be_nil
+      end
+      
+      it "should create 5 rows" do
+        1.upto(5).each do |i|
+          row = MassiveRecord::Wrapper::Row.new
+          row.id = "ID#{i}"
+          row.values = { :info => { :first_name => "John #{i}", :last_name => "Doe #{i}" } }
+          row.table = @table
+          row.save
+        end
+        
+        @table.all.size.should == 5
+      end
+            
+      it "should find rows" do
+        ids_list = [["ID1"], ["ID1", "ID2", "ID3"]]
+        ids_list.each do |ids|
+          @table.find(ids).each do |row|
+            ids.include?(row.id).should be_true
+          end
+        end
+      end
+      
+      it "should collect 5 IDs" do
+        @table.all.collect(&:id).should eql(1.upto(5).collect{|i| "ID#{i}"})
+      end
+      
+      it "should iterate through a collection of rows" do
+        @table.all.each do |row|
+          row.id.should_not be_nil
+        end
+      end
+      
+      it "should iterate through a collection of rows using a batch process" do
+        group_number = 0
+        @table.find_in_batches(:batch_size => 2, :start => "ID2", :select => ["info"]) do |group|
+          group_number += 1
+          group.each do |row|
+            row.id.should_not be_nil
+          end
+        end        
+        group_number.should == 2
+      end
+      
+      it "should exists in the database" do
+        @table.exists?.should be_true
+      end
+  
+      it "should destroy the test table" do
+        @table.destroy.should be_true
+      end
+  
+    end
+  
+  end
+  
+end