massive_record 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. data/.autotest +15 -0
  2. data/.gitignore +6 -0
  3. data/.rspec +2 -0
  4. data/Gemfile +4 -0
  5. data/Gemfile.lock +38 -0
  6. data/Manifest +24 -0
  7. data/README.md +225 -0
  8. data/Rakefile +16 -0
  9. data/TODO.md +8 -0
  10. data/autotest/discover.rb +1 -0
  11. data/lib/massive_record.rb +18 -0
  12. data/lib/massive_record/exceptions.rb +11 -0
  13. data/lib/massive_record/orm/attribute_methods.rb +61 -0
  14. data/lib/massive_record/orm/attribute_methods/dirty.rb +80 -0
  15. data/lib/massive_record/orm/attribute_methods/read.rb +23 -0
  16. data/lib/massive_record/orm/attribute_methods/write.rb +24 -0
  17. data/lib/massive_record/orm/base.rb +176 -0
  18. data/lib/massive_record/orm/callbacks.rb +52 -0
  19. data/lib/massive_record/orm/column.rb +18 -0
  20. data/lib/massive_record/orm/config.rb +47 -0
  21. data/lib/massive_record/orm/errors.rb +47 -0
  22. data/lib/massive_record/orm/finders.rb +125 -0
  23. data/lib/massive_record/orm/id_factory.rb +133 -0
  24. data/lib/massive_record/orm/persistence.rb +199 -0
  25. data/lib/massive_record/orm/schema.rb +4 -0
  26. data/lib/massive_record/orm/schema/column_families.rb +48 -0
  27. data/lib/massive_record/orm/schema/column_family.rb +102 -0
  28. data/lib/massive_record/orm/schema/column_interface.rb +91 -0
  29. data/lib/massive_record/orm/schema/common_interface.rb +48 -0
  30. data/lib/massive_record/orm/schema/field.rb +128 -0
  31. data/lib/massive_record/orm/schema/fields.rb +37 -0
  32. data/lib/massive_record/orm/schema/table_interface.rb +96 -0
  33. data/lib/massive_record/orm/table.rb +9 -0
  34. data/lib/massive_record/orm/validations.rb +52 -0
  35. data/lib/massive_record/spec/support/simple_database_cleaner.rb +52 -0
  36. data/lib/massive_record/thrift/hbase.rb +2307 -0
  37. data/lib/massive_record/thrift/hbase_constants.rb +14 -0
  38. data/lib/massive_record/thrift/hbase_types.rb +225 -0
  39. data/lib/massive_record/version.rb +3 -0
  40. data/lib/massive_record/wrapper/base.rb +28 -0
  41. data/lib/massive_record/wrapper/cell.rb +45 -0
  42. data/lib/massive_record/wrapper/column_families_collection.rb +19 -0
  43. data/lib/massive_record/wrapper/column_family.rb +22 -0
  44. data/lib/massive_record/wrapper/connection.rb +71 -0
  45. data/lib/massive_record/wrapper/row.rb +170 -0
  46. data/lib/massive_record/wrapper/scanner.rb +50 -0
  47. data/lib/massive_record/wrapper/table.rb +148 -0
  48. data/lib/massive_record/wrapper/tables_collection.rb +13 -0
  49. data/massive_record.gemspec +28 -0
  50. data/spec/config.yml.example +4 -0
  51. data/spec/orm/cases/attribute_methods_spec.rb +47 -0
  52. data/spec/orm/cases/auto_generate_id_spec.rb +54 -0
  53. data/spec/orm/cases/base_spec.rb +176 -0
  54. data/spec/orm/cases/callbacks_spec.rb +309 -0
  55. data/spec/orm/cases/column_spec.rb +49 -0
  56. data/spec/orm/cases/config_spec.rb +103 -0
  57. data/spec/orm/cases/dirty_spec.rb +129 -0
  58. data/spec/orm/cases/encoding_spec.rb +49 -0
  59. data/spec/orm/cases/finders_spec.rb +208 -0
  60. data/spec/orm/cases/hbase/connection_spec.rb +13 -0
  61. data/spec/orm/cases/i18n_spec.rb +32 -0
  62. data/spec/orm/cases/id_factory_spec.rb +75 -0
  63. data/spec/orm/cases/persistence_spec.rb +479 -0
  64. data/spec/orm/cases/table_spec.rb +81 -0
  65. data/spec/orm/cases/validation_spec.rb +92 -0
  66. data/spec/orm/models/address.rb +7 -0
  67. data/spec/orm/models/person.rb +15 -0
  68. data/spec/orm/models/test_class.rb +5 -0
  69. data/spec/orm/schema/column_families_spec.rb +186 -0
  70. data/spec/orm/schema/column_family_spec.rb +131 -0
  71. data/spec/orm/schema/column_interface_spec.rb +115 -0
  72. data/spec/orm/schema/field_spec.rb +196 -0
  73. data/spec/orm/schema/fields_spec.rb +126 -0
  74. data/spec/orm/schema/table_interface_spec.rb +171 -0
  75. data/spec/spec_helper.rb +15 -0
  76. data/spec/support/connection_helpers.rb +76 -0
  77. data/spec/support/mock_massive_record_connection.rb +80 -0
  78. data/spec/thrift/cases/encoding_spec.rb +48 -0
  79. data/spec/wrapper/cases/connection_spec.rb +53 -0
  80. data/spec/wrapper/cases/table_spec.rb +231 -0
  81. metadata +228 -0
@@ -0,0 +1,15 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ require 'yaml'
4
+
5
+ Bundler.require :default, :development
6
+
7
+ SPEC_DIR = File.dirname(__FILE__) unless defined? SPEC_DIR
8
+ MR_CONFIG = YAML.load_file(File.join(SPEC_DIR, 'config.yml')) unless defined? MR_CONFIG
9
+
10
+ Rspec.configure do |c|
11
+ #c.fail_fast = true
12
+ end
13
+
14
+ Dir["#{SPEC_DIR}/orm/models/*.rb"].each { |f| require f }
15
+ Dir["#{SPEC_DIR}/support/**/*.rb"].each { |f| require f }
@@ -0,0 +1,76 @@
1
+ require 'massive_record/spec/support/simple_database_cleaner'
2
+
3
+ module SetUpHbaseConnectionBeforeAll
4
+ extend ActiveSupport::Concern
5
+
6
+ included do
7
+ before(:all) do
8
+ unless @connection
9
+ @connection_configuration = {:host => MR_CONFIG['host'], :port => MR_CONFIG['port']}
10
+ MassiveRecord::ORM::Base.connection_configuration = @connection_configuration
11
+ @connection = MassiveRecord::Wrapper::Connection.new(@connection_configuration)
12
+ @connection.open
13
+ end
14
+ end
15
+ end
16
+ end
17
+
18
+ module SetTableNamesToTestTable
19
+ extend ActiveSupport::Concern
20
+
21
+ included do
22
+ include MassiveRecord::Rspec::SimpleDatabaseCleaner
23
+
24
+ after do
25
+ MassiveRecord::ORM::Base.reset_connection!
26
+ MassiveRecord::ORM::Base.descendants.each { |klass| klass.unmemoize_all }
27
+ end
28
+ end
29
+ end
30
+
31
+
32
+ module CreatePersonBeforeEach
33
+ extend ActiveSupport::Concern
34
+
35
+ included do
36
+ include SetUpHbaseConnectionBeforeAll
37
+ include SetTableNamesToTestTable
38
+
39
+ before do
40
+ @table = MassiveRecord::Wrapper::Table.new(@connection, Person.table_name)
41
+ @table.column_families.create(:info)
42
+ @table.save
43
+
44
+ @row = MassiveRecord::Wrapper::Row.new
45
+ @row.id = "ID1"
46
+ @row.values = {:info => {:name => "John Doe", :email => "john@base.com", :age => "20"}}
47
+ @row.table = @table
48
+ @row.save
49
+ end
50
+ end
51
+ end
52
+
53
+ module CreatePeopleBeforeEach
54
+ extend ActiveSupport::Concern
55
+
56
+ included do
57
+ include SetUpHbaseConnectionBeforeAll
58
+ include SetTableNamesToTestTable
59
+
60
+ before do
61
+ @table = MassiveRecord::Wrapper::Table.new(@connection, Person.table_name)
62
+ @table.column_families.create(:info)
63
+ @table.save
64
+
65
+ @table_size = 9
66
+
67
+ @table_size.times.each do |id|
68
+ @row = MassiveRecord::Wrapper::Row.new
69
+ @row.id = id + 1
70
+ @row.values = {:info => {:name => "John Doe", :email => "john@doe.com", :age => "20"}}
71
+ @row.table = @table
72
+ @row.save
73
+ end
74
+ end
75
+ end
76
+ end
@@ -0,0 +1,80 @@
1
+ #
2
+ # Set up mock MassiveRecord connection to speed things up and
3
+ # skip the actual database when it's not needed.
4
+ #
5
+ # ...now, the more I work on this thing, the more hackish it gets.
6
+ # So I guess we really should reconsider what we are doing here, and
7
+ # instead only do tests against a real connection.
8
+ #
9
+ module MockMassiveRecordConnection
10
+ extend ActiveSupport::Concern
11
+
12
+ included do
13
+ before do
14
+ #
15
+ # The following is needed to make all READ from the DB to go through
16
+ #
17
+
18
+ # Setting up expected connection configuration, or else an error will be raised
19
+ MassiveRecord::ORM::Base.connection_configuration = {:host => "foo", :port => 9001}
20
+
21
+ # Setting up a mock connection when asked for new
22
+ mock_connection = mock(MassiveRecord::Wrapper::Connection,
23
+ :open => true,
24
+ :tables => MassiveRecord::ORM::Table.descendants.collect(&:table_name)
25
+ )
26
+ MassiveRecord::Wrapper::Connection.stub(:new).and_return(mock_connection)
27
+
28
+ # Inject find method on tables so that we don't need to go through with
29
+ # the actual call to the database.
30
+ new_table_method = MassiveRecord::Wrapper::Table.method(:new)
31
+ MassiveRecord::Wrapper::Table.stub!(:new) do |*args|
32
+ table = new_table_method.call(*args)
33
+ # Defines a dummy find method which simply returns a hash where id is set to the first
34
+ # argument (Like Person.find(1)).
35
+ def table.find(*args)
36
+ row = MassiveRecord::Wrapper::Row.new
37
+ row.id = args[0]
38
+ row.values = {}
39
+ row
40
+ end
41
+
42
+ # Simply returning all known column families across all tables to make the need
43
+ # for creating new one on create disappear.
44
+ def table.fetch_column_families
45
+ MassiveRecord::ORM::Table.descendants.collect(&:column_families).compact.collect(&:to_a).flatten
46
+ end
47
+
48
+ table
49
+ end
50
+
51
+
52
+
53
+
54
+ #
55
+ # The following is needed to make all WRITE to the DB to go through
56
+ #
57
+ new_row_method = MassiveRecord::Wrapper::Row.method(:new)
58
+ MassiveRecord::Wrapper::Row.stub!(:new) do |*args|
59
+ row = new_row_method.call(*args)
60
+
61
+ def row.save
62
+ true
63
+ end
64
+
65
+ def row.destroy
66
+ true
67
+ end
68
+ row
69
+ end
70
+ end
71
+
72
+
73
+
74
+
75
+ after do
76
+ MassiveRecord::ORM::Base.descendants.each { |klass| klass.unmemoize_all }
77
+ MassiveRecord::ORM::Base.reset_connection!
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,48 @@
1
+ # encoding: utf-8
2
+ require 'spec_helper'
3
+
4
+ describe "encoding" do
5
+ before :all do
6
+ @table_name = "encoding_test" + ActiveSupport::SecureRandom.hex(3)
7
+ end
8
+
9
+ before do
10
+ transport = Thrift::BufferedTransport.new(Thrift::Socket.new(MR_CONFIG['host'], 9090))
11
+ protocol = Thrift::BinaryProtocol.new(transport)
12
+ @client = Apache::Hadoop::Hbase::Thrift::Hbase::Client.new(protocol)
13
+
14
+ transport.open()
15
+
16
+ @column_family = "info:"
17
+ end
18
+
19
+ it "should create a new table" do
20
+ column = Apache::Hadoop::Hbase::Thrift::ColumnDescriptor.new{|c| c.name = @column_family}
21
+ @client.createTable(@table_name, [column]).should be_nil
22
+ end
23
+
24
+ it "should save standard caracteres" do
25
+ m = Apache::Hadoop::Hbase::Thrift::Mutation.new
26
+ m.column = "info:first_name"
27
+ m.value = "Vincent"
28
+
29
+ m.value.encoding.should == Encoding::UTF_8
30
+ @client.mutateRow(@table_name, "ID1", [m]).should be_nil
31
+ end
32
+
33
+ it "should save UTF8 caracteres" do
34
+ pending "UTF8 enconding need to be fixed!"
35
+
36
+ m = Apache::Hadoop::Hbase::Thrift::Mutation.new
37
+ m.column = "info:first_name"
38
+ m.value = "Thorbjørn"
39
+
40
+ m.value.encoding.should == Encoding::UTF_8
41
+ @client.mutateRow(@table_name, "ID1", [m]).should be_nil
42
+ end
43
+
44
+ it "should destroy the table" do
45
+ @client.disableTable(@table_name).should be_nil
46
+ @client.deleteTable(@table_name).should be_nil
47
+ end
48
+ end
@@ -0,0 +1,53 @@
1
+ require 'spec_helper'
2
+
3
+ describe MassiveRecord::Wrapper::Connection do
4
+
5
+ before do
6
+ @connection = MassiveRecord::Wrapper::Connection.new(:host => MR_CONFIG['host'], :port => MR_CONFIG['port'])
7
+ end
8
+
9
+ it "should have a host and port attributes" do
10
+ connections = [@connection, MassiveRecord::Wrapper::Connection.new(:host => "somewhere")]
11
+
12
+ connections.each do |conn|
13
+ conn.host.to_s.should_not be_empty
14
+ conn.port.to_s.should_not be_empty
15
+ end
16
+ end
17
+
18
+ it "should not be active" do
19
+ pending "should we implement this, Vincent? :-)"
20
+ @connection.active?.should be_false
21
+ end
22
+
23
+ it "should not be able to open a new connection with a wrong configuration and Raise an error" do
24
+ @connection.port = 1234
25
+ lambda{@connection.open}.should raise_error(MassiveRecord::ConnectionException)
26
+ end
27
+
28
+ it "should be able to open a new connection with a good configuration" do
29
+ @connection.open.should be_true
30
+ end
31
+
32
+ it "should not be active if it is closed" do
33
+ @connection.open
34
+ @connection.active?.should be_true
35
+ @connection.close.should be_true
36
+ @connection.active?.should be_false
37
+ end
38
+
39
+ it "should have a collection of tables" do
40
+ @connection.open
41
+ @connection.tables.should be_a_kind_of(MassiveRecord::Wrapper::TablesCollection)
42
+ end
43
+
44
+ it "should reconnect on IOError" do
45
+ @connection.open
46
+ @connection.transport.open?.should be_true
47
+ @connection.getTableNames().should be_a_kind_of(Array)
48
+
49
+ @connection.close
50
+ @connection.transport.open?.should be_false
51
+ @connection.getTableNames().should be_a_kind_of(Array)
52
+ end
53
+ end
@@ -0,0 +1,231 @@
1
+ require 'spec_helper'
2
+
3
+ describe MassiveRecord::Wrapper::Table do
4
+
5
+ describe "with a new connection" do
6
+
7
+ before do
8
+ @connection = MassiveRecord::Wrapper::Connection.new(:host => MR_CONFIG['host'], :port => MR_CONFIG['port'])
9
+ @connection.open
10
+
11
+ @table = MassiveRecord::Wrapper::Table.new(@connection, MR_CONFIG['table'])
12
+ end
13
+
14
+ describe "and a new initialized table" do
15
+
16
+ it "should not exists is the database" do
17
+ @connection.tables.should_not include(MR_CONFIG['table'])
18
+ end
19
+
20
+ it "should not have any column families" do
21
+ @table.column_families.should be_empty
22
+ end
23
+
24
+ end
25
+
26
+ describe "and a new initialized table with column families" do
27
+
28
+ before do
29
+ @table.column_families.create(MassiveRecord::Wrapper::ColumnFamily.new(:info, :max_versions => 3))
30
+ @table.column_families.create(:misc)
31
+ end
32
+
33
+ it "should contains two column families" do
34
+ @table.column_families.size.should == 2
35
+ end
36
+
37
+ it "should create a test table" do
38
+ @table.save.should be_true
39
+ end
40
+
41
+ it "should load a table" do
42
+ @connection.load_table(MR_CONFIG['table']).class.should eql(MassiveRecord::Wrapper::Table)
43
+ @connection.tables.load(MR_CONFIG['table']).class.should eql(MassiveRecord::Wrapper::Table)
44
+ end
45
+
46
+ it "should fetch column families from the database" do
47
+ @table.fetch_column_families.size.should == 2
48
+ end
49
+
50
+ it "should add a row" do
51
+ row = MassiveRecord::Wrapper::Row.new
52
+ row.id = "ID1"
53
+ row.values = {
54
+ :info => { :first_name => "John", :last_name => "Doe", :email => "john@base.com" },
55
+ :misc => {
56
+ :like => ["Eating", "Sleeping", "Coding"],
57
+ :dislike => {
58
+ "Washing" => "Boring 6/10",
59
+ "Ironing" => "Boring 8/10"
60
+ },
61
+ :empty => {},
62
+ :value_to_increment => "1"
63
+ }
64
+ }
65
+ row.table = @table
66
+ row.save
67
+ end
68
+
69
+ it "should contains one row" do
70
+ @table.all.size.should == 1
71
+ end
72
+
73
+ it "should load the first row" do
74
+ @table.first.should be_a_kind_of(MassiveRecord::Wrapper::Row)
75
+ end
76
+
77
+ it "should list 5 column names" do
78
+ @table.column_names.size.should == 7
79
+ end
80
+
81
+ it "should only load one column family" do
82
+ @table.first(:select => ["info"]).column_families.should == ["info"]
83
+ @table.all(:limit => 1, :select => ["info"]).first.column_families.should == ["info"]
84
+ @table.find("ID1", :select => ["info"]).column_families.should == ["info"]
85
+ end
86
+
87
+ it "should update row values" do
88
+ row = @table.first
89
+ row.values["info:first_name"].should eql("John")
90
+
91
+ row.update_columns({ :info => { :first_name => "Bob" } })
92
+ row.values["info:first_name"].should eql("Bob")
93
+
94
+ row.update_column(:info, :email, "bob@base.com")
95
+ row.values["info:email"].should eql("bob@base.com")
96
+ end
97
+
98
+ it "should save row changes" do
99
+ row = @table.first
100
+ row.update_columns({ :info => { :first_name => "Bob" } })
101
+ row.save.should be_true
102
+ end
103
+
104
+ it "should merge data" do
105
+ row = @table.first
106
+ row.update_columns({ :misc => { :super_power => "Eating"} })
107
+ row.columns.collect{|k, v| k if k.include?("misc:")}.delete_if{|v| v.nil?}.sort.should(
108
+ eql(["misc:value_to_increment", "misc:like", "misc:empty", "misc:dislike", "misc:super_power"].sort)
109
+ )
110
+ end
111
+
112
+ it "should merge array data" do
113
+ row = @table.first
114
+ row.merge_columns({ :misc => { :like => ["Playing"] } })
115
+ row.columns["misc:like"].deserialize_value.should =~ ["Eating", "Sleeping", "Coding", "Playing"]
116
+ end
117
+
118
+ it "should merge hash data" do
119
+ row = @table.first
120
+ row.merge_columns({ :misc => { :dislike => { "Ironing" => "Boring 10/10", "Running" => "Boring 5/10" } } })
121
+ row.columns["misc:dislike"].deserialize_value["Ironing"].should eql("Boring 10/10") # Check updated value
122
+ row.columns["misc:dislike"].deserialize_value.keys.should =~ ["Washing", "Ironing", "Running"] # Check new value
123
+ end
124
+
125
+ it "should deserialize Array / Hash values from YAML automatically" do
126
+ row = @table.first
127
+ row.values["misc:like"].class.should eql(Array)
128
+ row.values["misc:dislike"].class.should eql(Hash)
129
+ row.values["misc:empty"].class.should eql(Hash)
130
+ end
131
+
132
+ it "should display the previous value (versioning) of the column 'info:first_name'" do
133
+ pending "should we implement this, Vincent? :-)"
134
+
135
+ row = @table.first
136
+ row.values["info:first_name"].should eql("Bob")
137
+
138
+ prev_row = row.prev
139
+ prev_row.values["info:first_name"].should eql("John")
140
+ end
141
+
142
+ it "should be able to perform partial updates" do
143
+ row = @table.first(:select => ["misc"])
144
+ row.update_columns({ :misc => { :genre => "M" } })
145
+ row.save
146
+
147
+ row = @table.first
148
+ row.values["info:first_name"].should == "Bob"
149
+ row.values["misc:genre"].should == "M"
150
+ end
151
+
152
+ it "should be able to do atomic increment call on values" do
153
+ row = @table.first
154
+ row.values["misc:value_to_increment"].should == "1"
155
+
156
+ result = row.atomic_increment("misc:value_to_increment")
157
+ result.should == "2"
158
+ end
159
+
160
+ it "should be able to pass inn what to incremet by" do
161
+ row = @table.first
162
+ row.values["misc:value_to_increment"].should == "2"
163
+ row.atomic_increment("misc:value_to_increment", 2)
164
+
165
+ row = @table.first
166
+ row.values["misc:value_to_increment"].should == "4"
167
+ end
168
+
169
+ it "should delete a row" do
170
+ @table.first.destroy.should be_true
171
+ end
172
+
173
+ it "should not contains any row" do
174
+ @table.first.should be_nil
175
+ end
176
+
177
+ it "should create 5 rows" do
178
+ 1.upto(5).each do |i|
179
+ row = MassiveRecord::Wrapper::Row.new
180
+ row.id = "ID#{i}"
181
+ row.values = { :info => { :first_name => "John #{i}", :last_name => "Doe #{i}" } }
182
+ row.table = @table
183
+ row.save
184
+ end
185
+
186
+ @table.all.size.should == 5
187
+ end
188
+
189
+ it "should find rows" do
190
+ ids_list = [["ID1"], ["ID1", "ID2", "ID3"]]
191
+ ids_list.each do |ids|
192
+ @table.find(ids).each do |row|
193
+ ids.include?(row.id).should be_true
194
+ end
195
+ end
196
+ end
197
+
198
+ it "should collect 5 IDs" do
199
+ @table.all.collect(&:id).should eql(1.upto(5).collect{|i| "ID#{i}"})
200
+ end
201
+
202
+ it "should iterate through a collection of rows" do
203
+ @table.all.each do |row|
204
+ row.id.should_not be_nil
205
+ end
206
+ end
207
+
208
+ it "should iterate through a collection of rows using a batch process" do
209
+ group_number = 0
210
+ @table.find_in_batches(:batch_size => 2, :start => "ID2", :select => ["info"]) do |group|
211
+ group_number += 1
212
+ group.each do |row|
213
+ row.id.should_not be_nil
214
+ end
215
+ end
216
+ group_number.should == 2
217
+ end
218
+
219
+ it "should exists in the database" do
220
+ @table.exists?.should be_true
221
+ end
222
+
223
+ it "should destroy the test table" do
224
+ @table.destroy.should be_true
225
+ end
226
+
227
+ end
228
+
229
+ end
230
+
231
+ end