massive_record 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. data/.autotest +15 -0
  2. data/.gitignore +6 -0
  3. data/.rspec +2 -0
  4. data/Gemfile +4 -0
  5. data/Gemfile.lock +38 -0
  6. data/Manifest +24 -0
  7. data/README.md +225 -0
  8. data/Rakefile +16 -0
  9. data/TODO.md +8 -0
  10. data/autotest/discover.rb +1 -0
  11. data/lib/massive_record.rb +18 -0
  12. data/lib/massive_record/exceptions.rb +11 -0
  13. data/lib/massive_record/orm/attribute_methods.rb +61 -0
  14. data/lib/massive_record/orm/attribute_methods/dirty.rb +80 -0
  15. data/lib/massive_record/orm/attribute_methods/read.rb +23 -0
  16. data/lib/massive_record/orm/attribute_methods/write.rb +24 -0
  17. data/lib/massive_record/orm/base.rb +176 -0
  18. data/lib/massive_record/orm/callbacks.rb +52 -0
  19. data/lib/massive_record/orm/column.rb +18 -0
  20. data/lib/massive_record/orm/config.rb +47 -0
  21. data/lib/massive_record/orm/errors.rb +47 -0
  22. data/lib/massive_record/orm/finders.rb +125 -0
  23. data/lib/massive_record/orm/id_factory.rb +133 -0
  24. data/lib/massive_record/orm/persistence.rb +199 -0
  25. data/lib/massive_record/orm/schema.rb +4 -0
  26. data/lib/massive_record/orm/schema/column_families.rb +48 -0
  27. data/lib/massive_record/orm/schema/column_family.rb +102 -0
  28. data/lib/massive_record/orm/schema/column_interface.rb +91 -0
  29. data/lib/massive_record/orm/schema/common_interface.rb +48 -0
  30. data/lib/massive_record/orm/schema/field.rb +128 -0
  31. data/lib/massive_record/orm/schema/fields.rb +37 -0
  32. data/lib/massive_record/orm/schema/table_interface.rb +96 -0
  33. data/lib/massive_record/orm/table.rb +9 -0
  34. data/lib/massive_record/orm/validations.rb +52 -0
  35. data/lib/massive_record/spec/support/simple_database_cleaner.rb +52 -0
  36. data/lib/massive_record/thrift/hbase.rb +2307 -0
  37. data/lib/massive_record/thrift/hbase_constants.rb +14 -0
  38. data/lib/massive_record/thrift/hbase_types.rb +225 -0
  39. data/lib/massive_record/version.rb +3 -0
  40. data/lib/massive_record/wrapper/base.rb +28 -0
  41. data/lib/massive_record/wrapper/cell.rb +45 -0
  42. data/lib/massive_record/wrapper/column_families_collection.rb +19 -0
  43. data/lib/massive_record/wrapper/column_family.rb +22 -0
  44. data/lib/massive_record/wrapper/connection.rb +71 -0
  45. data/lib/massive_record/wrapper/row.rb +170 -0
  46. data/lib/massive_record/wrapper/scanner.rb +50 -0
  47. data/lib/massive_record/wrapper/table.rb +148 -0
  48. data/lib/massive_record/wrapper/tables_collection.rb +13 -0
  49. data/massive_record.gemspec +28 -0
  50. data/spec/config.yml.example +4 -0
  51. data/spec/orm/cases/attribute_methods_spec.rb +47 -0
  52. data/spec/orm/cases/auto_generate_id_spec.rb +54 -0
  53. data/spec/orm/cases/base_spec.rb +176 -0
  54. data/spec/orm/cases/callbacks_spec.rb +309 -0
  55. data/spec/orm/cases/column_spec.rb +49 -0
  56. data/spec/orm/cases/config_spec.rb +103 -0
  57. data/spec/orm/cases/dirty_spec.rb +129 -0
  58. data/spec/orm/cases/encoding_spec.rb +49 -0
  59. data/spec/orm/cases/finders_spec.rb +208 -0
  60. data/spec/orm/cases/hbase/connection_spec.rb +13 -0
  61. data/spec/orm/cases/i18n_spec.rb +32 -0
  62. data/spec/orm/cases/id_factory_spec.rb +75 -0
  63. data/spec/orm/cases/persistence_spec.rb +479 -0
  64. data/spec/orm/cases/table_spec.rb +81 -0
  65. data/spec/orm/cases/validation_spec.rb +92 -0
  66. data/spec/orm/models/address.rb +7 -0
  67. data/spec/orm/models/person.rb +15 -0
  68. data/spec/orm/models/test_class.rb +5 -0
  69. data/spec/orm/schema/column_families_spec.rb +186 -0
  70. data/spec/orm/schema/column_family_spec.rb +131 -0
  71. data/spec/orm/schema/column_interface_spec.rb +115 -0
  72. data/spec/orm/schema/field_spec.rb +196 -0
  73. data/spec/orm/schema/fields_spec.rb +126 -0
  74. data/spec/orm/schema/table_interface_spec.rb +171 -0
  75. data/spec/spec_helper.rb +15 -0
  76. data/spec/support/connection_helpers.rb +76 -0
  77. data/spec/support/mock_massive_record_connection.rb +80 -0
  78. data/spec/thrift/cases/encoding_spec.rb +48 -0
  79. data/spec/wrapper/cases/connection_spec.rb +53 -0
  80. data/spec/wrapper/cases/table_spec.rb +231 -0
  81. metadata +228 -0
@@ -0,0 +1,125 @@
1
+ module MassiveRecord
2
+ module ORM
3
+ module Finders
4
+ extend ActiveSupport::Concern
5
+
6
+ module ClassMethods
7
+ #
8
+ # Interface for retrieving objects based on key.
9
+ # Has some convenience behaviour like find :first, :last, :all.
10
+ #
11
+ def find(*args)
12
+ options = args.extract_options!.to_options
13
+ raise ArgumentError.new("At least one argument required!") if args.empty?
14
+ raise RecordNotFound.new("Can't find a #{model_name.human} without an ID.") if args.first.nil?
15
+ raise ArgumentError.new("Sorry, conditions are not supported!") if options.has_key? :conditions
16
+ args << options
17
+
18
+ type = args.shift if args.first.is_a? Symbol
19
+ find_many = type == :all
20
+ expected_result_size = nil
21
+
22
+ return (find_many ? [] : nil) unless table.exists?
23
+
24
+ result_from_table = if type
25
+ table.send(type, *args) # first() / all()
26
+ else
27
+ options = args.extract_options!
28
+ what_to_find = args.first
29
+ expected_result_size = 1
30
+
31
+ if args.first.kind_of?(Array)
32
+ find_many = true
33
+ elsif args.length > 1
34
+ find_many = true
35
+ what_to_find = args
36
+ end
37
+
38
+ expected_result_size = what_to_find.length if what_to_find.is_a? Array
39
+ table.find(what_to_find, options)
40
+ end
41
+
42
+ # Filter out unexpected IDs (unless type is set (all/first), in that case
43
+ # we have no expectations on the returned rows' ids)
44
+ unless type || result_from_table.blank?
45
+ if find_many
46
+ result_from_table.select! { |result| what_to_find.include? result.id }
47
+ else
48
+ if result_from_table.id != what_to_find
49
+ result_from_table = nil
50
+ end
51
+ end
52
+ end
53
+
54
+ raise RecordNotFound.new("Could not find #{model_name} with id=#{what_to_find}") if result_from_table.blank? && type.nil?
55
+
56
+ if find_many && expected_result_size && expected_result_size != result_from_table.length
57
+ raise RecordNotFound.new("Expected to find #{expected_result_size} records, but found only #{result_from_table.length}")
58
+ end
59
+
60
+ records = [result_from_table].compact.flatten.collect do |row|
61
+ instantiate(transpose_hbase_columns_to_record_attributes(row))
62
+ end
63
+
64
+ find_many ? records : records.first
65
+ end
66
+
67
+ def first(*args)
68
+ find(:first, *args)
69
+ end
70
+
71
+ def last(*args)
72
+ raise "Sorry, not implemented!"
73
+ end
74
+
75
+ def all(*args)
76
+ find(:all, *args)
77
+ end
78
+
79
+ def find_in_batches(*args)
80
+ table.find_in_batches(*args) do |rows|
81
+ records = rows.collect do |row|
82
+ instantiate(transpose_hbase_columns_to_record_attributes(row))
83
+ end
84
+ yield records
85
+ end
86
+ end
87
+
88
+ def find_each(*args)
89
+ find_in_batches(*args) do |rows|
90
+ rows.each do |row|
91
+ yield row
92
+ end
93
+ end
94
+ end
95
+
96
+
97
+ def exists?(id)
98
+ !!find(id) rescue false
99
+ end
100
+
101
+
102
+ private
103
+
104
+ def transpose_hbase_columns_to_record_attributes(row)
105
+ attributes = {:id => row.id}
106
+
107
+ autoload_column_families_and_fields_with(row.columns.keys)
108
+
109
+ # Parse the schema to populate the instance attributes
110
+ attributes_schema.each do |key, field|
111
+ cell = row.columns[field.unique_name]
112
+ attributes[field.name] = cell.nil? ? nil : cell.deserialize_value
113
+ end
114
+ attributes
115
+ end
116
+
117
+ def instantiate(record)
118
+ allocate.tap do |model|
119
+ model.init_with('attributes' => record)
120
+ end
121
+ end
122
+ end
123
+ end
124
+ end
125
+ end
@@ -0,0 +1,133 @@
1
+ require 'singleton'
2
+
3
+ module MassiveRecord
4
+ module ORM
5
+
6
+ #
7
+ # A factory class for unique IDs for any given tables.
8
+ #
9
+ # Usage:
10
+ # IdFactory.next_for(:cars) # => 1
11
+ # IdFactory.next_for(:cars) # => 2
12
+ # IdFactory.next_for(AClassRespondingToTableName) # => 1
13
+ # IdFactory.next_for("a_class_responding_to_table_names") # => 2
14
+ #
15
+ #
16
+ # Storage:
17
+ # Stored in id_factories table, under column family named tables.
18
+ # Field name equals to tables it has generated ids for, and it's
19
+ # values is integers (if the adapter supports it).
20
+ #
21
+ class IdFactory < Table
22
+ include Singleton
23
+
24
+ COLUMN_FAMILY_FOR_TABLES = :tables
25
+ ID = "id_factory"
26
+
27
+ column_family COLUMN_FAMILY_FOR_TABLES do
28
+ autoload_fields
29
+ end
30
+
31
+ #
32
+ # Returns the factory, singleton class.
33
+ # It will be a reloaded version each time instance
34
+ # is retrieved, or else it will fetch self from the
35
+ # database, or if all other fails return a new of self.
36
+ #
37
+ def self.instance
38
+ if table_exists?
39
+ begin
40
+ if @instance
41
+ @instance.reload
42
+ else
43
+ @instance = find(ID)
44
+ end
45
+ rescue RecordNotFound
46
+ @instance = nil
47
+ end
48
+ end
49
+
50
+ @instance = new unless @instance
51
+ @instance
52
+ end
53
+
54
+ #
55
+ # Delegates to the instance, just a shout cut.
56
+ #
57
+ def self.next_for(table)
58
+ instance.next_for(table)
59
+ end
60
+
61
+
62
+
63
+ #
64
+ # Returns a new and unique id for a given table name
65
+ # Table can a symbol, string or an object responding to table_name
66
+ #
67
+ def next_for(table)
68
+ table = table.respond_to?(:table_name) ? table.table_name : table.to_s
69
+ next_id :table => table
70
+ end
71
+
72
+
73
+
74
+ def id
75
+ ID
76
+ end
77
+
78
+ private
79
+
80
+ #
81
+ # Method which actually does the increment work for
82
+ # a given table name as string
83
+ #
84
+ def next_id(options = {})
85
+ options.assert_valid_keys(:table)
86
+ table_name = options.delete :table
87
+
88
+ create_field_or_ensure_type_integer_for(table_name)
89
+ atomic_increment!(table_name)
90
+ end
91
+
92
+
93
+
94
+
95
+ def create_field_or_ensure_type_integer_for(table_name)
96
+ if has_field_for? table_name
97
+ ensure_type_integer_for(table_name)
98
+ else
99
+ create_field_for(table_name)
100
+ end
101
+ end
102
+
103
+
104
+ #
105
+ # Creates a field for a table name which is new
106
+ # Feels a bit hackish, hooking in and doing some of what the
107
+ # autoload-functionality of column_family block above does too.
108
+ # But at least, we can "dynamicly" assign new attributes to this object.
109
+ #
110
+ def create_field_for(table_name)
111
+ add_field_to_column_family COLUMN_FAMILY_FOR_TABLES, table_name, :integer, :default => 0
112
+ end
113
+
114
+ #
115
+ # Just makes sure that definition of a field is set to integer.
116
+ # This is needed as the autoload functionlaity sets all types to strings.
117
+ #
118
+ def ensure_type_integer_for(table_name)
119
+ column_family_for_tables.field_by_name(table_name).type = :integer
120
+ self[table_name] = 0 if self[table_name].blank?
121
+ end
122
+
123
+
124
+ def has_field_for?(table_name)
125
+ respond_to? table_name
126
+ end
127
+
128
+ def column_family_for_tables
129
+ @column_family_for_tables ||= column_families.family_by_name(COLUMN_FAMILY_FOR_TABLES)
130
+ end
131
+ end
132
+ end
133
+ end
@@ -0,0 +1,199 @@
1
+ module MassiveRecord
2
+ module ORM
3
+ module Persistence
4
+ extend ActiveSupport::Concern
5
+
6
+ module ClassMethods
7
+ def create(attributes = {})
8
+ new(attributes).tap do |record|
9
+ record.save
10
+ end
11
+ end
12
+
13
+ def destroy_all
14
+ all.each { |record| record.destroy }
15
+ end
16
+ end
17
+
18
+
19
+ def new_record?
20
+ @new_record
21
+ end
22
+
23
+ def persisted?
24
+ !(new_record? || destroyed?)
25
+ end
26
+
27
+ def destroyed?
28
+ @destroyed
29
+ end
30
+
31
+
32
+ def reload
33
+ self.attributes_raw = self.class.find(id).attributes
34
+ self
35
+ end
36
+
37
+ def save(*)
38
+ create_or_update
39
+ end
40
+
41
+ def save!(*)
42
+ create_or_update or raise RecordNotSaved
43
+ end
44
+
45
+ def update_attribute(attr_name, value)
46
+ send("#{attr_name}=", value)
47
+ save(:validate => false)
48
+ end
49
+
50
+ def update_attributes(attributes)
51
+ self.attributes = attributes
52
+ save
53
+ end
54
+
55
+ def update_attributes!(attributes)
56
+ self.attributes = attributes
57
+ save!
58
+ end
59
+
60
+ # TODO This actually does nothing atm, but it's here and callbacks on it
61
+ # is working.
62
+ def touch
63
+ true
64
+ end
65
+
66
+ def destroy
67
+ @destroyed = row_for_record.destroy and freeze
68
+ end
69
+ alias_method :delete, :destroy
70
+
71
+
72
+
73
+
74
+ def increment(attr_name, by = 1)
75
+ raise NotNumericalFieldError unless attributes_schema[attr_name.to_s].type == :integer
76
+ self[attr_name] ||= 0
77
+ self[attr_name] += by
78
+ self
79
+ end
80
+
81
+ def increment!(attr_name, by = 1)
82
+ increment(attr_name, by).update_attribute(attr_name, self[attr_name])
83
+ end
84
+
85
+ # Atomic increment of an attribute. Please note that it's the
86
+ # adapter (or the wrapper) which needs to guarantee that the update
87
+ # is atomic, and as of writing this the Thrift adapter / wrapper does
88
+ # not do this anatomic.
89
+ def atomic_increment!(attr_name, by = 1)
90
+ ensure_that_we_have_table_and_column_families!
91
+ attr_name = attr_name.to_s
92
+
93
+ row = row_for_record
94
+ row.values = attributes_to_row_values_hash([attr_name])
95
+ self[attr_name] = row.atomic_increment(attributes_schema[attr_name].unique_name, by).to_i
96
+ end
97
+
98
+ def decrement(attr_name, by = 1)
99
+ raise NotNumericalFieldError unless attributes_schema[attr_name.to_s].type == :integer
100
+ self[attr_name] ||= 0
101
+ self[attr_name] -= by
102
+ self
103
+ end
104
+
105
+ def decrement!(attr_name, by = 1)
106
+ decrement(attr_name, by).update_attribute(attr_name, self[attr_name])
107
+ end
108
+
109
+
110
+ private
111
+
112
+
113
+ def create_or_update
114
+ !!(new_record? ? create : update)
115
+ end
116
+
117
+ def create
118
+ ensure_that_we_have_table_and_column_families!
119
+
120
+ if saved = store_record_to_database
121
+ @new_record = false
122
+ end
123
+ saved
124
+ end
125
+
126
+ def update(attribute_names_to_update = attributes.keys)
127
+ ensure_that_we_have_table_and_column_families!
128
+
129
+ store_record_to_database(attribute_names_to_update)
130
+ end
131
+
132
+
133
+
134
+
135
+ #
136
+ # Takes care of the actual storing of the record to the database
137
+ # Both update and create is using this
138
+ #
139
+ def store_record_to_database(attribute_names_to_update = [])
140
+ row = row_for_record
141
+ row.values = attributes_to_row_values_hash(attribute_names_to_update)
142
+ row.save
143
+ end
144
+
145
+
146
+ #
147
+ # Iterates over tables and column families and ensure that we
148
+ # have what we need
149
+ #
150
+ def ensure_that_we_have_table_and_column_families!
151
+ if !self.class.connection.tables.include? self.class.table_name
152
+ missing_family_names = calculate_missing_family_names
153
+ self.class.table.create_column_families(missing_family_names) unless missing_family_names.empty?
154
+ self.class.table.save
155
+ end
156
+
157
+ raise ColumnFamiliesMissingError.new(calculate_missing_family_names) if !calculate_missing_family_names.empty?
158
+ end
159
+
160
+ #
161
+ # Calculate which column families are missing in the database in
162
+ # context of what the schema instructs.
163
+ #
164
+ def calculate_missing_family_names
165
+ existing_family_names = self.class.table.fetch_column_families.collect(&:name) rescue []
166
+ expected_family_names = column_families ? column_families.collect(&:name) : []
167
+
168
+ expected_family_names.collect(&:to_s) - existing_family_names.collect(&:to_s)
169
+ end
170
+
171
+ #
172
+ # Returns a Wrapper::Row class which we can manipulate this
173
+ # record in the database with
174
+ #
175
+ def row_for_record
176
+ raise IdMissing.new("You must set an ID before save.") if id.blank?
177
+
178
+ MassiveRecord::Wrapper::Row.new({
179
+ :id => id,
180
+ :table => self.class.table
181
+ })
182
+ end
183
+
184
+ #
185
+ # Returns attributes on a form which Wrapper::Row expects
186
+ #
187
+ def attributes_to_row_values_hash(only_attr_names = [])
188
+ values = Hash.new { |hash, key| hash[key] = Hash.new }
189
+
190
+ attributes_schema.each do |attr_name, orm_field|
191
+ next unless only_attr_names.empty? || only_attr_names.include?(attr_name)
192
+ values[orm_field.column_family.name][orm_field.column] = send(attr_name)
193
+ end
194
+
195
+ values
196
+ end
197
+ end
198
+ end
199
+ end