dm-adapter-simpledb 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. data/.gitignore +1 -0
  2. data/History.txt +21 -0
  3. data/README +21 -8
  4. data/Rakefile +35 -23
  5. data/VERSION +1 -1
  6. data/dm-adapter-simpledb.gemspec +44 -24
  7. data/lib/dm-adapter-simpledb.rb +17 -0
  8. data/lib/dm-adapter-simpledb/adapters/simpledb_adapter.rb +339 -0
  9. data/lib/dm-adapter-simpledb/chunked_string.rb +54 -0
  10. data/lib/dm-adapter-simpledb/migrations/simpledb_adapter.rb +45 -0
  11. data/lib/dm-adapter-simpledb/rake.rb +43 -0
  12. data/lib/dm-adapter-simpledb/record.rb +318 -0
  13. data/lib/{simpledb_adapter → dm-adapter-simpledb}/sdb_array.rb +0 -0
  14. data/lib/dm-adapter-simpledb/table.rb +40 -0
  15. data/lib/dm-adapter-simpledb/utils.rb +15 -0
  16. data/lib/simpledb_adapter.rb +2 -469
  17. data/scripts/simple_benchmark.rb +1 -1
  18. data/spec/{associations_spec.rb → integration/associations_spec.rb} +0 -0
  19. data/spec/{compliance_spec.rb → integration/compliance_spec.rb} +0 -0
  20. data/spec/{date_spec.rb → integration/date_spec.rb} +0 -0
  21. data/spec/{limit_and_order_spec.rb → integration/limit_and_order_spec.rb} +0 -0
  22. data/spec/{migrations_spec.rb → integration/migrations_spec.rb} +0 -0
  23. data/spec/{multiple_records_spec.rb → integration/multiple_records_spec.rb} +0 -0
  24. data/spec/{nils_spec.rb → integration/nils_spec.rb} +0 -0
  25. data/spec/{sdb_array_spec.rb → integration/sdb_array_spec.rb} +4 -5
  26. data/spec/{simpledb_adapter_spec.rb → integration/simpledb_adapter_spec.rb} +65 -0
  27. data/spec/{spec_helper.rb → integration/spec_helper.rb} +8 -3
  28. data/spec/unit/record_spec.rb +346 -0
  29. data/spec/unit/simpledb_adapter_spec.rb +80 -0
  30. data/spec/unit/unit_spec_helper.rb +26 -0
  31. metadata +58 -24
  32. data/tasks/devver.rake +0 -167
@@ -0,0 +1,54 @@
1
+ module DmAdapterSimpledb
2
+ class ChunkedString < String
3
+ MAX_CHUNK_SIZE = 1019
4
+
5
+ def self.valid?(values)
6
+ values.all?{|v| v =~ /^\d{4}:/}
7
+ end
8
+
9
+ def initialize(string_or_array)
10
+ case string_or_array
11
+ when Array then super(chunks_to_string(string_or_array))
12
+ else super(string_or_array)
13
+ end
14
+ end
15
+
16
+ def to_ary
17
+ string_to_chunks(self)
18
+ end
19
+
20
+ alias_method :to_a, :to_ary
21
+
22
+ private
23
+
24
+ def string_to_chunks(value)
25
+ return [value] if value.size <= 1019
26
+ chunks = value.to_s.scan(%r/.{1,1019}/m) # 1024 - '1024:'.size
27
+ i = -1
28
+ fmt = '%04d:'
29
+ chunks.map!{|chunk| [(fmt % (i += 1)), chunk].join}
30
+ raise ArgumentError, 'that is just too big yo!' if chunks.size >= 256
31
+ chunks
32
+ end
33
+
34
+ def chunks_to_string(value)
35
+ begin
36
+ chunks =
37
+ Array(value).flatten.map do |chunk|
38
+ index, text = chunk.split(%r/:/, 2)
39
+ [Float(index).to_i, text]
40
+ end
41
+ chunks.replace chunks.sort_by{|index, text| index}
42
+ string_result = chunks.map!{|index, text| text}.join
43
+ string_result
44
+ rescue ArgumentError, TypeError
45
+ #return original value, they could have put strings in the system not
46
+ #using the adapter or previous versions that are larger than chunk size,
47
+ #but less than 1024
48
+ value
49
+ end
50
+ end
51
+
52
+
53
+ end
54
+ end
@@ -0,0 +1,45 @@
1
+ module DataMapper
2
+ module Migrations
3
+ #integrated from http://github.com/edward/dm-simpledb/tree/master
4
+ module SimpledbAdapter
5
+
6
+ module ClassMethods
7
+
8
+ end
9
+
10
+ def self.included(other)
11
+ other.extend ClassMethods
12
+
13
+ DataMapper.extend(::DataMapper::Migrations::SingletonMethods)
14
+
15
+ [ :Repository, :Model ].each do |name|
16
+ ::DataMapper.const_get(name).send(:include, Migrations.const_get(name))
17
+ end
18
+ end
19
+
20
+ # Returns whether the storage_name exists.
21
+ # @param storage_name<String> a String defining the name of a domain
22
+ # @return <Boolean> true if the storage exists
23
+ def storage_exists?(storage_name)
24
+ domains = sdb.list_domains[:domains]
25
+ domains.detect {|d| d == storage_name }!=nil
26
+ end
27
+
28
+ def create_model_storage(model)
29
+ sdb.create_domain(@sdb_options[:domain])
30
+ end
31
+
32
+ #On SimpleDB you probably don't want to destroy the whole domain
33
+ #if you are just adding fields it is automatically supported
34
+ #default to non destructive migrate, to destroy run
35
+ #rake db:automigrate destroy=true
36
+ def destroy_model_storage(model)
37
+ if ENV['destroy']!=nil && ENV['destroy']=='true'
38
+ sdb.delete_domain(@sdb_options[:domain])
39
+ end
40
+ end
41
+
42
+ end # module Migration
43
+ end # module Migration
44
+ end
45
+
@@ -0,0 +1,43 @@
1
+ namespace :simpledb do
2
+ desc "Migrate records to be compatable with current DM/SimpleDB adapter"
3
+ task :migrate, :domain do |t, args|
4
+ raise "THIS IS A WORK IN PROGRESS AND WILL DESTROY YOUR DATA"
5
+ require 'progressbar'
6
+ require 'right_aws'
7
+ require 'dm-adapter-simpledb/record'
8
+
9
+ puts "Initializing connection..."
10
+ domain = args.domain
11
+ sdb = RightAws::SdbInterface.new
12
+ puts "Counting records..."
13
+ num_legacy_records = 0
14
+ query = "select count(*) from #{domain} where (simpledb_type is not null) and (__dm_metadata is null)"
15
+ next_token = nil
16
+ while(results = sdb.select(query, next_token)) do
17
+ next_token = results[:next_token]
18
+ count = results[:items].first["Domain"]["Count"].first.to_i
19
+ num_legacy_records += count
20
+ break if next_token.nil?
21
+ end
22
+ puts "Found #{num_legacy_records} to migrate"
23
+
24
+ pbar = ProgressBar.new("migrate", num_legacy_records)
25
+ query = "select * from #{domain} where (simpledb_type is not null) and (__dm_metadata is null)"
26
+ while(results = sdb.select(query, next_token)) do
27
+ next_token = results[:next_token]
28
+ items = results[:items]
29
+ items.each do |item|
30
+ legacy_record = DmAdapterSimpledb::Record.from_simpledb_hash(item)
31
+ new_record = legacy_record.migrate
32
+ updates = new_record.writable_attributes
33
+ deletes = new_record.deletable_attributes
34
+ sdb.put_attributes(domain, new_record.item_name, updates)
35
+ sdb.delete_attributes(domain, new_record.item_name, deletes)
36
+ pbar.inc
37
+ end
38
+ break if next_token.nil?
39
+ end
40
+ pbar.finish
41
+
42
+ end
43
+ end
@@ -0,0 +1,318 @@
1
+ require 'dm-core'
2
+ require 'dm-adapter-simpledb/utils'
3
+ require 'dm-adapter-simpledb/chunked_string'
4
+ require 'dm-adapter-simpledb/table'
5
+
6
+ # TODO
7
+ # * V1.1: Store type in __dm_metadata
8
+ # * V1.1: Store type as non-munged class name
9
+
10
+ module DmAdapterSimpledb
11
+ class Record
12
+ include Utils
13
+
14
+ METADATA_KEY = "__dm_metadata"
15
+ STORAGE_NAME_KEY = "simpledb_type"
16
+ META_KEYS = [METADATA_KEY, STORAGE_NAME_KEY]
17
+ CURRENT_VERSION = "01.01.00"
18
+
19
+ def self.from_simpledb_hash(hash)
20
+ data_version = data_version(simpledb_attributes(hash))
21
+ versions.fetch(data_version) do
22
+ raise "Unknown data version for: #{hash.inspect}"
23
+ end.new(hash)
24
+ end
25
+
26
+ def self.from_resource(resource)
27
+ versions.fetch(CURRENT_VERSION).new(resource)
28
+ end
29
+
30
+ def self.register(klass, version)
31
+ versions[version] = klass
32
+ end
33
+
34
+ def self.versions
35
+ @versions ||= {}
36
+ end
37
+
38
+ def self.version(version=nil)
39
+ if version
40
+ Record.register(self, version)
41
+ @version = version
42
+ else
43
+ @version
44
+ end
45
+ end
46
+
47
+ def self.data_version(simpledb_attributes)
48
+ simpledb_attributes.fetch(METADATA_KEY){[]}.grep(/v\d\d\.\d\d\.\d\d/) do
49
+ |version_stamp|
50
+ return version_stamp[1..-1]
51
+ end
52
+ return "00.00.00"
53
+ end
54
+
55
+ def self.simpledb_attributes(hash)
56
+ hash.values.first
57
+ end
58
+
59
+ attr_reader :simpledb_attributes
60
+ attr_reader :deletable_attributes
61
+ attr_reader :item_name
62
+ alias_method :writable_attributes, :simpledb_attributes
63
+
64
+ def initialize(hash_or_resource)
65
+ case hash_or_resource
66
+ when DataMapper::Resource then
67
+ attrs_to_update, attrs_to_delete = extract_attributes(hash_or_resource)
68
+ @simpledb_attributes = attrs_to_update
69
+ @deletable_attributes = attrs_to_delete
70
+ @item_name = item_name_for_resource(hash_or_resource)
71
+ when Hash
72
+ hash = hash_or_resource
73
+ @item_name = hash.keys.first
74
+ @simpledb_attributes = hash.values.first
75
+ @deletable_attributes = []
76
+ else
77
+ raise "Don't know how to initialize from #{hash_or_resource.inspect}"
78
+ end
79
+ end
80
+
81
+ # Convert to a Hash suitable for initializing a Resource
82
+ #
83
+ # @param [PropertySet] fields
84
+ # The fields to extract
85
+ def to_resource_hash(fields)
86
+ result = transform_hash(fields) {|hash, property|
87
+ hash[property.name.to_s] = self[property.field, property]
88
+ }
89
+ result
90
+ end
91
+
92
+ # Deprecated - we are moving the type information under the metadata key
93
+ def storage_name
94
+ simpledb_attributes[STORAGE_NAME_KEY].first
95
+ end
96
+
97
+ def [](attribute, type)
98
+ values = Array(simpledb_attributes[attribute])
99
+ coerce_to(values, type)
100
+ end
101
+
102
+ def coerce_to(values, type_or_property)
103
+ case type_or_property
104
+ when DataMapper::Property
105
+ coerce_to_property(values, type_or_property)
106
+ when Class
107
+ coerce_to_type(values, type_or_property)
108
+ else raise "Should never get here"
109
+ end
110
+ end
111
+
112
+ def coerce_to_property(value, property)
113
+ property.typecast(coerce_to_type(value, property.type))
114
+ end
115
+
116
+ def coerce_to_type(values, type)
117
+ case
118
+ when type <= String
119
+ case values.size
120
+ when 0
121
+ nil
122
+ when 1
123
+ values.first
124
+ else
125
+ ChunkedString.new(values)
126
+ end
127
+ when type <= Array, type <= DataMapper::Types::SdbArray
128
+ values
129
+ else
130
+ values.first
131
+ end
132
+ end
133
+
134
+ def version
135
+ self.class.version || self.class.data_version(simpledb_attributes)
136
+ end
137
+
138
+ def version_token
139
+ "v#{version}"
140
+ end
141
+
142
+ # Returns the "Table" this record belongs to. SimpleDB has no concept of
143
+ # tables, but we fake it with metadata.
144
+ def table
145
+ Table.name_from_metadata(metadata) ||
146
+ storage_name
147
+ end
148
+
149
+ def metadata
150
+ simpledb_attributes[METADATA_KEY]
151
+ end
152
+
153
+ # Returns a record of the current version
154
+ def migrate
155
+ new_record = Record.versions[CURRENT_VERSION].allocate
156
+ new_record.item_name = item_name
157
+ data = transform_hash(simpledb_attributes) {
158
+ |hash, key, values|
159
+ hash[key] = coerce_heuristically(values)
160
+ }
161
+ updates = {}
162
+ deletes = []
163
+ data.each_pair do |key, values|
164
+ if Array(values).empty?
165
+ deletes << key
166
+ else
167
+ updates[key] = values
168
+ end
169
+ end
170
+ new_record.add_metadata_to!(updates, table)
171
+ new_record.simpledb_attributes = updates
172
+ new_record.deletable_attributes = deletes
173
+ new_record
174
+ end
175
+
176
+ def add_metadata_to!(hash, table_name)
177
+ hash.merge!({
178
+ STORAGE_NAME_KEY => [table_name],
179
+ METADATA_KEY => [version_token, Table.token_for(table_name)]
180
+ })
181
+ end
182
+
183
+ protected
184
+
185
+ attr_writer :item_name
186
+ attr_writer :simpledb_attributes
187
+ attr_writer :deletable_attributes
188
+
189
+ private
190
+
191
+ def app_data
192
+ transform_hash(simpledb_attributes) {|h,k,v|
193
+ h[k] = v unless META_KEYS.include?(k)
194
+ }
195
+ end
196
+
197
+ def extract_attributes(resource)
198
+ attributes = resource.attributes(:property)
199
+ attributes = attributes.to_a.map {|a| [a.first.name.to_s, a.last]}.to_hash
200
+ attributes = adjust_to_sdb_attributes(attributes)
201
+ updates, deletes = attributes.partition{|name,value|
202
+ !Array(value).empty?
203
+ }
204
+ attrs_to_update = updates.inject({}){|h, (k,v)| h[k] = v; h}
205
+ table = Table.new(resource.model)
206
+ if resource.new?
207
+ add_metadata_to!(attrs_to_update, table.simpledb_type)
208
+ end
209
+ attrs_to_delete = deletes.inject({}){|h, (k,v)| h[k] = v; h}.keys
210
+ [attrs_to_update, attrs_to_delete]
211
+ end
212
+
213
+ # hack for converting and storing strings longer than 1024 one thing to
214
+ # note if you use string longer than 1019 chars you will loose the ability
215
+ # to do full text matching on queries as the string can be broken at any
216
+ # place during chunking
217
+ def adjust_to_sdb_attributes(attrs)
218
+ attrs = transform_hash(attrs) do |result, key, value|
219
+ if primitive_value_of(value.class) <= String
220
+ result[key] = ChunkedString.new(value).to_a
221
+ elsif value.class == Object # This is for SdbArray
222
+ result[key] = value.to_ary
223
+ elsif primitive_value_of(value.class) <= Array
224
+ result[key] = value
225
+ elsif value.nil?
226
+ result[key] = nil
227
+ else
228
+ result[key] = [value.to_s]
229
+ end
230
+ end
231
+ # Stringify keys
232
+ transform_hash(attrs) {|h, k, v| h[k.to_s] = v}
233
+ end
234
+
235
+ def primitive_value_of(type)
236
+ if type < DataMapper::Type
237
+ type.primitive
238
+ else
239
+ type
240
+ end
241
+ end
242
+
243
+ # Creates an item name for a resource
244
+ def item_name_for_resource(resource)
245
+ table = Table.new(resource.model)
246
+ sdb_type = table.simpledb_type
247
+
248
+ item_name = "#{sdb_type}+"
249
+ keys = table.keys_for_model
250
+ item_name += keys.map do |property|
251
+ property.get(resource)
252
+ end.join('-')
253
+
254
+ Digest::SHA1.hexdigest(item_name)
255
+ end
256
+
257
+ def coerce_heuristically(values)
258
+ if values
259
+ case values.size
260
+ when 0
261
+ values
262
+ when 1
263
+ value = coerce_to_type(values, String)
264
+ value.nil? ? [] : [value]
265
+ else
266
+ if ChunkedString.valid?(values)
267
+ string = ChunkedString.new(values)
268
+ coerced_string = coerce_to_type([string], Array).first
269
+ ChunkedString.new(coerced_string).to_a
270
+ else
271
+ coerce_to_type(values, Array)
272
+ end
273
+ end
274
+ else
275
+ []
276
+ end
277
+ end
278
+
279
+ end
280
+
281
+ # Version 0 records are records that have no associated version
282
+ # metadata. Any records created by versions of the DataMapper/SimplDB adapter
283
+ # prior to 1.1.0 are considered to be version 0.
284
+ #
285
+ # Version 0 records have a few distinguishing characteristics:
286
+ # * An attribute with the token "nil" as its sole member is treated as a
287
+ # null/empty attribute.
288
+ # * The token "[[[NEWLINE]]]" inside of String attributes is replaced with \n
289
+ class RecordV0 < Record
290
+ version "00.00.00"
291
+
292
+ def coerce_to_type(values, type)
293
+ values = values.map{|v| replace_newline_placeholders(v)}
294
+ result = super(values, type)
295
+
296
+ if result == "nil"
297
+ nil
298
+ elsif result == ["nil"]
299
+ []
300
+ elsif result && type <= String
301
+ # TODO redundant
302
+ replace_newline_placeholders(result)
303
+ else
304
+ result
305
+ end
306
+ end
307
+
308
+ private
309
+
310
+ def replace_newline_placeholders(value)
311
+ value.gsub("[[[NEWLINE]]]", "\n")
312
+ end
313
+ end
314
+
315
+ class RecordV1_1 < Record
316
+ version "01.01.00"
317
+ end
318
+ end