dm-adapter-simpledb 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +1 -0
- data/History.txt +21 -0
- data/README +21 -8
- data/Rakefile +35 -23
- data/VERSION +1 -1
- data/dm-adapter-simpledb.gemspec +44 -24
- data/lib/dm-adapter-simpledb.rb +17 -0
- data/lib/dm-adapter-simpledb/adapters/simpledb_adapter.rb +339 -0
- data/lib/dm-adapter-simpledb/chunked_string.rb +54 -0
- data/lib/dm-adapter-simpledb/migrations/simpledb_adapter.rb +45 -0
- data/lib/dm-adapter-simpledb/rake.rb +43 -0
- data/lib/dm-adapter-simpledb/record.rb +318 -0
- data/lib/{simpledb_adapter → dm-adapter-simpledb}/sdb_array.rb +0 -0
- data/lib/dm-adapter-simpledb/table.rb +40 -0
- data/lib/dm-adapter-simpledb/utils.rb +15 -0
- data/lib/simpledb_adapter.rb +2 -469
- data/scripts/simple_benchmark.rb +1 -1
- data/spec/{associations_spec.rb → integration/associations_spec.rb} +0 -0
- data/spec/{compliance_spec.rb → integration/compliance_spec.rb} +0 -0
- data/spec/{date_spec.rb → integration/date_spec.rb} +0 -0
- data/spec/{limit_and_order_spec.rb → integration/limit_and_order_spec.rb} +0 -0
- data/spec/{migrations_spec.rb → integration/migrations_spec.rb} +0 -0
- data/spec/{multiple_records_spec.rb → integration/multiple_records_spec.rb} +0 -0
- data/spec/{nils_spec.rb → integration/nils_spec.rb} +0 -0
- data/spec/{sdb_array_spec.rb → integration/sdb_array_spec.rb} +4 -5
- data/spec/{simpledb_adapter_spec.rb → integration/simpledb_adapter_spec.rb} +65 -0
- data/spec/{spec_helper.rb → integration/spec_helper.rb} +8 -3
- data/spec/unit/record_spec.rb +346 -0
- data/spec/unit/simpledb_adapter_spec.rb +80 -0
- data/spec/unit/unit_spec_helper.rb +26 -0
- metadata +58 -24
- data/tasks/devver.rake +0 -167
@@ -0,0 +1,54 @@
|
|
1
|
+
module DmAdapterSimpledb
|
2
|
+
class ChunkedString < String
|
3
|
+
MAX_CHUNK_SIZE = 1019
|
4
|
+
|
5
|
+
def self.valid?(values)
|
6
|
+
values.all?{|v| v =~ /^\d{4}:/}
|
7
|
+
end
|
8
|
+
|
9
|
+
def initialize(string_or_array)
|
10
|
+
case string_or_array
|
11
|
+
when Array then super(chunks_to_string(string_or_array))
|
12
|
+
else super(string_or_array)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def to_ary
|
17
|
+
string_to_chunks(self)
|
18
|
+
end
|
19
|
+
|
20
|
+
alias_method :to_a, :to_ary
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def string_to_chunks(value)
|
25
|
+
return [value] if value.size <= 1019
|
26
|
+
chunks = value.to_s.scan(%r/.{1,1019}/m) # 1024 - '1024:'.size
|
27
|
+
i = -1
|
28
|
+
fmt = '%04d:'
|
29
|
+
chunks.map!{|chunk| [(fmt % (i += 1)), chunk].join}
|
30
|
+
raise ArgumentError, 'that is just too big yo!' if chunks.size >= 256
|
31
|
+
chunks
|
32
|
+
end
|
33
|
+
|
34
|
+
def chunks_to_string(value)
|
35
|
+
begin
|
36
|
+
chunks =
|
37
|
+
Array(value).flatten.map do |chunk|
|
38
|
+
index, text = chunk.split(%r/:/, 2)
|
39
|
+
[Float(index).to_i, text]
|
40
|
+
end
|
41
|
+
chunks.replace chunks.sort_by{|index, text| index}
|
42
|
+
string_result = chunks.map!{|index, text| text}.join
|
43
|
+
string_result
|
44
|
+
rescue ArgumentError, TypeError
|
45
|
+
#return original value, they could have put strings in the system not
|
46
|
+
#using the adapter or previous versions that are larger than chunk size,
|
47
|
+
#but less than 1024
|
48
|
+
value
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module DataMapper
|
2
|
+
module Migrations
|
3
|
+
#integrated from http://github.com/edward/dm-simpledb/tree/master
|
4
|
+
module SimpledbAdapter
|
5
|
+
|
6
|
+
module ClassMethods
|
7
|
+
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.included(other)
|
11
|
+
other.extend ClassMethods
|
12
|
+
|
13
|
+
DataMapper.extend(::DataMapper::Migrations::SingletonMethods)
|
14
|
+
|
15
|
+
[ :Repository, :Model ].each do |name|
|
16
|
+
::DataMapper.const_get(name).send(:include, Migrations.const_get(name))
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
# Returns whether the storage_name exists.
|
21
|
+
# @param storage_name<String> a String defining the name of a domain
|
22
|
+
# @return <Boolean> true if the storage exists
|
23
|
+
def storage_exists?(storage_name)
|
24
|
+
domains = sdb.list_domains[:domains]
|
25
|
+
domains.detect {|d| d == storage_name }!=nil
|
26
|
+
end
|
27
|
+
|
28
|
+
def create_model_storage(model)
|
29
|
+
sdb.create_domain(@sdb_options[:domain])
|
30
|
+
end
|
31
|
+
|
32
|
+
#On SimpleDB you probably don't want to destroy the whole domain
|
33
|
+
#if you are just adding fields it is automatically supported
|
34
|
+
#default to non destructive migrate, to destroy run
|
35
|
+
#rake db:automigrate destroy=true
|
36
|
+
def destroy_model_storage(model)
|
37
|
+
if ENV['destroy']!=nil && ENV['destroy']=='true'
|
38
|
+
sdb.delete_domain(@sdb_options[:domain])
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
end # module Migration
|
43
|
+
end # module Migration
|
44
|
+
end
|
45
|
+
|
@@ -0,0 +1,43 @@
|
|
1
|
+
namespace :simpledb do
|
2
|
+
desc "Migrate records to be compatable with current DM/SimpleDB adapter"
|
3
|
+
task :migrate, :domain do |t, args|
|
4
|
+
raise "THIS IS A WORK IN PROGRESS AND WILL DESTROY YOUR DATA"
|
5
|
+
require 'progressbar'
|
6
|
+
require 'right_aws'
|
7
|
+
require 'dm-adapter-simpledb/record'
|
8
|
+
|
9
|
+
puts "Initializing connection..."
|
10
|
+
domain = args.domain
|
11
|
+
sdb = RightAws::SdbInterface.new
|
12
|
+
puts "Counting records..."
|
13
|
+
num_legacy_records = 0
|
14
|
+
query = "select count(*) from #{domain} where (simpledb_type is not null) and (__dm_metadata is null)"
|
15
|
+
next_token = nil
|
16
|
+
while(results = sdb.select(query, next_token)) do
|
17
|
+
next_token = results[:next_token]
|
18
|
+
count = results[:items].first["Domain"]["Count"].first.to_i
|
19
|
+
num_legacy_records += count
|
20
|
+
break if next_token.nil?
|
21
|
+
end
|
22
|
+
puts "Found #{num_legacy_records} to migrate"
|
23
|
+
|
24
|
+
pbar = ProgressBar.new("migrate", num_legacy_records)
|
25
|
+
query = "select * from #{domain} where (simpledb_type is not null) and (__dm_metadata is null)"
|
26
|
+
while(results = sdb.select(query, next_token)) do
|
27
|
+
next_token = results[:next_token]
|
28
|
+
items = results[:items]
|
29
|
+
items.each do |item|
|
30
|
+
legacy_record = DmAdapterSimpledb::Record.from_simpledb_hash(item)
|
31
|
+
new_record = legacy_record.migrate
|
32
|
+
updates = new_record.writable_attributes
|
33
|
+
deletes = new_record.deletable_attributes
|
34
|
+
sdb.put_attributes(domain, new_record.item_name, updates)
|
35
|
+
sdb.delete_attributes(domain, new_record.item_name, deletes)
|
36
|
+
pbar.inc
|
37
|
+
end
|
38
|
+
break if next_token.nil?
|
39
|
+
end
|
40
|
+
pbar.finish
|
41
|
+
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,318 @@
|
|
1
|
+
require 'dm-core'
|
2
|
+
require 'dm-adapter-simpledb/utils'
|
3
|
+
require 'dm-adapter-simpledb/chunked_string'
|
4
|
+
require 'dm-adapter-simpledb/table'
|
5
|
+
|
6
|
+
# TODO
|
7
|
+
# * V1.1: Store type in __dm_metadata
|
8
|
+
# * V1.1: Store type as non-munged class name
|
9
|
+
|
10
|
+
module DmAdapterSimpledb
|
11
|
+
class Record
|
12
|
+
include Utils
|
13
|
+
|
14
|
+
METADATA_KEY = "__dm_metadata"
|
15
|
+
STORAGE_NAME_KEY = "simpledb_type"
|
16
|
+
META_KEYS = [METADATA_KEY, STORAGE_NAME_KEY]
|
17
|
+
CURRENT_VERSION = "01.01.00"
|
18
|
+
|
19
|
+
def self.from_simpledb_hash(hash)
|
20
|
+
data_version = data_version(simpledb_attributes(hash))
|
21
|
+
versions.fetch(data_version) do
|
22
|
+
raise "Unknown data version for: #{hash.inspect}"
|
23
|
+
end.new(hash)
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.from_resource(resource)
|
27
|
+
versions.fetch(CURRENT_VERSION).new(resource)
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.register(klass, version)
|
31
|
+
versions[version] = klass
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.versions
|
35
|
+
@versions ||= {}
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.version(version=nil)
|
39
|
+
if version
|
40
|
+
Record.register(self, version)
|
41
|
+
@version = version
|
42
|
+
else
|
43
|
+
@version
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def self.data_version(simpledb_attributes)
|
48
|
+
simpledb_attributes.fetch(METADATA_KEY){[]}.grep(/v\d\d\.\d\d\.\d\d/) do
|
49
|
+
|version_stamp|
|
50
|
+
return version_stamp[1..-1]
|
51
|
+
end
|
52
|
+
return "00.00.00"
|
53
|
+
end
|
54
|
+
|
55
|
+
def self.simpledb_attributes(hash)
|
56
|
+
hash.values.first
|
57
|
+
end
|
58
|
+
|
59
|
+
attr_reader :simpledb_attributes
|
60
|
+
attr_reader :deletable_attributes
|
61
|
+
attr_reader :item_name
|
62
|
+
alias_method :writable_attributes, :simpledb_attributes
|
63
|
+
|
64
|
+
def initialize(hash_or_resource)
|
65
|
+
case hash_or_resource
|
66
|
+
when DataMapper::Resource then
|
67
|
+
attrs_to_update, attrs_to_delete = extract_attributes(hash_or_resource)
|
68
|
+
@simpledb_attributes = attrs_to_update
|
69
|
+
@deletable_attributes = attrs_to_delete
|
70
|
+
@item_name = item_name_for_resource(hash_or_resource)
|
71
|
+
when Hash
|
72
|
+
hash = hash_or_resource
|
73
|
+
@item_name = hash.keys.first
|
74
|
+
@simpledb_attributes = hash.values.first
|
75
|
+
@deletable_attributes = []
|
76
|
+
else
|
77
|
+
raise "Don't know how to initialize from #{hash_or_resource.inspect}"
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
# Convert to a Hash suitable for initializing a Resource
|
82
|
+
#
|
83
|
+
# @param [PropertySet] fields
|
84
|
+
# The fields to extract
|
85
|
+
def to_resource_hash(fields)
|
86
|
+
result = transform_hash(fields) {|hash, property|
|
87
|
+
hash[property.name.to_s] = self[property.field, property]
|
88
|
+
}
|
89
|
+
result
|
90
|
+
end
|
91
|
+
|
92
|
+
# Deprecated - we are moving the type information under the metadata key
|
93
|
+
def storage_name
|
94
|
+
simpledb_attributes[STORAGE_NAME_KEY].first
|
95
|
+
end
|
96
|
+
|
97
|
+
def [](attribute, type)
|
98
|
+
values = Array(simpledb_attributes[attribute])
|
99
|
+
coerce_to(values, type)
|
100
|
+
end
|
101
|
+
|
102
|
+
def coerce_to(values, type_or_property)
|
103
|
+
case type_or_property
|
104
|
+
when DataMapper::Property
|
105
|
+
coerce_to_property(values, type_or_property)
|
106
|
+
when Class
|
107
|
+
coerce_to_type(values, type_or_property)
|
108
|
+
else raise "Should never get here"
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
def coerce_to_property(value, property)
|
113
|
+
property.typecast(coerce_to_type(value, property.type))
|
114
|
+
end
|
115
|
+
|
116
|
+
def coerce_to_type(values, type)
|
117
|
+
case
|
118
|
+
when type <= String
|
119
|
+
case values.size
|
120
|
+
when 0
|
121
|
+
nil
|
122
|
+
when 1
|
123
|
+
values.first
|
124
|
+
else
|
125
|
+
ChunkedString.new(values)
|
126
|
+
end
|
127
|
+
when type <= Array, type <= DataMapper::Types::SdbArray
|
128
|
+
values
|
129
|
+
else
|
130
|
+
values.first
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
def version
|
135
|
+
self.class.version || self.class.data_version(simpledb_attributes)
|
136
|
+
end
|
137
|
+
|
138
|
+
def version_token
|
139
|
+
"v#{version}"
|
140
|
+
end
|
141
|
+
|
142
|
+
# Returns the "Table" this record belongs to. SimpleDB has no concept of
|
143
|
+
# tables, but we fake it with metadata.
|
144
|
+
def table
|
145
|
+
Table.name_from_metadata(metadata) ||
|
146
|
+
storage_name
|
147
|
+
end
|
148
|
+
|
149
|
+
def metadata
|
150
|
+
simpledb_attributes[METADATA_KEY]
|
151
|
+
end
|
152
|
+
|
153
|
+
# Returns a record of the current version
|
154
|
+
def migrate
|
155
|
+
new_record = Record.versions[CURRENT_VERSION].allocate
|
156
|
+
new_record.item_name = item_name
|
157
|
+
data = transform_hash(simpledb_attributes) {
|
158
|
+
|hash, key, values|
|
159
|
+
hash[key] = coerce_heuristically(values)
|
160
|
+
}
|
161
|
+
updates = {}
|
162
|
+
deletes = []
|
163
|
+
data.each_pair do |key, values|
|
164
|
+
if Array(values).empty?
|
165
|
+
deletes << key
|
166
|
+
else
|
167
|
+
updates[key] = values
|
168
|
+
end
|
169
|
+
end
|
170
|
+
new_record.add_metadata_to!(updates, table)
|
171
|
+
new_record.simpledb_attributes = updates
|
172
|
+
new_record.deletable_attributes = deletes
|
173
|
+
new_record
|
174
|
+
end
|
175
|
+
|
176
|
+
def add_metadata_to!(hash, table_name)
|
177
|
+
hash.merge!({
|
178
|
+
STORAGE_NAME_KEY => [table_name],
|
179
|
+
METADATA_KEY => [version_token, Table.token_for(table_name)]
|
180
|
+
})
|
181
|
+
end
|
182
|
+
|
183
|
+
protected
|
184
|
+
|
185
|
+
attr_writer :item_name
|
186
|
+
attr_writer :simpledb_attributes
|
187
|
+
attr_writer :deletable_attributes
|
188
|
+
|
189
|
+
private
|
190
|
+
|
191
|
+
def app_data
|
192
|
+
transform_hash(simpledb_attributes) {|h,k,v|
|
193
|
+
h[k] = v unless META_KEYS.include?(k)
|
194
|
+
}
|
195
|
+
end
|
196
|
+
|
197
|
+
def extract_attributes(resource)
|
198
|
+
attributes = resource.attributes(:property)
|
199
|
+
attributes = attributes.to_a.map {|a| [a.first.name.to_s, a.last]}.to_hash
|
200
|
+
attributes = adjust_to_sdb_attributes(attributes)
|
201
|
+
updates, deletes = attributes.partition{|name,value|
|
202
|
+
!Array(value).empty?
|
203
|
+
}
|
204
|
+
attrs_to_update = updates.inject({}){|h, (k,v)| h[k] = v; h}
|
205
|
+
table = Table.new(resource.model)
|
206
|
+
if resource.new?
|
207
|
+
add_metadata_to!(attrs_to_update, table.simpledb_type)
|
208
|
+
end
|
209
|
+
attrs_to_delete = deletes.inject({}){|h, (k,v)| h[k] = v; h}.keys
|
210
|
+
[attrs_to_update, attrs_to_delete]
|
211
|
+
end
|
212
|
+
|
213
|
+
# hack for converting and storing strings longer than 1024 one thing to
|
214
|
+
# note if you use string longer than 1019 chars you will loose the ability
|
215
|
+
# to do full text matching on queries as the string can be broken at any
|
216
|
+
# place during chunking
|
217
|
+
def adjust_to_sdb_attributes(attrs)
|
218
|
+
attrs = transform_hash(attrs) do |result, key, value|
|
219
|
+
if primitive_value_of(value.class) <= String
|
220
|
+
result[key] = ChunkedString.new(value).to_a
|
221
|
+
elsif value.class == Object # This is for SdbArray
|
222
|
+
result[key] = value.to_ary
|
223
|
+
elsif primitive_value_of(value.class) <= Array
|
224
|
+
result[key] = value
|
225
|
+
elsif value.nil?
|
226
|
+
result[key] = nil
|
227
|
+
else
|
228
|
+
result[key] = [value.to_s]
|
229
|
+
end
|
230
|
+
end
|
231
|
+
# Stringify keys
|
232
|
+
transform_hash(attrs) {|h, k, v| h[k.to_s] = v}
|
233
|
+
end
|
234
|
+
|
235
|
+
def primitive_value_of(type)
|
236
|
+
if type < DataMapper::Type
|
237
|
+
type.primitive
|
238
|
+
else
|
239
|
+
type
|
240
|
+
end
|
241
|
+
end
|
242
|
+
|
243
|
+
# Creates an item name for a resource
|
244
|
+
def item_name_for_resource(resource)
|
245
|
+
table = Table.new(resource.model)
|
246
|
+
sdb_type = table.simpledb_type
|
247
|
+
|
248
|
+
item_name = "#{sdb_type}+"
|
249
|
+
keys = table.keys_for_model
|
250
|
+
item_name += keys.map do |property|
|
251
|
+
property.get(resource)
|
252
|
+
end.join('-')
|
253
|
+
|
254
|
+
Digest::SHA1.hexdigest(item_name)
|
255
|
+
end
|
256
|
+
|
257
|
+
def coerce_heuristically(values)
|
258
|
+
if values
|
259
|
+
case values.size
|
260
|
+
when 0
|
261
|
+
values
|
262
|
+
when 1
|
263
|
+
value = coerce_to_type(values, String)
|
264
|
+
value.nil? ? [] : [value]
|
265
|
+
else
|
266
|
+
if ChunkedString.valid?(values)
|
267
|
+
string = ChunkedString.new(values)
|
268
|
+
coerced_string = coerce_to_type([string], Array).first
|
269
|
+
ChunkedString.new(coerced_string).to_a
|
270
|
+
else
|
271
|
+
coerce_to_type(values, Array)
|
272
|
+
end
|
273
|
+
end
|
274
|
+
else
|
275
|
+
[]
|
276
|
+
end
|
277
|
+
end
|
278
|
+
|
279
|
+
end
|
280
|
+
|
281
|
+
# Version 0 records are records that have no associated version
|
282
|
+
# metadata. Any records created by versions of the DataMapper/SimplDB adapter
|
283
|
+
# prior to 1.1.0 are considered to be version 0.
|
284
|
+
#
|
285
|
+
# Version 0 records have a few distinguishing characteristics:
|
286
|
+
# * An attribute with the token "nil" as its sole member is treated as a
|
287
|
+
# null/empty attribute.
|
288
|
+
# * The token "[[[NEWLINE]]]" inside of String attributes is replaced with \n
|
289
|
+
class RecordV0 < Record
|
290
|
+
version "00.00.00"
|
291
|
+
|
292
|
+
def coerce_to_type(values, type)
|
293
|
+
values = values.map{|v| replace_newline_placeholders(v)}
|
294
|
+
result = super(values, type)
|
295
|
+
|
296
|
+
if result == "nil"
|
297
|
+
nil
|
298
|
+
elsif result == ["nil"]
|
299
|
+
[]
|
300
|
+
elsif result && type <= String
|
301
|
+
# TODO redundant
|
302
|
+
replace_newline_placeholders(result)
|
303
|
+
else
|
304
|
+
result
|
305
|
+
end
|
306
|
+
end
|
307
|
+
|
308
|
+
private
|
309
|
+
|
310
|
+
def replace_newline_placeholders(value)
|
311
|
+
value.gsub("[[[NEWLINE]]]", "\n")
|
312
|
+
end
|
313
|
+
end
|
314
|
+
|
315
|
+
class RecordV1_1 < Record
|
316
|
+
version "01.01.00"
|
317
|
+
end
|
318
|
+
end
|