dm-adapter-simpledb 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. data/.gitignore +1 -0
  2. data/History.txt +21 -0
  3. data/README +21 -8
  4. data/Rakefile +35 -23
  5. data/VERSION +1 -1
  6. data/dm-adapter-simpledb.gemspec +44 -24
  7. data/lib/dm-adapter-simpledb.rb +17 -0
  8. data/lib/dm-adapter-simpledb/adapters/simpledb_adapter.rb +339 -0
  9. data/lib/dm-adapter-simpledb/chunked_string.rb +54 -0
  10. data/lib/dm-adapter-simpledb/migrations/simpledb_adapter.rb +45 -0
  11. data/lib/dm-adapter-simpledb/rake.rb +43 -0
  12. data/lib/dm-adapter-simpledb/record.rb +318 -0
  13. data/lib/{simpledb_adapter → dm-adapter-simpledb}/sdb_array.rb +0 -0
  14. data/lib/dm-adapter-simpledb/table.rb +40 -0
  15. data/lib/dm-adapter-simpledb/utils.rb +15 -0
  16. data/lib/simpledb_adapter.rb +2 -469
  17. data/scripts/simple_benchmark.rb +1 -1
  18. data/spec/{associations_spec.rb → integration/associations_spec.rb} +0 -0
  19. data/spec/{compliance_spec.rb → integration/compliance_spec.rb} +0 -0
  20. data/spec/{date_spec.rb → integration/date_spec.rb} +0 -0
  21. data/spec/{limit_and_order_spec.rb → integration/limit_and_order_spec.rb} +0 -0
  22. data/spec/{migrations_spec.rb → integration/migrations_spec.rb} +0 -0
  23. data/spec/{multiple_records_spec.rb → integration/multiple_records_spec.rb} +0 -0
  24. data/spec/{nils_spec.rb → integration/nils_spec.rb} +0 -0
  25. data/spec/{sdb_array_spec.rb → integration/sdb_array_spec.rb} +4 -5
  26. data/spec/{simpledb_adapter_spec.rb → integration/simpledb_adapter_spec.rb} +65 -0
  27. data/spec/{spec_helper.rb → integration/spec_helper.rb} +8 -3
  28. data/spec/unit/record_spec.rb +346 -0
  29. data/spec/unit/simpledb_adapter_spec.rb +80 -0
  30. data/spec/unit/unit_spec_helper.rb +26 -0
  31. metadata +58 -24
  32. data/tasks/devver.rake +0 -167
@@ -1,5 +1,5 @@
1
1
  require 'pathname'
2
- require Pathname(__FILE__).dirname.parent.expand_path + 'lib/simpledb_adapter'
2
+ require Pathname(__FILE__).dirname.parent.expand_path + 'lib/dm-adapter-simpledb'
3
3
  require 'ruby-debug'
4
4
  require 'benchmark'
5
5
 
@@ -1,14 +1,13 @@
1
1
  require 'pathname'
2
2
  require Pathname(__FILE__).dirname.expand_path + 'spec_helper'
3
- require Pathname(__FILE__).dirname.expand_path + '../lib/simpledb_adapter/sdb_array'
4
- require 'spec/autorun'
3
+ require 'dm-adapter-simpledb/sdb_array'
5
4
 
6
5
  describe 'with multiple records saved' do
7
6
 
8
7
  class Hobbyist
9
8
  include DataMapper::Resource
10
9
  property :name, String, :key => true
11
- property :hobbies, SdbArray
10
+ property :hobbies, SdbArray
12
11
  end
13
12
 
14
13
  before(:each) do
@@ -35,7 +34,7 @@ describe 'with multiple records saved' do
35
34
  person.save
36
35
  @adapter.wait_for_consistency
37
36
  lego_person = Hobbyist.first(:name => 'Jeremy Boles')
38
- lego_person.hobbies.should == "lego"
37
+ lego_person.hobbies.should == ["lego"]
39
38
  end
40
39
 
41
40
  it 'should allow deletion of array' do
@@ -44,7 +43,7 @@ describe 'with multiple records saved' do
44
43
  person.save
45
44
  @adapter.wait_for_consistency
46
45
  lego_person = Hobbyist.first(:name => 'Jeremy Boles')
47
- lego_person.hobbies.should == nil
46
+ lego_person.hobbies.should == []
48
47
  end
49
48
 
50
49
  it 'should find all records with diving hobby' do
@@ -28,6 +28,15 @@ end
28
28
 
29
29
  describe DataMapper::Adapters::SimpleDBAdapter do
30
30
 
31
+ class Project
32
+ include DataMapper::Resource
33
+ property :id, Integer, :key => true
34
+ property :project_repo, String
35
+ property :repo_user, String
36
+ property :description, String
37
+ end
38
+
39
+
31
40
  LONG_VALUE =<<-EOF
32
41
  #!/bin/sh
33
42
 
@@ -159,4 +168,60 @@ EOF
159
168
  end
160
169
  end
161
170
  end
171
+
172
+ context "given a pre-existing v0 record" do
173
+ before :each do
174
+ @record_name = "33d9e5a6fcbd746dc40904a6766d4166e14305fe"
175
+ record_attributes = {
176
+ "simpledb_type" => ["projects"],
177
+ "project_repo" => ["git://github.com/TwP/servolux.git"],
178
+ "files_complete" => ["nil"],
179
+ "repo_user" => ["nil"],
180
+ "id" => ["1077338529"],
181
+ "description" => [
182
+ "0002:line 2[[[NEWLINE]]]line 3[[[NEW",
183
+ "0001:line 1[[[NEWLINE]]]",
184
+ "0003:LINE]]]line 4"
185
+ ]
186
+ }
187
+ @sdb.put_attributes(@domain, @record_name, record_attributes)
188
+ sleep 0.4
189
+ @record = Project.get(1077338529)
190
+ end
191
+
192
+ it "should interpret legacy nil values correctly" do
193
+ @record.repo_user.should be_nil
194
+ end
195
+
196
+ it "should interpret legacy strings correctly" do
197
+ @record.description.should ==
198
+ "line 1\nline 2\nline 3\nline 4"
199
+ end
200
+
201
+ it "should save legacy records without adding new metadata" do
202
+ @record.repo_user = "steve"
203
+ @record.save
204
+ sleep 0.4
205
+ attributes = @sdb.get_attributes(@domain, @record_name)[:attributes]
206
+ attributes.should_not include("__dm_metadata")
207
+ end
208
+ end
209
+
210
+ describe "given a brand-new record" do
211
+ before :each do
212
+ @record = Project.new(
213
+ :repo_user => "steve",
214
+ :id => 123,
215
+ :project_repo => "git://example.org/foo")
216
+ end
217
+
218
+ it "should add metadata to the record on save" do
219
+ @record.save
220
+ sleep 0.4
221
+ items = @sdb.select("select * from #{@domain} where id = '123'")[:items]
222
+ attributes = items.first.values.first
223
+ attributes["__dm_metadata"].should include("v01.01.00")
224
+ attributes["__dm_metadata"].should include("table:projects")
225
+ end
226
+ end
162
227
  end
@@ -1,8 +1,11 @@
1
1
  require 'pathname'
2
- require Pathname(__FILE__).dirname.parent.expand_path + 'lib/simpledb_adapter'
2
+ ROOT = File.expand_path('../..', File.dirname(__FILE__))
3
+ $LOAD_PATH.unshift(File.join(ROOT,'lib'))
4
+ require 'simpledb_adapter'
3
5
  require 'logger'
4
6
  require 'fileutils'
5
7
  require 'spec'
8
+ require 'spec/autorun'
6
9
 
7
10
  DOMAIN_FILE_MESSAGE = <<END
8
11
  !!! ATTENTION !!!
@@ -20,7 +23,7 @@ END
20
23
  Spec::Runner.configure do |config|
21
24
  access_key = ENV['AMAZON_ACCESS_KEY_ID']
22
25
  secret_key = ENV['AMAZON_SECRET_ACCESS_KEY']
23
- domain_file = File.expand_path('../THROW_AWAY_SDB_DOMAIN', File.dirname(__FILE__))
26
+ domain_file = File.join(ROOT, 'THROW_AWAY_SDB_DOMAIN')
24
27
  test_domain = if File.exist?(domain_file)
25
28
  File.read(domain_file).strip
26
29
  else
@@ -30,7 +33,7 @@ Spec::Runner.configure do |config|
30
33
 
31
34
  #For those that don't like to mess up their ENV
32
35
  if access_key==nil && secret_key==nil
33
- lines = File.readlines(File.join(File.dirname(__FILE__),'..','aws_config'))
36
+ lines = File.readlines(File.join(ROOT, 'aws_config'))
34
37
  access_key = lines[0].strip
35
38
  secret_key = lines[1].strip
36
39
  end
@@ -41,6 +44,8 @@ Spec::Runner.configure do |config|
41
44
  log_file = "log/dm-sdb.log"
42
45
  FileUtils.touch(log_file)
43
46
  log = Logger.new(log_file)
47
+ log.level = ::Logger::DEBUG
48
+ DataMapper.logger.level = :debug
44
49
 
45
50
  $control_sdb ||= RightAws::SdbInterface.new(
46
51
  access_key, secret_key, :domain => test_domain)
@@ -0,0 +1,346 @@
1
+ require File.expand_path('unit_spec_helper', File.dirname(__FILE__))
2
+ require 'dm-adapter-simpledb/record'
3
+ require 'dm-adapter-simpledb/sdb_array'
4
+
5
+ describe DmAdapterSimpledb::Record do
6
+
7
+
8
+ context "given a record from SimpleDB" do
9
+ before :each do
10
+ @thing_class = Class.new do
11
+ include DataMapper::Resource
12
+
13
+ property :foo, Integer
14
+ end
15
+ @it = DmAdapterSimpledb::Record.from_simpledb_hash(
16
+ {"KEY" => {
17
+ "foo" => ["123"],
18
+ "baz" => ["456"],
19
+ "simpledb_type" => ["thingies"]
20
+ }
21
+ })
22
+ end
23
+
24
+ it "should return nil when asked for a non-existant attribute as String" do
25
+ @it["bar", String].should be_nil
26
+ end
27
+
28
+ it "should return nil when asked for a non-existant attribute as Integer" do
29
+ @it["bar", Integer].should be_nil
30
+ end
31
+
32
+ it "should return [] when asked for a non-existant attribute as Array" do
33
+ @it["bar", Array].should == []
34
+ end
35
+
36
+ it "should be able to coerce based on a property" do
37
+ @it["foo", @thing_class.properties[:foo]].should == 123
38
+ end
39
+
40
+ it "should be able to coerce based on a simple type" do
41
+ @it["foo", Integer].should == "123"
42
+ end
43
+
44
+ context "converted to a resource hash" do
45
+ before :each do
46
+ @hash = @it.to_resource_hash(@thing_class.properties)
47
+ end
48
+
49
+ it "should only include properties specified in the field set" do
50
+ @hash.should_not include(:bar)
51
+ end
52
+ end
53
+
54
+ end
55
+
56
+
57
+ context "given a record with no version info" do
58
+ before :each do
59
+ @resource_class = Class.new do
60
+ include DataMapper::Resource
61
+
62
+ property :foo, Integer, :key => true
63
+ end
64
+
65
+ @it = DmAdapterSimpledb::Record.from_simpledb_hash(
66
+ {"KEY" => {
67
+ "foo" => ["123"],
68
+ "text" => [
69
+ "0001:line 1[[[NEWLINE]]]line 2",
70
+ "0002:[[[NEWLINE]]]line 3[[[NEW",
71
+ "0003:LINE]]]line 4"
72
+ ],
73
+ "short_text" => ["foo[[[NEWLINE]]]bar"],
74
+ "simpledb_type" => ["thingies"],
75
+ "null_field" => ["nil"],
76
+ "array" => ["foo[[[NEWLINE]]]bar", "baz"]
77
+ }
78
+ })
79
+ end
80
+
81
+ it "should identify the record as version 0" do
82
+ @it.version.should == "00.00.00"
83
+ end
84
+
85
+ it "should be able to convert the record to a DM-friendly hash" do
86
+ @it.to_resource_hash(@resource_class.properties).should == {
87
+ "foo" => 123,
88
+ }
89
+ end
90
+
91
+ it "should be able to extract the storage name" do
92
+ @it.storage_name.should == "thingies"
93
+ end
94
+
95
+ it "should subtitute newlines for newline placeholders" do
96
+ @it["text", String].should ==
97
+ "line 1\nline 2\nline 3\nline 4"
98
+ end
99
+
100
+ it "should identify the table from the simpledb_type attributes" do
101
+ @it.table.should == "thingies"
102
+ end
103
+
104
+ it "should not write any V1.1 metadata" do
105
+ @it.writable_attributes.should_not include("__dm_metadata")
106
+ end
107
+
108
+ it "should interpret ['nil'] as the null value" do
109
+ @it["null_field", String].should == nil
110
+ @it["null_field", Array].should == []
111
+ end
112
+
113
+ describe "migrated to latest version" do
114
+ before :each do
115
+ @it = @it.migrate
116
+ end
117
+
118
+ it "should be the latest version" do
119
+ @it.version.should == "01.01.00"
120
+ end
121
+
122
+ it "should mark nil attributes as deletable" do
123
+ @it.writable_attributes.should_not include("null_field")
124
+ @it.deletable_attributes.should include("null_field")
125
+ end
126
+
127
+ it "should contain valued attributes" do
128
+ @it.writable_attributes["foo"].should == ["123"]
129
+ @it.writable_attributes["text"].should ==
130
+ ["line 1\nline 2\nline 3\nline 4"]
131
+ @it.writable_attributes["short_text"].should ==
132
+ ["foo\nbar"]
133
+ @it.writable_attributes["array"].should ==
134
+ ["foo\nbar", "baz"]
135
+ end
136
+
137
+
138
+ it "should have writable metadata attributes" do
139
+ @it.writable_attributes["__dm_metadata"].should include("v01.01.00")
140
+ @it.writable_attributes["__dm_metadata"].should include("table:thingies")
141
+ @it.writable_attributes["simpledb_type"].should == ["thingies"]
142
+ end
143
+ end
144
+ end
145
+
146
+ context "given a version 1.1 record" do
147
+ before :each do
148
+ @resource_class = Class.new do
149
+ include DataMapper::Resource
150
+
151
+ property :bar, Integer
152
+ end
153
+
154
+ @it = DmAdapterSimpledb::Record.from_simpledb_hash(
155
+ {"KEY" => {
156
+ "__dm_metadata" => ["v01.01.00", "table:mystuff"],
157
+ "bar" => ["456"],
158
+ "text" => [
159
+ "0001:line 1[[[NEWLINE]]]line 2",
160
+ "0002:line 3[[[NEW",
161
+ "0003:LINE]]]line 4"
162
+ ],
163
+ }
164
+ })
165
+ end
166
+
167
+ it "should be a V1 record" do
168
+ @it.should be_a_kind_of(DmAdapterSimpledb::RecordV1_1)
169
+ end
170
+
171
+ it "should identify the record as version 1" do
172
+ @it.version.should == "01.01.00"
173
+ end
174
+
175
+ it "should be able to convert the record to a DM-friendly hash" do
176
+ @it.to_resource_hash(@resource_class.properties).should == {
177
+ "bar" => 456
178
+ }
179
+ end
180
+
181
+ it "should not substitute newline tokens" do
182
+ @it["text", String].should ==
183
+ "line 1[[[NEWLINE]]]line 2line 3[[[NEWLINE]]]line 4"
184
+ end
185
+
186
+ it "should find the table given in the metadata" do
187
+ @it.table.should == "mystuff"
188
+ end
189
+ end
190
+
191
+ context "given a V1.1 record with a chunked string" do
192
+ class Poem
193
+ include ::DataMapper::Resource
194
+ property :text, String
195
+ end
196
+
197
+ before :each do
198
+ @it = DmAdapterSimpledb::Record.from_simpledb_hash(
199
+ {"KEY" => {
200
+ "__dm_metadata" => ["v01.01.00"],
201
+ "text" => [
202
+ "0002:did gyre and gimbal in the wabe",
203
+ "0001:twas brillig and the slithy toves\n",
204
+ ]
205
+ }
206
+ })
207
+ end
208
+
209
+ it "should unchunk the text when asked to read it as a String" do
210
+ @it["text",String].should == "twas brillig and the slithy toves\n" +
211
+ "did gyre and gimbal in the wabe"
212
+ end
213
+
214
+ it "should return the chunks when asked to read it as an Array" do
215
+ @it["text",Array].should == [
216
+ "0002:did gyre and gimbal in the wabe",
217
+ "0001:twas brillig and the slithy toves\n",
218
+ ]
219
+ end
220
+
221
+ it "should return the first chunk when asked to read it as anything else" do
222
+ @it["text", Integer].should == "0002:did gyre and gimbal in the wabe"
223
+ end
224
+
225
+ it "should be able to construct a resource hash" do
226
+ @it.to_resource_hash(Poem.properties).should == {
227
+ "text" => "twas brillig and the slithy toves\ndid gyre and gimbal in the wabe"
228
+ }
229
+ end
230
+
231
+ end
232
+
233
+ describe "given an unsaved (new) datamapper resource" do
234
+ before :each do
235
+ @resource_class = Class.new do
236
+ include DataMapper::Resource
237
+ storage_names[:default] = "books"
238
+
239
+ property :author, String, :key => true
240
+ property :date, Date
241
+ property :text, DataMapper::Types::Text
242
+ property :tags, DataMapper::Types::SdbArray
243
+ property :isbn, String
244
+ end
245
+ @text = "lorem ipsum\n" * 100
246
+ @date = Date.new(2001,1,1)
247
+ @author = "Cicero"
248
+ @resource = @resource_class.new(
249
+ :text => @text,
250
+ :date => @date,
251
+ :author => @author,
252
+ :tags => ['latin', 'classic'],
253
+ :isbn => nil)
254
+
255
+ @it = DmAdapterSimpledb::Record.from_resource(@resource)
256
+ end
257
+
258
+ it "should be able to generate an item name" do
259
+ @it.item_name.should ==
260
+ Digest::SHA1.hexdigest("books+Cicero")
261
+ end
262
+
263
+ context "as a SimpleDB hash" do
264
+ before :each do
265
+ @hash = @it.writable_attributes
266
+ @deletes = @it.deletable_attributes
267
+ end
268
+
269
+ it "should translate primitives successfully" do
270
+ @hash["author"].should == ["Cicero"]
271
+ @hash["date"].should == ["2001-01-01"]
272
+ end
273
+
274
+ it "should chunk large text sections" do
275
+ @hash["text"].should have(2).chunks
276
+ end
277
+
278
+ it "should be able to round-trip the text it chunks" do
279
+ DmAdapterSimpledb::Record.from_simpledb_hash({"NAME" => @hash})["text", String].should ==
280
+ @text
281
+ end
282
+
283
+ it "should translate arrays properly" do
284
+ @hash["tags"].should == ['latin', 'classic']
285
+ end
286
+
287
+ it "should be able to round-trip arrays" do
288
+ DmAdapterSimpledb::Record.from_simpledb_hash({"NAME" => @hash})["tags", DataMapper::Types::SdbArray].should ==
289
+ ['latin', 'classic']
290
+ end
291
+
292
+ it "should not include nil values in writable attributes" do
293
+ @hash.should_not include("isbn")
294
+ end
295
+
296
+ it "should include resource type in writable attributes" do
297
+ @hash["simpledb_type"].should == ["books"]
298
+ end
299
+
300
+ it "should include nil values in deleteable attributes" do
301
+ @deletes.should include("isbn")
302
+ end
303
+
304
+ it "should include version in writable attributes" do
305
+ @hash["__dm_metadata"].should include("v01.01.00")
306
+ end
307
+
308
+ it "should include type in writable attributes" do
309
+ @hash["__dm_metadata"].should include("table:books")
310
+ end
311
+
312
+ end
313
+ end
314
+
315
+ describe "given a saved datamapper resource" do
316
+ before :each do
317
+ @resource_class = Class.new do
318
+ include DataMapper::Resource
319
+ storage_names[:default] = "books"
320
+
321
+ property :author, String, :key => true
322
+ property :date, Date
323
+ property :text, DataMapper::Types::Text
324
+ property :tags, DataMapper::Types::SdbArray
325
+ property :isbn, String
326
+ end
327
+ @date = Date.new(2001,1,1)
328
+ @author = "Cicero"
329
+ @resource = @resource_class.new(
330
+ :text => "",
331
+ :date => @date,
332
+ :author => @author,
333
+ :tags => ['latin', 'classic'],
334
+ :isbn => nil)
335
+ @resource.stub!(:saved? => true)
336
+ @resource.stub!(:new? => false)
337
+
338
+ @it = DmAdapterSimpledb::Record.from_resource(@resource)
339
+ end
340
+
341
+ it "should not include metadata in writable attributes" do
342
+ @it.writable_attributes.should_not include("__dm_metadata")
343
+ end
344
+ end
345
+
346
+ end