dm-adapter-simpledb 1.0.0 → 1.1.0

data/scripts/simple_benchmark.rb

@@ -1,5 +1,5 @@
 require 'pathname'
-require Pathname(__FILE__).dirname.parent.expand_path + 'lib/simpledb_adapter'
+require Pathname(__FILE__).dirname.parent.expand_path + 'lib/dm-adapter-simpledb'
 require 'ruby-debug'
 require 'benchmark'
 

data/spec/{sdb_array_spec.rb → integration/sdb_array_spec.rb}

@@ -1,14 +1,13 @@
 require 'pathname'
 require Pathname(__FILE__).dirname.expand_path + 'spec_helper'
-require Pathname(__FILE__).dirname.expand_path + '../lib/simpledb_adapter/sdb_array'
-require 'spec/autorun'
+require 'dm-adapter-simpledb/sdb_array'
 
 describe 'with multiple records saved' do
   
   class Hobbyist
     include DataMapper::Resource
     property :name,       String, :key => true
-    property :hobbies,     SdbArray
+    property :hobbies,    SdbArray
   end
   
   before(:each) do
@@ -35,7 +34,7 @@ describe 'with multiple records saved' do
     person.save
     @adapter.wait_for_consistency
     lego_person = Hobbyist.first(:name => 'Jeremy Boles')
-    lego_person.hobbies.should == "lego"
+    lego_person.hobbies.should == ["lego"]
   end
   
   it 'should allow deletion of array' do
@@ -44,7 +43,7 @@ describe 'with multiple records saved' do
     person.save
     @adapter.wait_for_consistency
     lego_person = Hobbyist.first(:name => 'Jeremy Boles')
-    lego_person.hobbies.should == nil
+    lego_person.hobbies.should == []
   end
   
   it 'should find all records with diving hobby' do

data/spec/{simpledb_adapter_spec.rb → integration/simpledb_adapter_spec.rb}

@@ -28,6 +28,15 @@ end
 
 describe DataMapper::Adapters::SimpleDBAdapter do
 
+  class Project
+    include DataMapper::Resource
+    property :id, Integer, :key => true
+    property :project_repo, String
+    property :repo_user, String
+    property :description, String
+  end
+
+
   LONG_VALUE =<<-EOF
     #!/bin/sh
 
@@ -159,4 +168,60 @@ EOF
       end
     end
   end
+
+  context "given a pre-existing v0 record" do
+    before :each do
+      @record_name = "33d9e5a6fcbd746dc40904a6766d4166e14305fe"
+      record_attributes = {
+        "simpledb_type"  => ["projects"], 
+        "project_repo"   => ["git://github.com/TwP/servolux.git"], 
+        "files_complete" => ["nil"], 
+        "repo_user"      => ["nil"], 
+        "id"             => ["1077338529"], 
+        "description"    => [
+            "0002:line 2[[[NEWLINE]]]line 3[[[NEW",
+            "0001:line 1[[[NEWLINE]]]",
+            "0003:LINE]]]line 4"
+          ]
+      }
+      @sdb.put_attributes(@domain, @record_name, record_attributes)
+      sleep 0.4
+      @record = Project.get(1077338529)
+    end
+
+    it "should interpret legacy nil values correctly" do
+      @record.repo_user.should be_nil
+    end
+
+    it "should interpret legacy strings correctly" do
+      @record.description.should ==
+        "line 1\nline 2\nline 3\nline 4"
+    end
+
+    it "should save legacy records without adding new metadata" do
+      @record.repo_user = "steve"
+      @record.save
+      sleep 0.4
+      attributes = @sdb.get_attributes(@domain, @record_name)[:attributes]
+      attributes.should_not include("__dm_metadata")
+    end
+  end
+
+  describe "given a brand-new record" do
+    before :each do
+      @record = Project.new(
+        :repo_user    => "steve", 
+        :id           => 123, 
+        :project_repo => "git://example.org/foo")
+    end
+
+    it "should add metadata to the record on save" do
+      @record.save
+      sleep 0.4
+      items = @sdb.select("select * from #{@domain} where id = '123'")[:items]
+      attributes = items.first.values.first
+      attributes["__dm_metadata"].should include("v01.01.00")
+      attributes["__dm_metadata"].should include("table:projects")
+    end
+  end
 end

data/spec/{spec_helper.rb → integration/spec_helper.rb}

@@ -1,8 +1,11 @@
 require 'pathname'
-require Pathname(__FILE__).dirname.parent.expand_path + 'lib/simpledb_adapter'
+ROOT = File.expand_path('../..', File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(ROOT,'lib'))
+require 'simpledb_adapter'
 require 'logger'
 require 'fileutils'
 require 'spec'
+require 'spec/autorun'
 
 DOMAIN_FILE_MESSAGE = <<END
 !!! ATTENTION !!!
@@ -20,7 +23,7 @@ END
 Spec::Runner.configure do |config|
   access_key  = ENV['AMAZON_ACCESS_KEY_ID']
   secret_key  = ENV['AMAZON_SECRET_ACCESS_KEY']
-  domain_file = File.expand_path('../THROW_AWAY_SDB_DOMAIN', File.dirname(__FILE__))
+  domain_file = File.join(ROOT, 'THROW_AWAY_SDB_DOMAIN')
   test_domain = if File.exist?(domain_file)
                   File.read(domain_file).strip
                 else
@@ -30,7 +33,7 @@ Spec::Runner.configure do |config|
 
   #For those that don't like to mess up their ENV
   if access_key==nil && secret_key==nil
-    lines = File.readlines(File.join(File.dirname(__FILE__),'..','aws_config'))
+    lines = File.readlines(File.join(ROOT, 'aws_config'))
     access_key = lines[0].strip
     secret_key = lines[1].strip
   end
@@ -41,6 +44,8 @@ Spec::Runner.configure do |config|
     log_file = "log/dm-sdb.log"
     FileUtils.touch(log_file)
     log = Logger.new(log_file)
+    log.level = ::Logger::DEBUG
+    DataMapper.logger.level = :debug
 
     $control_sdb ||= RightAws::SdbInterface.new(
       access_key, secret_key, :domain => test_domain)

data/spec/unit/record_spec.rb

@@ -0,0 +1,346 @@
+require File.expand_path('unit_spec_helper', File.dirname(__FILE__))
+require 'dm-adapter-simpledb/record'
+require 'dm-adapter-simpledb/sdb_array'
+
+describe DmAdapterSimpledb::Record do
+
+
+  context "given a record from SimpleDB" do
+    before :each do
+      @thing_class = Class.new do
+        include DataMapper::Resource
+
+        property :foo, Integer
+      end
+      @it = DmAdapterSimpledb::Record.from_simpledb_hash(
+        {"KEY" => {
+            "foo"           => ["123"],
+            "baz"           => ["456"],
+            "simpledb_type" => ["thingies"]
+          }
+        })
+    end
+
+    it "should return nil when asked for a non-existant attribute as String" do
+      @it["bar", String].should be_nil
+    end
+
+    it "should return nil when asked for a non-existant attribute as Integer" do
+      @it["bar", Integer].should be_nil
+    end
+
+    it "should return [] when asked for a non-existant attribute as Array" do
+      @it["bar", Array].should == []
+    end
+
+    it "should be able to coerce based on a property" do
+      @it["foo", @thing_class.properties[:foo]].should == 123
+    end
+
+    it "should be able to coerce based on a simple type" do
+      @it["foo", Integer].should == "123"
+    end
+
+    context "converted to a resource hash" do
+      before :each do
+        @hash = @it.to_resource_hash(@thing_class.properties)
+      end
+
+      it "should only include properties specified in the field set" do
+        @hash.should_not include(:bar)
+      end
+    end
+
+  end
+
+
+  context "given a record with no version info" do
+    before :each do
+      @resource_class = Class.new do
+        include DataMapper::Resource
+
+        property :foo, Integer, :key => true
+      end
+
+      @it = DmAdapterSimpledb::Record.from_simpledb_hash(
+        {"KEY" => {
+            "foo"           => ["123"],
+            "text"          => [
+              "0001:line 1[[[NEWLINE]]]line 2",
+              "0002:[[[NEWLINE]]]line 3[[[NEW",
+              "0003:LINE]]]line 4"
+            ],
+            "short_text"    => ["foo[[[NEWLINE]]]bar"],
+            "simpledb_type" => ["thingies"],
+            "null_field"    => ["nil"],
+            "array"         => ["foo[[[NEWLINE]]]bar", "baz"]
+          }
+        })
+    end
+
+    it "should identify the record as version 0" do
+      @it.version.should == "00.00.00"
+    end
+
+    it "should be able to convert the record to a DM-friendly hash" do
+      @it.to_resource_hash(@resource_class.properties).should == {
+        "foo" => 123,
+      }
+    end
+
+    it "should be able to extract the storage name" do
+      @it.storage_name.should == "thingies"
+    end
+
+    it "should subtitute newlines for newline placeholders" do
+      @it["text", String].should == 
+        "line 1\nline 2\nline 3\nline 4"
+    end
+
+    it "should identify the table from the simpledb_type attributes" do
+      @it.table.should == "thingies"
+    end
+
+    it "should not write any V1.1 metadata" do
+      @it.writable_attributes.should_not include("__dm_metadata")
+    end
+
+    it "should interpret ['nil'] as the null value" do
+      @it["null_field", String].should == nil
+      @it["null_field", Array].should == []
+    end
+
+    describe "migrated to latest version" do
+      before :each do
+        @it = @it.migrate
+      end
+
+      it "should be the latest version" do
+        @it.version.should == "01.01.00"
+      end
+
+      it "should mark nil attributes as deletable" do
+        @it.writable_attributes.should_not include("null_field")
+        @it.deletable_attributes.should include("null_field")
+      end
+
+      it "should contain valued attributes" do
+        @it.writable_attributes["foo"].should == ["123"]
+        @it.writable_attributes["text"].should ==
+          ["line 1\nline 2\nline 3\nline 4"]
+        @it.writable_attributes["short_text"].should == 
+          ["foo\nbar"]
+        @it.writable_attributes["array"].should ==
+          ["foo\nbar", "baz"]
+      end
+
+
+      it "should have writable metadata attributes" do
+        @it.writable_attributes["__dm_metadata"].should include("v01.01.00")
+        @it.writable_attributes["__dm_metadata"].should include("table:thingies")
+        @it.writable_attributes["simpledb_type"].should == ["thingies"]
+      end
+    end
+  end
+
+  context "given a version 1.1 record" do
+    before :each do
+      @resource_class = Class.new do
+        include DataMapper::Resource
+      
+        property :bar, Integer
+      end
+
+      @it = DmAdapterSimpledb::Record.from_simpledb_hash(
+        {"KEY" => {
+            "__dm_metadata" => ["v01.01.00", "table:mystuff"],
+            "bar"           => ["456"],
+            "text"          => [
+              "0001:line 1[[[NEWLINE]]]line 2",
+              "0002:line 3[[[NEW",
+              "0003:LINE]]]line 4"
+            ],
+          }
+        })
+    end
+
+    it "should be a V1 record" do
+      @it.should be_a_kind_of(DmAdapterSimpledb::RecordV1_1)
+    end
+
+    it "should identify the record as version 1" do
+      @it.version.should == "01.01.00"
+    end
+
+    it "should be able to convert the record to a DM-friendly hash" do
+      @it.to_resource_hash(@resource_class.properties).should == {
+        "bar" => 456
+      }
+    end
+
+    it "should not substitute newline tokens" do
+      @it["text", String].should ==
+        "line 1[[[NEWLINE]]]line 2line 3[[[NEWLINE]]]line 4"
+    end
+
+    it "should find the table given in the metadata" do
+      @it.table.should == "mystuff"
+    end
+  end
+
+  context "given a V1.1 record with a chunked string" do
+    class Poem
+      include ::DataMapper::Resource
+      property :text, String
+    end
+
+    before :each do 
+      @it = DmAdapterSimpledb::Record.from_simpledb_hash(
+        {"KEY" => {
+            "__dm_metadata" => ["v01.01.00"],
+            "text" => [
+              "0002:did gyre and gimbal in the wabe",
+              "0001:twas brillig and the slithy toves\n",
+            ]
+          }
+        })
+    end
+
+    it "should unchunk the text when asked to read it as a String" do
+      @it["text",String].should == "twas brillig and the slithy toves\n" +
+        "did gyre and gimbal in the wabe"
+    end
+
+    it "should return the chunks when asked to read it as an Array" do
+      @it["text",Array].should == [
+        "0002:did gyre and gimbal in the wabe",
+        "0001:twas brillig and the slithy toves\n",
+      ]
+    end
+
+    it "should return the first chunk when asked to read it as anything else" do
+      @it["text", Integer].should == "0002:did gyre and gimbal in the wabe"
+    end
+
+    it "should be able to construct a resource hash" do
+      @it.to_resource_hash(Poem.properties).should == {
+        "text" => "twas brillig and the slithy toves\ndid gyre and gimbal in the wabe"
+      }
+    end
+
+  end
+
+  describe "given an unsaved (new) datamapper resource" do
+    before :each do
+      @resource_class = Class.new do
+        include DataMapper::Resource
+        storage_names[:default] = "books"
+
+        property :author,       String, :key => true
+        property :date,         Date
+        property :text,         DataMapper::Types::Text
+        property :tags,         DataMapper::Types::SdbArray
+        property :isbn,         String
+      end
+      @text   = "lorem ipsum\n" * 100
+      @date   = Date.new(2001,1,1)
+      @author = "Cicero"
+      @resource = @resource_class.new(
+        :text => @text,
+        :date => @date,
+        :author => @author,
+        :tags  => ['latin', 'classic'],
+        :isbn  => nil)
+
+      @it = DmAdapterSimpledb::Record.from_resource(@resource)
+    end
+
+    it "should be able to generate an item name" do
+      @it.item_name.should == 
+        Digest::SHA1.hexdigest("books+Cicero")
+    end
+
+    context "as a SimpleDB hash" do
+      before :each do
+        @hash = @it.writable_attributes
+        @deletes = @it.deletable_attributes
+      end
+
+      it "should translate primitives successfully" do
+        @hash["author"].should == ["Cicero"]
+        @hash["date"].should   == ["2001-01-01"]
+      end
+
+      it "should chunk large text sections" do
+        @hash["text"].should have(2).chunks
+      end
+
+      it "should be able to round-trip the text it chunks" do
+        DmAdapterSimpledb::Record.from_simpledb_hash({"NAME" => @hash})["text", String].should ==
+          @text
+      end
+
+      it "should translate arrays properly" do
+        @hash["tags"].should == ['latin', 'classic']
+      end
+
+      it "should be able to round-trip arrays" do
+        DmAdapterSimpledb::Record.from_simpledb_hash({"NAME" => @hash})["tags", DataMapper::Types::SdbArray].should ==
+          ['latin', 'classic']
+      end
+
+      it "should not include nil values in writable attributes" do
+        @hash.should_not include("isbn")
+      end
+
+      it "should include resource type in writable attributes" do
+        @hash["simpledb_type"].should == ["books"]
+      end
+
+      it "should include nil values in deleteable attributes" do
+        @deletes.should include("isbn")
+      end
+
+      it "should include version in writable attributes" do
+        @hash["__dm_metadata"].should include("v01.01.00")
+      end
+
+      it "should include type in writable attributes" do
+        @hash["__dm_metadata"].should include("table:books")
+      end
+
+    end
+  end
+
+  describe "given a saved datamapper resource" do
+    before :each do
+      @resource_class = Class.new do
+        include DataMapper::Resource
+        storage_names[:default] = "books"
+
+        property :author,       String, :key => true
+        property :date,         Date
+        property :text,         DataMapper::Types::Text
+        property :tags,         DataMapper::Types::SdbArray
+        property :isbn,         String
+      end
+      @date   = Date.new(2001,1,1)
+      @author = "Cicero"
+      @resource = @resource_class.new(
+        :text => "",
+        :date => @date,
+        :author => @author,
+        :tags  => ['latin', 'classic'],
+        :isbn  => nil)
+      @resource.stub!(:saved? => true)
+      @resource.stub!(:new? => false)
+
+      @it = DmAdapterSimpledb::Record.from_resource(@resource)
+    end
+
+    it "should not include metadata in writable attributes" do
+      @it.writable_attributes.should_not include("__dm_metadata")
+    end
+  end
+
+end