sdbport 0.1.0 → 0.1.1

data/CHANGELOG

@@ -1,3 +1,9 @@
+## HEAD:
+
+## 0.1.1:
+
+* Read domains with more than 100 items
+
 ## 0.1.0:
 
 * Added export subcommand

data/README.md

@@ -1,3 +1,5 @@
+[![Build Status](https://secure.travis-ci.org/brettweavnet/sdbport.png)](http://travis-ci.org/brettweavnet/sdbport)
+
 # Sdbport
 
 Sdbport exports & imports data from AWS SimpleDB domains. It can be used as a class or stand alone CLI.
@@ -27,12 +29,6 @@ Import into new domain:
 sdbport import -a $AWS_ACCESS_KEY_ID -s $AWS_SECRET_ACCESS_KEY -r us-west-1 -n new-domain -i /tmp/test-domain-dump
 ```
 
-Purge new domain:
-
-```
-sdbport purge -a $AWS_ACCESS_KEY_ID -s $AWS_SECRET_ACCESS_KEY -r us-west-1 -n new-domain
-```
-
 To list CLI subcommands:
 
 ```
@@ -47,8 +43,7 @@ sdbport export -h
 
 ## Known Limitations
 
-* Only performs a single query for exports, which gives it a maxmimum of 1,000 entries.
-* Single serialiazed process.
+* Single serialiazed process for import. Need to switch to batch put attributes.
 * Only supports importing into empty domain.
 
 ## Contributing

data/lib/sdbport/aws/simpledb.rb

@@ -22,8 +22,21 @@ module Sdbport
         sdb.create_domain(domain) unless domain_exists?(domain)
       end
 
-      def select(query)
-        sdb.select(query).body['Items']
+      def select(query, options = {})
+        sdb.select(query, options).body
+      end
+
+      def select_and_follow_tokens(query, options = {})
+        data = {}
+        next_token = nil
+        final_token = false
+        while true
+          options.merge! 'NextToken' => next_token
+          chunk = sdb.select(query, options).body
+          data.merge! chunk['Items']
+          next_token = chunk['NextToken']
+          return data unless next_token
+        end
       end
 
       def count(domain)

data/lib/sdbport/domain/export.rb

@@ -31,7 +31,7 @@ module Sdbport
       end
 
       def export_domain
-        sdb.select "select * from `#{@name}`"
+        sdb.select_and_follow_tokens "select * from `#{@name}`"
       end
 
       def convert_to_string(item)

data/lib/sdbport/domain/purge.rb

@@ -12,7 +12,7 @@ module Sdbport
 
       def purge
         @logger.info "Purging #{@name} in #{@region}."
-        data = sdb.select "select * from `#{@name}`"
+        data = sdb.select_and_follow_tokens "select * from `#{@name}`"
         data.keys.each do |key|
           @logger.debug "Deleting #{key}."
           sdb.delete @name, key

data/lib/sdbport/version.rb

@@ -1,3 +1,3 @@
 module Sdbport
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/spec/aws/simpledb_spec.rb

@@ -4,7 +4,7 @@ describe Sdbport do
 
   before do
     @fog_mock = mock 'fog'
-    @body_stub = mock 'body'
+    @body_stub = stub 'body'
     Fog::AWS::SimpleDB.should_receive(:new).
                        with(:aws_access_key_id     => 'the-key',
                             :aws_secret_access_key => 'the-secret',
@@ -22,6 +22,42 @@ describe Sdbport do
     @sdb.domains.should == ['domain1']
   end
 
+  it "should perform select query given" do
+    @fog_mock.should_receive(:select).
+              with('select * from name', 'ConsistentRead' => true).
+              and_return @body_stub
+    @body_stub.stub :body => 'thegoods'
+    @sdb.select('select * from name', 'ConsistentRead' => true).
+         should == 'thegoods'
+  end
+
+  it "should perform select query given and next tokens" do
+    body_stub0 = stub 'body0', :body => { 'Items' => 
+                                            { 'id1' => 'val1' }, 
+                                          'NextToken' => '1' 
+                                        }
+    body_stub1 = stub 'body1', :body => { 'Items' => 
+                                            { 'id2' => 'val2' }, 
+                                          'NextToken' => '2' 
+                                        }
+    body_stub2 = stub 'body2', :body => { 'Items' => 
+                                            { 'id3' => 'val3' }
+                                        }
+    @fog_mock.should_receive(:select).
+              with('select * from name', 'NextToken' => nil).
+              and_return body_stub0
+    @fog_mock.should_receive(:select).
+              with('select * from name', 'NextToken' => '1').
+              and_return body_stub1
+    @fog_mock.should_receive(:select).
+              with('select * from name', 'NextToken' => '2').
+              and_return body_stub2
+    @sdb.select_and_follow_tokens('select * from name').
+         should == { 'id1' => 'val1',
+                     'id2' => 'val2',
+                     'id3' => 'val3' }
+  end
+
   it "should create a new domain when it does not exist" do
     @fog_mock.stub :list_domains => @body_stub
     @body_stub.stub :body => { 'Domains' => [] }

data/spec/domain/export_spec.rb

@@ -20,7 +20,7 @@ describe Sdbport do
              'item2' =>
              { 'attribute' => [ 'different' ] }
            }
-    @sdb_mock.should_receive(:select).
+    @sdb_mock.should_receive(:select_and_follow_tokens).
               with('select * from `name`').
               and_return data
     @file_mock.should_receive(:write).with("[\"item1\",{\"attribute\":[\"value\"]}]")

data/spec/domain/purge_spec.rb

@@ -15,7 +15,7 @@ describe Sdbport do
     result = { 'item1' =>
                { 'attribute' => [ 'value' ] }
              }
-    @sdb_mock.should_receive(:select).
+    @sdb_mock.should_receive(:select_and_follow_tokens).
               with('select * from `name`').
               and_return result 
     @sdb_mock.should_receive(:delete).with 'name', 'item1'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sdbport
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70332196552280 !ruby/object:Gem::Requirement
+  requirement: &70237921138580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70332196552280
+  version_requirements: *70237921138580
 - !ruby/object:Gem::Dependency
   name: fog
-  requirement: &70332196551760 !ruby/object:Gem::Requirement
+  requirement: &70237921138140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70332196551760
+  version_requirements: *70237921138140
 - !ruby/object:Gem::Dependency
   name: trollop
-  requirement: &70332196551200 !ruby/object:Gem::Requirement
+  requirement: &70237921137660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,7 +43,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70332196551200
+  version_requirements: *70237921137660
 description: Import and export AWS SimpleDB domains.
 email:
 - brett@weav.net
@@ -96,7 +96,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3221497568157798114
+      hash: -1634563892266774728
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -105,7 +105,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3221497568157798114
+      hash: -1634563892266774728
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.16