sdbport 0.3.0 → 0.4.0

data/CHANGELOG

@@ -1,3 +1,9 @@
+## 0.4.0:
+
+* Added sequential export
+* Added specs for CLI
+* Updated README
+
 ## 0.3.0:
 
 * Added support for reading multiple env configs

data/README.md

@@ -6,7 +6,9 @@ Sdbport exports & imports data from AWS SimpleDB domains. It can be used as a cl
 
 ## Installation
 
+```
 gem install sdbport
+```
 
 ## Getting Started
 
@@ -20,13 +22,20 @@ export AWS_SECRET_ACCESS_KEY=your_aws_secret
 Export SimpleDB domain from us-west-1:
 
 ```
-sdbport export -a $AWS_ACCESS_KEY_ID -s $AWS_SECRET_ACCESS_KEY -r us-west-1 -n data -o /tmp/test-domain-dump
+sdbport export -k $AWS_ACCESS_KEY_ID -s $AWS_SECRET_ACCESS_KEY -r us-west-1 -n data -o /tmp/test-domain-dump
+```
+
+To export larger SimpleDB domains, add -w.  This writes each chunk to file as it is received rather than storing in memory:
+
+```
+sdbport export -k $AWS_ACCESS_KEY_ID -s $AWS_SECRET_ACCESS_KEY -r us-west-1 -n data -o /tmp/test-domain-dump -w
 ```
 
 Import into domain in us-east-1
 
 ```
-sdbport import -a $AWS_ACCESS_KEY_ID -s $AWS_SECRET_ACCESS_KEY -r us-west-1 -n data -i /tmp/test-domain-dump
+sdbport import -k $AWS_ACCESS_KEY_ID -s $AWS_SECRET_ACCESS_KEY -r us-west-1 -n data -i /tmp/test-domain-dump
+```
 
 ## Exporting and importing from multiple accounts.
 

data/lib/sdbport/aws/simpledb.rb

@@ -35,6 +35,17 @@ module Sdbport
         end
       end
 
+      def select_and_store_chunk_of_tokens(query, options = {})
+        options.merge! 'NextToken' => @token_for_next_chunk
+        chunk = sdb.select(query, options).body
+        @token_for_next_chunk = chunk['NextToken']
+        return chunk['Items']
+      end
+
+      def more_chunks?
+        @token_for_next_chunk != nil
+      end
+
       def count(domain)
         body = sdb.select("SELECT count(*) FROM `#{domain}`").body
         body['Items']['Domain']['Count'].first.to_i

data/lib/sdbport/cli/export.rb

@@ -19,7 +19,12 @@ module Sdbport
                             :access_key => access_key,
                             :secret_key => secret_key,
                             :logger     => logger
-        exit 1 unless domain.export opts[:output]
+
+        if opts[:write_as_you_go]
+          exit 1 unless domain.export_sequential_write opts[:output]
+        else
+          exit 1 unless domain.export opts[:output]
+        end
       end
 
       def read_options
@@ -44,6 +49,7 @@ EOS
           opt :access_key, "AWS Access Key ID", :type  => :string,
                                                 :short => 'k'
           opt :secret_key, "AWS Secret Access Key", :type => :string
+          opt :write_as_you_go, "Write chunks to disk as they are received from Simple DB"
         end
       end
     end

data/lib/sdbport/domain.rb

@@ -18,6 +18,10 @@ module Sdbport
       domain_export.export output
     end
 
+    def export_sequential_write(output)
+      domain_export.export_sequential_write output
+    end
+
     def purge
       domain_purge.purge
     end

data/lib/sdbport/domain/export.rb

@@ -13,8 +13,7 @@ module Sdbport
       end
 
       def export(output)
-        @logger.info "Export #{@name} in #{@region} to #{output}"
-        file = File.open(output, 'w')
+        file = setup_file output
         export_domain.each do |item| 
           file.write convert_to_string item
           file.write "\n"
@@ -22,8 +21,27 @@ module Sdbport
         return true if file.close.nil?
       end
 
+      def export_sequential_write(output)
+        file = setup_file output
+        @logger.info "Writing to disk as records received."
+
+        while true
+          export_domain_with_sequential_write.each do |item| 
+            file.write convert_to_string item
+            file.write "\n"
+          end
+          break unless sdb.more_chunks?
+        end
+        return true if file.close.nil?
+      end
+
       private
 
+      def setup_file(output)
+        @logger.info "Export #{@name} in #{@region} to #{output}"
+        File.open(output, 'w')
+      end
+
       def sdb
         @sdb ||= AWS::SimpleDB.new :access_key => @access_key,
                                    :secret_key => @secret_key,
@@ -34,6 +52,10 @@ module Sdbport
         sdb.select_and_follow_tokens "select * from `#{@name}`"
       end
 
+      def export_domain_with_sequential_write
+        sdb.select_and_store_chunk_of_tokens "select * from `#{@name}`"
+      end
+
       def convert_to_string(item)
         item.to_json
       end

data/lib/sdbport/version.rb

@@ -1,3 +1,3 @@
 module Sdbport
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/spec/aws/simpledb_spec.rb

@@ -49,6 +49,29 @@ describe Sdbport do
                      'id3' => 'val3' }
   end
 
+  it "should perform select query given and store next token" do
+    body_stub0 = stub 'body0', :body => { 'Items' => 
+                                            { 'id1' => 'val1' }, 
+                                          'NextToken' => '1' 
+                                        }
+    body_stub1 = stub 'body1', :body => { 'Items' => 
+                                            { 'id2' => 'val2' }, 
+                                          'NextToken' => nil
+                                        }
+    @fog_mock.should_receive(:select).
+              with('select * from name', 'NextToken' => nil).
+              and_return body_stub0
+    @sdb.select_and_store_chunk_of_tokens('select * from name').
+         should == { 'id1' => 'val1' }
+    @sdb.more_chunks?.should be_true
+    @fog_mock.should_receive(:select).
+              with('select * from name', 'NextToken' => '1').
+              and_return body_stub1
+    @sdb.select_and_store_chunk_of_tokens('select * from name').
+         should == { 'id2' => 'val2' }
+    @sdb.more_chunks?.should be_false
+  end
+
   it "should create a new domain when it does not exist" do
     @fog_mock.stub :list_domains => @body_stub
     @body_stub.stub :body => { 'Domains' => [] }

data/spec/cli/destroy_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+describe Sdbport::CLI::Destroy do
+  before do
+    @domain_mock = mock "domain"
+    @logger_stub = stub "logger"
+    @options = { :name       => 'daname',
+                 :region     => 'us-west-1',
+                 :secret_key => 'private',
+                 :access_key => 'abc',
+                 :level      => 'debug' }
+
+    Sdbport::SdbportLogger.should_receive(:new).
+                           with(:log_level => 'debug').
+                           and_return @logger_stub
+    @destroy = Sdbport::CLI::Destroy.new
+  end
+
+  it "should perform destroy the domain" do
+    Trollop.stub :options => @options
+    Sdbport::Domain.should_receive(:new).
+                    with(:name       => 'daname',
+                         :region     => 'us-west-1',
+                         :secret_key => 'private',
+                         :access_key => 'abc',
+                         :logger     => @logger_stub).
+                    and_return @domain_mock
+    @domain_mock.should_receive(:destroy).and_return true
+    @destroy.destroy
+  end
+
+end

data/spec/cli/export_spec.rb

@@ -0,0 +1,49 @@
+require 'spec_helper'
+
+describe Sdbport::CLI::Export do
+  before do
+    @domain_mock = mock "domain"
+    @logger_stub = stub "logger"
+    @options = { :name       => 'daname',
+                 :region     => 'us-west-1',
+                 :secret_key => 'private',
+                 :access_key => 'abc',
+                 :level      => 'debug',
+                 :output     => '/test/file' }
+
+    Sdbport::SdbportLogger.should_receive(:new).
+                           with(:log_level => 'debug').
+                           and_return @logger_stub
+    @export = Sdbport::CLI::Export.new
+  end
+
+  it "should perform a in memeory write" do
+    Trollop.stub :options => @options
+    Sdbport::Domain.should_receive(:new).
+                    with(:name       => 'daname',
+                         :region     => 'us-west-1',
+                         :secret_key => 'private',
+                         :access_key => 'abc',
+                         :logger     => @logger_stub).
+                    and_return @domain_mock
+    @domain_mock.should_receive(:export).with('/test/file').
+                 and_return true
+    @export.export
+  end
+
+  it "should perform a sequential write" do
+    @options.merge! :write_as_you_go => true
+    Trollop.stub :options => @options
+    Sdbport::Domain.should_receive(:new).
+                    with(:name       => 'daname',
+                         :region     => 'us-west-1',
+                         :secret_key => 'private',
+                         :access_key => 'abc',
+                         :logger     => @logger_stub).
+                    and_return @domain_mock
+    @domain_mock.should_receive(:export_sequential_write).
+                 with('/test/file').
+                 and_return true
+    @export.export
+  end
+end

data/spec/cli/import_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+describe Sdbport::CLI::Import do
+  before do
+    @domain_mock = mock "domain"
+    @logger_stub = stub "logger"
+    @options = { :name       => 'daname',
+                 :region     => 'us-west-1',
+                 :secret_key => 'private',
+                 :access_key => 'abc',
+                 :level      => 'debug',
+                 :input      => '/test/file' }
+
+    Sdbport::SdbportLogger.should_receive(:new).
+                           with(:log_level => 'debug').
+                           and_return @logger_stub
+    @import = Sdbport::CLI::Import.new
+  end
+
+  it "should perform an import from file" do
+    Trollop.stub :options => @options
+    Sdbport::Domain.should_receive(:new).
+                    with(:name       => 'daname',
+                         :region     => 'us-west-1',
+                         :secret_key => 'private',
+                         :access_key => 'abc',
+                         :logger     => @logger_stub).
+                    and_return @domain_mock
+    @domain_mock.should_receive(:import).with('/test/file').
+                 and_return true
+    @import.import
+  end
+
+end

data/spec/cli/purge_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+describe Sdbport::CLI::Purge do
+  before do
+    @domain_mock = mock "domain"
+    @logger_stub = stub "logger"
+    @options = { :name       => 'daname',
+                 :region     => 'us-west-1',
+                 :secret_key => 'private',
+                 :access_key => 'abc',
+                 :level      => 'debug' }
+
+    Sdbport::SdbportLogger.should_receive(:new).
+                           with(:log_level => 'debug').
+                           and_return @logger_stub
+    @purge = Sdbport::CLI::Purge.new
+  end
+
+  it "should perform a purge on the domain" do
+    Trollop.stub :options => @options
+    Sdbport::Domain.should_receive(:new).
+                    with(:name       => 'daname',
+                         :region     => 'us-west-1',
+                         :secret_key => 'private',
+                         :access_key => 'abc',
+                         :logger     => @logger_stub).
+                    and_return @domain_mock
+    @domain_mock.should_receive(:purge).and_return true
+    @purge.purge
+  end
+
+end

data/spec/domain/export_spec.rb

@@ -30,4 +30,22 @@ describe Sdbport do
     @export.export('/tmp/file').should be_true
   end
 
+  it "should export the given domain sequentially to disk" do
+    File.should_receive(:open).with('/tmp/file', 'w').
+                               and_return @file_mock
+    data = { 'item1' => 
+             { 'attribute' => [ 'value' ] },
+             'item2' =>
+             { 'attribute' => [ 'different' ] }
+           }
+    @sdb_mock.should_receive(:select_and_store_chunk_of_tokens).
+              with('select * from `name`').
+              and_return data
+    @file_mock.should_receive(:write).with("[\"item1\",{\"attribute\":[\"value\"]}]")
+    @file_mock.should_receive(:write).with("[\"item2\",{\"attribute\":[\"different\"]}]")
+    @sdb_mock.should_receive(:more_chunks?).and_return false
+    @file_mock.should_receive(:write).with("\n").exactly(2).times
+    @file_mock.should_receive(:close).and_return nil
+    @export.export_sequential_write('/tmp/file').should be_true
+  end
 end

data/spec/domain_spec.rb

@@ -15,7 +15,7 @@ describe Sdbport do
     @domain.import('/tmp/file').should be_true
   end
 
-  it "should call domain_export from the given output" do
+  it "should call export from the given output" do
     Sdbport::Domain::Export.should_receive(:new).
                             with(:args1 => 'val1').
                             and_return @mock
@@ -24,6 +24,15 @@ describe Sdbport do
     @domain.export('/tmp/file').should be_true
   end
 
+  it "should call export_sequential_write with the given output" do
+    Sdbport::Domain::Export.should_receive(:new).
+                            with(:args1 => 'val1').
+                            and_return @mock
+    @mock.should_receive(:export_sequential_write).with('/tmp/file').
+          and_return true
+    @domain.export_sequential_write('/tmp/file').should be_true
+  end
+
   it "should call domain_purge" do
     Sdbport::Domain::Purge.stub :new => @mock
     @mock.should_receive(:purge).and_return true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sdbport
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-02 00:00:00.000000000 Z
+date: 2012-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70269257047280 !ruby/object:Gem::Requirement
+  requirement: &70336407307380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70269257047280
+  version_requirements: *70336407307380
 - !ruby/object:Gem::Dependency
   name: fog
-  requirement: &70269257046800 !ruby/object:Gem::Requirement
+  requirement: &70336407306100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70269257046800
+  version_requirements: *70336407306100
 - !ruby/object:Gem::Dependency
   name: trollop
-  requirement: &70269257046300 !ruby/object:Gem::Requirement
+  requirement: &70336407304460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,7 +43,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70269257046300
+  version_requirements: *70336407304460
 description: Import and export AWS SimpleDB domains.
 email:
 - brett@weav.net
@@ -79,6 +79,10 @@ files:
 - lib/sdbport/version.rb
 - sdbport.gemspec
 - spec/aws/simpledb_spec.rb
+- spec/cli/destroy_spec.rb
+- spec/cli/export_spec.rb
+- spec/cli/import_spec.rb
+- spec/cli/purge_spec.rb
 - spec/cli_spec.rb
 - spec/config_spec.rb
 - spec/domain/destroy_spec.rb
@@ -102,7 +106,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3777096327641096662
+      hash: 3976557266021725498
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -111,7 +115,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3777096327641096662
+      hash: 3976557266021725498
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.16
@@ -120,6 +124,10 @@ specification_version: 3
 summary: Import and export AWS SimpleDB domains.
 test_files:
 - spec/aws/simpledb_spec.rb
+- spec/cli/destroy_spec.rb
+- spec/cli/export_spec.rb
+- spec/cli/import_spec.rb
+- spec/cli/purge_spec.rb
 - spec/cli_spec.rb
 - spec/config_spec.rb
 - spec/domain/destroy_spec.rb