sdbtools 0.0.1

data/Rakefile

@@ -0,0 +1,48 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "sdbtools"
+    gem.summary = %Q{A high-level OO interface to Amazon SimpleDB}
+    gem.description = <<END
+SDBTools layers a higher-level OO interface on top of RightAWS, as well as
+providing some command-line utilities for working with SimpleDB.
+END
+    gem.email = "devs@devver.net"
+    gem.homepage = "http://github.com/devver/sdbtools"
+    gem.authors = ["Avdi Grimm"]
+    gem.add_dependency "right_aws", "~> 1.10"
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "sdbtools #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/bin/sdbtool

@@ -0,0 +1,353 @@
+#!/usr/bin/env ruby
+
+gem 'main',        '~> 4.2'
+gem 'fattr',       '~> 2.1'
+gem 'highline',    '~> 1.5'
+gem 'right_aws',   '~> 1.10'
+gem 'progressbar', '~> 0.0.3'
+gem 'libxml-ruby', '~> 1.1'
+
+require 'main'
+require 'highline/import'
+require 'fattr'
+require 'right_aws'
+require 'progressbar'
+require 'pstore'
+require 'pathname'
+require 'fileutils'
+require 'yaml'
+
+require File.expand_path('../lib/sdbtools', File.dirname(__FILE__))
+
+at_exit do
+  if $!.kind_of?(Exception) && (!$!.kind_of?(SystemExit))
+    $stderr.puts <<"END"
+This program has encountered an error and cannot continue. Please see the log
+for details.
+END
+  end
+end
+
+
+Main do
+  description 'A tool for working with Amazon SimpleDB'
+
+  class NullObject
+    def methods_missing(*args, &block)
+      self
+    end
+  end
+
+  def initialize
+    HighLine.track_eof = false
+    RightAws::RightAWSParser.xml_lib = 'libxml'
+  end
+
+  def timestamp
+    Time.now.strftime('%Y%m%d%H%M')
+  end
+
+  option 'access_key' do
+    description 'Amazon AWS access key'
+    argument :required
+    default ENV.fetch('AMAZON_ACCESS_KEY_ID') {
+      ENV.fetch('AWS_ACCESS_KEY_ID')
+    }
+    attr
+  end
+
+  option 'secret_key' do
+    description 'Amazon AWS secret key'
+    argument :required
+    default ENV.fetch('AMAZON_SECRET_ACCESS_KEY') {
+      ENV.fetch('AWS_SECRET_ACCESS_KEY')
+    }
+    attr
+  end
+
+  option 'server' do
+    description 'SimpleDB server hostname'
+    argument :required
+    default 'sdb.amazonaws.com'
+  end
+
+  option 'port' do
+    description 'SimpleDB server port number'
+    argument :required
+    cast :int
+    default 443
+  end
+
+  option 'protocol' do
+    description 'SimpleDB protocol'
+    argument :required
+    default 'https'
+  end
+
+  option 'nil-rep' do
+    description 'How to represent nil values'
+    argument :required
+    default '<[<[<NIL>]>]>'
+  end
+
+  option 'progress' do
+    description "Show progress bar"
+    cast :bool
+    default true
+    attr
+  end
+
+  option 'verbose' do
+    description "Verbose output"
+    cast :bool
+    default false
+    attr
+  end
+  
+  option 'chunk-size', 'n' do
+    description "The number of items to process at once"
+    cast :int
+    default 100
+  end
+
+  option 'log' do
+    description 'File to log to'
+    default 'sdbtool.log'
+    attr
+  end
+
+  mixin 'domain' do
+    argument 'domain' do
+      description 'The SimpleDB domain to work on'
+      optional
+    end
+
+    fattr(:domain) { 
+      params['domain'].value || choose(*db.domains) do |q|
+        q.prompt = "Domain to operate on?"
+      end
+    }
+  end
+
+
+  fattr(:db) {
+    SimpleDB::Database.new(
+      access_key,
+      secret_key,
+      :server => params['server'].value,
+      :port   => params['port'].value,
+      :logger => logger,
+      :nil_representation => params['nil-rep'].value,
+      :protocol => params['protocol'].value)
+  }
+
+  def after_parse_parameters
+  end
+
+  def before_run
+    stderr 'sdbtool-errors.log'
+    logger ::Logger.new(log)
+    unless verbose
+      logger.level = ::Logger::WARN
+    end
+  end
+
+  mode 'dump' do
+    description 'Dump the contents of a domain to a file'
+
+    mixin 'domain'
+
+    argument 'file' do
+      description 'File to dump into'
+      optional
+      attr
+    end
+
+    def run
+      self.file ||= ask("Filename for this dump?") do |q|
+        q.default = "#{domain}-#{timestamp}.yaml"
+      end
+      dump_domain = db.domain(domain)
+      logger.info "Preparing to dump domain #{dump_domain.name}"
+      dump = db.make_dump(dump_domain, file)
+      dump.chunk_size = params['chunk-size'].value
+      count = dump.incomplete_count
+      logger.info "Dumping #{count} objects to #{file}"
+      pbar = progress ? ProgressBar.new("dump", count,stdout) : NullObject.new
+      dump.callback = lambda do |item_name|
+        logger.info "Dumped #{item_name}"
+        pbar.inc
+      end
+      dump.start!
+      pbar.finish
+      logger.info "Dump succeeded"
+    end
+  end
+
+  mode 'count' do
+    description 'Count the number of items in a domain'
+    mixin 'domain'
+
+    def run
+      domain = db.domain(params['domain'].value)
+      puts domain.count
+    end
+  end
+
+  mode 'load' do
+    description 'Load a domain from a file'
+
+    mixin 'domain'
+
+    argument 'file' do
+      description 'The file to load from'
+      attr
+    end
+
+    def run
+      target_domain = if db.domain_exists?(domain)
+                        db.domain(domain)
+                      else
+                        logger.info "Creating domain #{domain}"
+                        db.create_domain(domain)
+                      end
+      say "About to load domain #{target_domain.name} from #{file}"
+      return unless agree("Are you sure you want to continue?")
+      load = db.make_load(target_domain, file)
+      load.chunk_size = params['chunk-size'].value
+      count = load.incomplete_count
+      pbar = progress ? ProgressBar.new("load", count, stdout) : NullObject.new
+      load.callback = lambda do |item_name|
+        logger.info "Loaded #{item_name}"
+        pbar.inc
+      end
+      load.start!
+      pbar.finish
+      logger.info "Finished loading #{file} into #{domain}"
+    end
+
+  end
+
+  mode 'status' do
+    description 'Show the status of an operation'
+    
+    argument 'title' do
+      description 'The title of the operation'
+      optional
+      attr
+    end
+
+    option 'failed' do
+      description 'Show details about failed items'
+      default false
+      cast :bool
+    end
+
+    def run
+      self.title ||= choose(*Dir['*.simpledb_op_status'])
+      status_file = if title =~ /\.simpledb_op_status$/
+                      title
+                    else
+                      title + '.simpledb_op_status'
+                    end
+      task = SimpleDB::Task.new(status_file, logger)
+      if params['failed'].given?
+        task.failed_items.each do |item_name, info|
+          puts item_name + ": "
+          puts
+          puts info
+          puts "-" * 60
+        end
+      else
+        puts task.report
+      end
+    end
+  end
+
+  mode 'info' do
+    description 'Show information about a dump file'
+
+    argument 'file' do
+      description 'The dump file to examine'
+      attr
+    end
+
+    def run
+      dump_file = SimpleDB::DumpFile.new(file)
+      puts "File #{file} contains #{dump_file.size} records"
+    end
+  end
+
+  mode 'delete-domain' do
+    description "Delete a SimpleDB domain"
+    mixin 'domain'
+
+    def run
+      2.times do 
+        return unless agree("Are you sure you want to delete #{params['domain'].value}?") # "
+      end
+      db.delete_domain(params['domain'].value)
+    end
+  end
+
+  mode 'create-domain' do
+    description "Create a SimpleDB domain"
+
+    mixin 'domain'
+
+    def run
+      db.create_domain(params['domain'].value)
+    end
+  end
+
+  mode 'show' do
+    description "Show a single SimpleDB item"
+    mixin 'domain'
+
+    argument 'item_name' do
+      description "The name of the item to show"
+      arity -1                  # inifinite
+    end
+
+    def run
+      db.domain(domain).items(params['item_name'].values).each do |item|
+        puts item.to_yaml
+      end
+    end
+  end
+
+  mode 'list' do
+    mixin 'domain'
+    description "List all objects matching SELECT query"
+    
+    argument 'query' do
+      description "A SimpleDB SELECT where-clause"
+    end
+
+    def run
+      logger.info "Selecting items where '#{params['query'].value}'"
+      items = db.domain(domain).select(params['query'].value)
+      logger.info "Selected #{items.count} items"
+      items.each do |item|
+        puts item['itemName()']
+      end
+    end
+  end
+
+  mode 'remove' do
+    mixin 'domain'
+    description "Delete an item by name"
+
+    argument 'item_name' do
+      description "The name of the item to delete"
+    end
+
+    def run
+      db.domain(domain).delete(params['item_name'].value)
+    end
+  end
+
+  def run
+    help!
+  end
+end

data/lib/sdbtools.rb

@@ -0,0 +1,348 @@
+require 'fattr'
+require 'right_aws'
+
+module SimpleDB
+  class Operation
+    include Enumerable
+
+    def initialize(sdb, method, *args)
+      @sdb    = sdb
+      @method = method
+      @args   = args
+    end
+
+    def each
+      next_token = nil
+      begin
+        args = @args.dup
+        args << next_token
+        results = @sdb.send(@method, *args)
+        yield(results)
+        next_token = results[:next_token]
+      end while next_token
+    end
+
+  end
+
+  class Database
+    def initialize(access_key=nil, secret_key=nil, options={})
+      @sdb = RightAws::SdbInterface.new(access_key, secret_key, options)
+      @logger = options.fetch(:logger){::Logger.new($stderr)}
+    end
+
+    def domains
+      domains_op = Operation.new(@sdb, :list_domains, nil)
+      domains_op.inject([]) {|domains, results|
+        domains.concat(results[:domains])
+        domains
+      }
+    end
+
+    def domain(domain_name)
+      Domain.new(@sdb, domain_name)
+    end
+
+    def domain_exists?(domain_name)
+      domains.include?(domain_name)
+    end
+
+    def create_domain(domain_name)
+      @sdb.create_domain(domain_name)
+      domain(domain_name)
+    end
+
+    def delete_domain(domain_name)
+      @sdb.delete_domain(domain_name)
+    end
+
+    def make_dump(domain, filename)
+      Dump.new(domain, filename, @logger)
+    end
+
+    def make_load(domain, filename)
+      Load.new(domain, filename, @logger)
+    end
+
+    private
+  end
+
+  class Domain
+    attr_reader :name
+
+    def initialize(sdb, name)
+      @sdb        = sdb
+      @name       = name
+      @item_names = nil
+      @count      = nil
+    end
+
+    def [](item_name)
+      @sdb.get_attributes(name, item_name)[:attributes]
+    end
+
+    def item_names
+      return @item_names if @item_names
+      query = Operation.new(@sdb, :query, @name, nil, nil)
+      @item_names = query.inject([]) {|names, results| 
+        names.concat(results[:items])
+        names
+      }
+    end
+
+    def count
+      return @count if @count
+      op = Operation.new(@sdb, :select, "select count(*) from #{name}")
+      @count = op.inject(0) {|count, results|
+        count += results[:items].first["Domain"]["Count"].first.to_i
+        count
+      }
+    end
+
+    def items(item_names)
+      names  = item_names.map{|n| "'#{n}'"}.join(', ')
+      query  = "select * from #{name} where itemName() in (#{names})"
+      select = Operation.new(@sdb, :select, query)
+      select.inject({}) {|items, results|
+        results[:items].each do |item|
+          item_name  = item.keys.first
+          item_value = item.values.first
+          items[item_name] = item_value
+        end
+        items
+      }
+    end
+
+    def put(item_name, attributes)
+      @sdb.put_attributes(@name, item_name, attributes)
+    end
+
+    def select(query)
+      op = Operation.new(@sdb, :select, "select * from #{name} where #{query}")
+      op.inject([]){|items,results|
+        batch_items = results[:items].map{|pair|
+          item = pair.values.first
+          item.merge!({'itemName()' => pair.keys.first})
+          item
+        }
+        items.concat(batch_items)
+      }
+    end
+
+    def delete(item_name)
+      @sdb.delete_attributes(@name, item_name)
+    end
+  end
+
+  class Task
+    fattr :callback    => lambda{}
+    fattr :chunk_size  => 100
+
+    def initialize(status_file, logger)
+      @logger = logger
+      @status_file = Pathname(status_file)
+      @status = PStore.new(@status_file.to_s)
+      unless @status_file.exist?
+        initialize_status!(yield)
+      end
+      @status.transaction(false) do
+        @logger.info "Initializing #{@status_file} for PID #{$$}"
+        @status[$$] = {}
+        @status[$$][:working_items] = []
+      end
+    end
+
+    def session
+      yield
+    ensure
+      release_working_items!
+    end
+
+    def incomplete_count
+      @status.transaction(true) {|status|
+        Array(status[:incomplete_items]).size
+      }
+    end
+
+    def failed_items
+      @status.transaction(true) do |status|
+        status[:failed_items]
+      end
+    end
+
+    def reserve_items!(size)
+      @status.transaction(false) do |status|
+        chunk = status[:incomplete_items].slice!(0,size)
+        status[$$][:working_items].concat(chunk)
+        @logger.info("Reserved #{chunk.size} items")
+        chunk
+      end
+    end
+
+    def finish_items!(items)
+      @status.transaction(false) do |status|
+        items.each do |item_name|
+          status[:complete_items] << 
+            status[$$][:working_items].delete(item_name)
+          @logger.info("Marked item #{item_name} complete")
+        end
+      end
+    end
+
+    def release_working_items!
+      @logger.info("Releasing working items")
+      @status.transaction(false) do |status|
+        items = status[$$][:working_items]
+        status[:incomplete_items].concat(items)
+        status[$$][:working_items].clear
+      end
+    end
+
+    def report
+      @status.transaction(true) do |status|
+        done     = status[:complete_items].size
+        not_done = status[:incomplete_items].size
+        failed   = status[:failed_items].size
+        puts "Items (not done/done/failed): #{not_done}/#{done}/#{failed}"
+        status.roots.select{|root| root.kind_of?(Integer)}.each do |root|
+          pid = root
+          items = status[root][:working_items].size
+          puts "Process #{pid} working on #{items} items"
+        end
+      end
+    end
+
+    private
+
+    def record_failed_item!(item_name, info)
+      @logger.info "Problem with item #{item_name}"
+      @status.transaction(false) do |status|
+        status[:failed_items] ||= {}
+        status[:failed_items][item_name] = info
+      end
+    end
+    
+    def initialize_status!(item_names)
+      @status.transaction(false) do |status|
+        status[:incomplete_items] = item_names
+        status[:failed_items]     = {}
+        status[:complete_items]   = []
+      end
+    end
+  end
+
+  class Dump < Task
+    def initialize(domain, filename, logger)
+      @domain                   = domain
+      @dump_filename            = Pathname(filename)
+      super(status_filename, logger) {
+        @domain.item_names
+      }
+    end
+
+    def status_filename
+      Pathname(
+        @dump_filename.basename(@dump_filename.extname).to_s + 
+        ".simpledb_op_status")
+    end
+
+    def start!
+      session do
+        until (chunk = reserve_items!(chunk_size)).empty?
+          items = @domain.items(chunk)
+          dump_items(items)
+          finish_items!(chunk)
+          items.each do |item| callback.call(item) end
+        end
+      end
+    end
+
+    private
+
+    def dump_items(items)
+      @logger.info "Dumping #{items.size} items to #{@dump_filename}"
+      FileUtils.touch(@dump_filename) unless @dump_filename.exist?
+      file = File.new(@dump_filename.to_s)
+      file.flock(File::LOCK_EX)
+      @dump_filename.open('a+') do |f|
+        items.each_pair do |item_name, attributes|
+          @logger.info "Dumping item #{item_name}"
+          YAML.dump({item_name => attributes}, f)
+        end
+      end
+    ensure
+        file.flock(File::LOCK_UN)
+    end
+
+  end
+
+  class Load < Task
+    def initialize(domain, filename, logger)
+      @domain    = domain
+      @dump_filename = Pathname(filename)
+      @dump_file = DumpFile.new(filename)
+      super(status_filename, logger) {
+        @dump_file.item_names
+      }
+    end
+
+    def status_filename
+      Pathname(
+        @dump_filename.basename(@dump_filename.extname).to_s + 
+        "-load-#{@domain.name}.simpledb_op_status")
+    end
+
+    def start!
+      session do
+        chunk    = []
+        reserved = Set.new
+        @dump_file.each do |item_name, attributes|
+          if reserved.empty?
+            finish_items!(chunk)
+            reserved.replace(chunk = reserve_items!(chunk_size))
+            break if chunk.empty?
+          end
+          if reserved.include?(item_name)
+            @logger.info("#{item_name} is reserved, loading to #{@domain.name}")
+            begin
+              @domain.put(item_name, attributes)
+            rescue
+              record_failed_item!(
+                item_name, 
+                "#{$!.class.name}: #{$!.message}")
+            end
+            reserved.delete(item_name)
+            callback.call(item_name)
+          end
+        end
+        finish_items!(chunk)
+      end
+    end
+  end
+
+  class DumpFile
+    include Enumerable
+
+    def initialize(path)
+      @path = Pathname(path)
+    end
+
+    def item_names
+      map{|item_name, item| item_name}
+    end
+
+    def size
+      inject(0){|size, item_name, item|
+        size += 1
+      }
+    end
+
+    def each
+      @path.open('r') {|f|
+        YAML.load_documents(f) {|doc|
+          yield doc.keys.first, doc.values.first
+        }
+      }
+    end
+  end
+
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification 
+name: sdbtools
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Avdi Grimm
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-01-18 00:00:00 -05:00
+default_executable: sdbtool
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: right_aws
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: "1.10"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.9
+    version: 
+description: |
+  SDBTools layers a higher-level OO interface on top of RightAWS, as well as
+  providing some command-line utilities for working with SimpleDB.
+
+email: devs@devver.net
+executables: 
+- sdbtool
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Rakefile
+- VERSION
+- bin/sdbtool
+- lib/sdbtools.rb
+has_rdoc: true
+homepage: http://github.com/devver/sdbtools
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: A high-level OO interface to Amazon SimpleDB
+test_files: []
+