forforf-aws-sdb 0.5.0

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2007, 2008 Tim Dysinger
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README

@@ -0,0 +1,8 @@
+Fork of Tim Dysinger's aws-sdb
+
+Features added in this fork
+
+* Support for AWS Signature Version 2 (SHA256)
+* Updated to support queries using Select Expressions (Query was deprecated)
+* Automatically retries on 503 errors 
+* Fixed specs to run and passing all specs

data/Rakefile

@@ -0,0 +1,20 @@
+require 'rubygems'
+require 'spec/rake/spectask'
+require 'rake/gempackagetask'
+
+Spec::Rake::SpecTask.new
+
+gem_spec = eval(IO.read(File.join(File.dirname(__FILE__), "aws-sdb.gemspec")))
+
+desc "Open an irb session preloaded with this library"
+task :console do
+  sh "irb -rubygems -I lib -r aws_sdb.rb"
+end
+
+Rake::GemPackageTask.new(gem_spec) do |pkg|
+  pkg.gem_spec = gem_spec
+end
+
+task :install => [:package] do
+  sh %{sudo gem install pkg/#{gem_spec.name}-#{gem_spec.version}}
+end

data/lib/aws_sdb.rb

@@ -0,0 +1,3 @@
+require 'aws_sdb/error'
+require 'aws_sdb/service'
+

data/lib/aws_sdb/error.rb

@@ -0,0 +1,52 @@
+module AwsSdb
+
+  class Error < RuntimeError ; end
+
+  class RequestError < Error
+    attr_reader :request_id
+
+    def initialize(message, request_id=nil)
+      super(message)
+      @request_id = request_id
+    end
+  end
+  
+    class ServiceUnavailableError < Error
+    attr_reader :request_id
+
+    def initialize(message, request_id=nil)
+      puts "SUESUE"
+      super(message)
+      @request_id = request_id
+    end
+  end
+
+  class InvalidDomainNameError < RequestError ; end
+  class InvalidParameterValueError < RequestError ; end
+  class InvalidNextTokenError < RequestError ; end
+  class InvalidNumberPredicatesError < RequestError ; end
+  class InvalidNumberValueTestsError < RequestError ; end
+  class InvalidQueryExpressionError < RequestError ; end
+  class MissingParameterError < RequestError ; end
+  class NoSuchDomainError < RequestError ; end
+  class NumberDomainsExceededError < RequestError ; end
+  class NumberDomainAttributesExceededError < RequestError ; end
+  class NumberDomainBytesExceededError < RequestError ; end
+  class NumberItemAttributesExceededError < RequestError ; end
+  class RequestTimeoutError < RequestError ; end
+
+  class FeatureDeprecatedError < RequestError ; end
+
+  class ConnectionError < Error
+    attr_reader :response
+
+    def initialize(response)
+      super(
+        "#{response.code} \
+         #{response.message if response.respond_to?(:message)}"
+      )
+      @response = response
+    end
+  end
+
+end

data/lib/aws_sdb/service.rb

@@ -0,0 +1,270 @@
+require 'logger'
+require 'time'
+require 'cgi'
+require 'uri'
+require 'base64'
+require 'openssl'
+require 'rexml/document'
+require 'rexml/xpath'
+require 'curb-fu'
+
+#duck punch so that space converts to %20
+class CGI
+  @@accept_charset="UTF-8" unless defined?(@@accept_charset)
+
+  def CGI::escape(string)
+    string.gsub(/([^a-zA-Z0-9_.-]+)/n) do
+      '%' + $1.unpack('H2' * $1.size).join('%').upcase
+    end
+  end
+  
+end
+
+module AwsSdb
+
+  class Service
+    def initialize(options={})
+      @access_key_id = options[:access_key_id] || ENV['AMAZON_ACCESS_KEY_ID']
+      @secret_access_key = options[:secret_access_key] || ENV['AMAZON_SECRET_ACCESS_KEY']
+      @base_url = options[:url] || 'http://sdb.amazonaws.com'
+      @logger = options[:logger] || Logger.new("aws_sdb.log")
+    end
+    
+    def list_domains(max = nil, token = nil)
+      params = { 'Action' => 'ListDomains' }
+      params['NextToken'] =
+        token unless token.nil? || token.empty?
+      params['MaxNumberOfDomains'] =
+        max.to_s unless max.nil? || max.to_i == 0
+      doc = call(:get, params)
+      results = []
+      REXML::XPath.each(doc, '//DomainName/text()') do |domain|
+        results << domain.to_s
+      end
+      return results, REXML::XPath.first(doc, '//NextToken/text()').to_s
+    end
+
+    def create_domain(domain)
+      domain.strip! if domain
+      call(:post, { 'Action' => 'CreateDomain', 'DomainName'=> domain.to_s })
+      nil
+    end
+
+    def delete_domain(domain)
+      call(
+        :delete,
+        { 'Action' => 'DeleteDomain', 'DomainName' => domain.to_s }
+      )
+      nil
+    end
+     
+    def select(query, max = nil, token = nil, consistent_read = 'true')
+      query 
+      params = {
+        'Action' => 'Select',
+        'SelectExpression' => query,
+        'ConsistentRead' => consistent_read
+      }
+      params['NextToken'] =
+        token unless token.nil? || token.empty?
+      params['MaxNumberOfItems'] =
+        max.to_s unless max.nil? || max.to_i == 0
+
+      @logger.debug { "SELECT EXPRESSION BEFORE CALL: #{query.inspect}" } if @logger.debug?
+      doc = call(:get, params)
+      item_doc = REXML::XPath.match(doc, '//Item')
+      items = {}
+      attributes = {}
+      
+      #build the result hash
+      REXML::XPath.each(item_doc, '//Item/Name') do |item_name_doc|
+        item_name = item_name_doc.text
+        REXML::XPath.each(item_doc, '//Attribute') do |attrib_doc|
+          attrib_name = nil
+          attrib_value = nil
+          attrib_doc.elements.each('Name') do |name|
+            attrib_name = CGI.unescape(name.text)
+          end
+          attrib_doc.elements.each('Value') do |value|
+            attrib_value = CGI.unescape(value.text)
+          end
+          add_value(attributes, attrib_name, attrib_value)
+        end
+        items[item_name] = attributes
+      end
+      attr_names = REXML::XPath.match(item_doc, '//Attribute/Name/text()')
+      results = items
+
+      return results, REXML::XPath.first(doc, '//NextToken/text()').to_s
+    end
+    
+    def put_attributes(domain, item, attributes, replace = true)
+      params = {
+        'Action' => 'PutAttributes',
+        'DomainName' => domain.to_s,
+        'ItemName' => item.to_s
+      }
+      count = 0
+      #escaping key and value so signature computes correctly
+      attributes.each do | key, values |
+        ([]<<values).flatten.each do |value|
+          params["Attribute.#{count}.Name"] = CGI.escape(key.to_s)
+          params["Attribute.#{count}.Value"] = CGI.escape(value.to_s)
+          params["Attribute.#{count}.Replace"] = replace
+          count += 1
+        end
+      end
+      call(:put, params)
+      nil
+    end
+
+    #updated to support consitent read
+    def get_attributes(domain, item, consistent_read = 'true')
+      doc = call(
+        :get,
+        {
+          'Action' => 'GetAttributes',
+          'DomainName' => domain.to_s,
+          'ItemName' => item.to_s,
+          'ConsistentRead' =>  consistent_read
+        }
+      )
+      attributes = {}
+      REXML::XPath.each(doc, "//Attribute") do |attr|
+        unesc_key = REXML::XPath.first(attr, './Name/text()').to_s
+        unesc_value = REXML::XPath.first(attr, './Value/text()').to_s
+        #unescape key and value to return to normal
+        key = CGI.unescape(unesc_key)
+        value = CGI.unescape(unesc_value)
+        ( attributes[key] ||= [] ) << value
+      end
+      attributes
+    end
+
+    def delete_attributes(domain, item)
+      call(
+        :delete,
+        {
+          'Action' => 'DeleteAttributes',
+          'DomainName' => domain.to_s,
+          'ItemName' => item.to_s
+        }
+      )
+      nil
+    end
+
+    protected
+    
+    def build_canonical_query_string(q_params)
+      q_params.sort.collect { |k,v| [CGI.escape(k.to_s), CGI.escape(v.to_s)].join('=')}.join('&')
+    end
+
+    def build_actual_query_string(q_params)
+      q_params.sort.collect { |k,v| [CGI.escape(k.to_s), CGI.escape(v.to_s)].join('=')}.join('&')
+    end
+
+    def call(method, params)
+      #updated to support signtature version 2
+      params.merge!( {
+          'Version' => '2009-04-15',
+          'SignatureMethod' => 'HmacSHA256',
+          #'SignatureMethod' => 'HmacSHA1',
+          'SignatureVersion' => '2',
+          'AWSAccessKeyId' => @access_key_id,
+          'Timestamp' => Time.now.gmtime.iso8601
+        }
+      )
+      
+      @logger.debug { "CALL: #{method} with #{params.inspect}" } if @logger.debug?
+      
+  
+      canonical_querystring = build_canonical_query_string(params)
+      
+      @logger.debug { "CANONICAL: #{canonical_querystring.inspect}" } if @logger.debug?
+      
+      string_to_sign= "GET\n#{URI.parse(@base_url).host}\n/\n#{canonical_querystring}"
+      
+      #sha256
+      digest = OpenSSL::Digest::Digest.new('sha256')
+      signature = Base64.encode64(OpenSSL::HMAC.digest(digest, @secret_access_key, string_to_sign)).chomp
+      
+      #sha1
+      #digest = OpenSSL::Digest::Digest.new('sha256')
+      #signature = Base64.encode64(OpenSSL::HMAC.digest(digest, @secret_access_key, string_to_sign)).chomp      
+      
+      params['Signature'] = signature
+      querystring = build_actual_query_string(params)
+      
+      @logger.debug { "ACTUALQUERY: #{querystring.inspect}" } if @logger.debug?
+      
+      url = "#{@base_url}?#{querystring}"
+      uri = URI.parse(url)
+
+      resp = request(url)
+      resp_body = resp.body
+      
+      @logger.debug { "RESP: #{resp_body.inspect}" } if @logger.debug?
+ 
+      doc = REXML::Document.new(resp_body)
+      
+      @logger.debug { "DECODED DOC:" } if @logger.debug?
+      
+      error = doc.get_elements('*/Errors/Error')[0]
+      
+      raise(
+        Module.class_eval(
+          "AwsSdb::#{error.get_elements('Code')[0].text}Error"
+        ).new(
+          error.get_elements('Message')[0].text,
+          doc.get_elements('*/RequestID')[0].text
+        )
+      ) unless error.nil?
+      doc
+    end
+    
+    #request with retries
+    def request(url, retry_data={})
+      resp = CurbFu.get(url) 
+      if resp.nil? || resp.status == 503
+        resp = retry_req(url, retry_data)
+      end
+      raise "No response!!" unless resp
+      raise "No Body in Response" unless resp.body
+      resp
+    end
+    
+    private
+    
+    def retry_req(url, retry_data)
+      resp = :retry
+      #retry parameters
+      max_retries = 10||retry_data[:max_retries]
+      init_wait = 0.2||retry_data[:init_wait]
+      wait_increase = 0.3||retry_data[:wait_increase]
+      retry_data[:wait] = init_wait||retry_data[:wait]
+      
+      #wait a tiny bit before the first retry and reset retry data
+      #then retry 
+      1.upto(max_retries)  do |retry_att|
+        sleep retry_data[:wait]
+        #puts "RETRY: #{retry_att}, WAIT: #{retry_data[:wait]}"
+        resp = request(url, retry_data)
+        break if resp && resp.status && resp.status != 503 && resp.body
+        retry_data[:wait] += wait_increase
+        retry_data[:wait_increase] = wait_increase * retry_att #request back off
+      end
+      resp
+    end
+    
+    def add_value(attributes, attrib_name, attrib_value)
+      if attributes[attrib_name]
+        attributes[attrib_name] << attrib_value
+      else
+        attributes[attrib_name] = [attrib_value]
+      end
+      attributes
+    end
+
+  end#class
+
+end#module

data/spec/aws_sdb/aws_sdb.log

@@ -0,0 +1,13 @@
+# Logfile created on 2010-12-23 20:04:18 +0000 by logger.rb/25413
+D, [2010-12-23T20:04:18.454524 #25893] DEBUG -- : http://sdb.amazonaws.com?Action=DeleteDomain&AWSAccessKeyId=AKIAJZT2KIZFM36DNBVQ&DomainName=&SignatureVersion=1&Timestamp=2010-12-23T20%3A04%3A18Z&Version=2007-11-07&Signature=xCfsCVEy3dpskdJjmkwYfVIQudY%3D
+D, [2010-12-23T20:04:18.476771 #25893] DEBUG -- : 400
+<?xml version="1.0"?>
+<Response><Errors><Error><Code>InvalidParameterValue</Code><Message>Value () for parameter DomainName is invalid. </Message><BoxUsage>0.0055590278</BoxUsage></Error></Errors><RequestID>5ec30ba5-4b3c-f1f7-c1b2-0c65d16c57b9</RequestID></Response>
+D, [2010-12-23T20:04:18.479966 #25893] DEBUG -- : http://sdb.amazonaws.com?Action=DeleteDomain&AWSAccessKeyId=AKIAJZT2KIZFM36DNBVQ&DomainName=&SignatureVersion=1&Timestamp=2010-12-23T20%3A04%3A18Z&Version=2007-11-07&Signature=xCfsCVEy3dpskdJjmkwYfVIQudY%3D
+D, [2010-12-23T20:04:18.490482 #25893] DEBUG -- : 400
+<?xml version="1.0"?>
+<Response><Errors><Error><Code>InvalidParameterValue</Code><Message>Value () for parameter DomainName is invalid. </Message><BoxUsage>0.0055590278</BoxUsage></Error></Errors><RequestID>1ec92b8e-0053-2452-0140-015fdefa7f8c</RequestID></Response>
+D, [2010-12-23T20:04:18.493872 #25893] DEBUG -- : http://sdb.amazonaws.com?Action=DeleteDomain&AWSAccessKeyId=AKIAJZT2KIZFM36DNBVQ&DomainName=&SignatureVersion=1&Timestamp=2010-12-23T20%3A04%3A18Z&Version=2007-11-07&Signature=xCfsCVEy3dpskdJjmkwYfVIQudY%3D
+D, [2010-12-23T20:04:18.507715 #25893] DEBUG -- : 400
+<?xml version="1.0"?>
+<Response><Errors><Error><Code>InvalidParameterValue</Code><Message>Value () for parameter DomainName is invalid. </Message><BoxUsage>0.0055590278</BoxUsage></Error></Errors><RequestID>756ba7ca-e9f7-e73a-83ed-7d8396498550</RequestID></Response>

data/spec/aws_sdb/service_spec.rb

@@ -0,0 +1,208 @@
+require File.dirname(__FILE__) + '/../spec_helper.rb'
+
+require 'digest/sha1'
+require 'net/http'
+require 'rexml/document'
+
+require 'rubygems'
+require 'uuidtools'
+
+include AwsSdb
+
+describe Service, "when creating a new domain" do
+  include UUIDTools
+  
+  before(:all) do
+    @service = AwsSdb::Service.new
+    @domain = "test-#{UUID.random_create.to_s}"
+    domains = @service.list_domains[0]
+    domains.each do |d|
+      @service.delete_domain(d) if d =~ /^test/
+    end
+  end
+  
+  after(:all) do
+    @service.delete_domain(@domain)
+  end
+
+  it "should not raise an error if a valid new domain name is given" do
+    lambda {
+      @service.create_domain("test-#{UUID.random_create.to_s}")
+    }.should_not raise_error
+  end
+
+  it "should not raise an error if the domain name already exists" do
+    domain = "test-#{UUID.random_create.to_s}"
+    lambda {
+      @service.create_domain(domain)
+      @service.create_domain(domain)
+    }.should_not raise_error
+  end
+
+  it "should raise an error if an a nil or '' domain name is given" do
+    lambda {
+      @service.create_domain('')
+    }.should raise_error(InvalidParameterValueError)
+    lambda {
+      @service.create_domain(nil)
+    }.should raise_error(InvalidParameterValueError)
+    lambda {
+      @service.create_domain('     ')
+    }.should raise_error(InvalidParameterValueError)
+  end
+
+  it "should raise an error if the domain name length is < 3 or > 255" do
+    lambda {
+      @service.create_domain('xx')
+    }.should raise_error(InvalidParameterValueError)
+    lambda {
+      @service.create_domain('x'*256)
+    }.should raise_error(InvalidParameterValueError)
+  end
+
+  it "should only accept domain names with a-z, A-Z, 0-9, '_', '-', and '.' " do
+    lambda {
+      @service.create_domain('@$^*()')
+    }.should raise_error(InvalidParameterValueError)
+  end
+
+  #it "should only accept a maximum of 100 domain names"
+
+  #it "should not have to call amazon to determine domain name correctness"
+
+end
+
+describe Service, "when listing domains" do
+  include UUIDTools
+  
+  before(:all) do
+    @service = AwsSdb::Service.new
+    @domain = "test-#{UUID.random_create.to_s}"
+    @service.list_domains[0].each do |d|
+      @service.delete_domain(d) if d =~ /^test/
+    end
+    @service.create_domain(@domain)
+  end
+  
+  after(:all) do
+    @service.delete_domain(@domain)
+  end
+
+  it "should return a complete list" do
+    result = nil
+    lambda { result = @service.list_domains[0] }.should_not raise_error
+    result.should_not be_nil
+    result.should_not be_empty
+    result.include?(@domain).should == true
+  end
+end
+
+describe Service, "when deleting domains" do
+  include UUIDTools
+  before(:all) do
+    @service = AwsSdb::Service.new
+    @domain = "test-#{UUID.random_create.to_s}"
+    @service.list_domains[0].each do |d|
+      @service.delete_domain(d) if d =~ /^test/
+    end
+    @service.create_domain(@domain)
+  end
+  
+  before(:each) do
+    #sleep 0.2
+  end
+
+  after do
+    @service.delete_domain(@domain)
+  end
+
+  it "should be able to delete an existing domain" do
+    lambda { @service.delete_domain(@domain) }.should_not raise_error
+  end
+
+  it "should not raise an error trying to delete a non-existing domain" do
+    lambda {
+      @service.delete_domain(UUID.random_create.to_s)
+    }.should_not raise_error
+  end
+end
+
+describe Service, "when managing items" do
+    include UUIDTools
+    
+  before(:all) do
+    @service = AwsSdb::Service.new
+    @domain = "test-#{UUID.random_create.to_s}"
+    @service.list_domains[0].each do |d|
+      @service.delete_domain(d) if d =~ /^test/
+    end
+    @service.create_domain(@domain)
+    @item = "test-#{UUID.random_create.to_s}"
+    @attributes = {
+      :question => 'What is the answer?',
+      :answer => [ true, 'testing123', 4.2, 42, 420 ]
+    }
+  end
+  
+  before(:each) do
+    #sleep 0.2
+  end
+
+  after(:all) do
+     @service.delete_domain(@domain)
+  end
+
+  it "should be able to put attributes" do
+    lambda {
+      @service.put_attributes(@domain, @item, @attributes)
+    }.should_not raise_error
+  end
+
+  it "should be able to get attributes" do
+    result = nil
+    lambda {
+      result = @service.get_attributes(@domain, @item)
+    }.should_not raise_error
+    result.should_not be_nil
+    result.should_not be_empty
+    result.has_key?('answer').should == true
+    @attributes[:answer].each do |v|
+      result['answer'].include?(v.to_s).should == true
+    end
+  end
+
+  it "should be able to select all from domain" do
+    query_str = "select * from `#{@domain}`"
+    result = nil
+    lambda {
+      result = @service.select(query_str)
+    }.should_not raise_error
+    result.should_not be_nil
+    result.should_not be_empty
+    result.should_not be_nil
+    result[0].keys.include?(@item).should == true
+    result[0].values.size.should == 1
+    #attribute names
+    result[0].values[0].keys.sort.should == ['answer', 'question']
+    #questions
+    result[0].values[0]['question'].should == ['What is the answer?']
+    #answers
+    result[0].values[0]['answer'].size == 5
+    result[0].values[0]['answer'].include?("42").should == true
+    result[0].values[0]['answer'].include?("testing123").should == true
+  end
+
+
+  it "should be able to delete attributes" do
+    lambda {
+      @service.delete_attributes(@domain, @item)
+    }.should_not raise_error
+  end
+end
+
+# TODO Pull the specs from the amazon docs and write more rspec
+# 100 attributes per each call
+# 256 total attribute name-value pairs per item
+# 250 million attributes per domain
+# 10 GB of total user data storage per domain
+# ...etc...

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+$TESTING=true
+$:.push File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'aws_sdb'

metadata

@@ -0,0 +1,86 @@
+--- !ruby/object:Gem::Specification 
+name: forforf-aws-sdb
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 5
+  - 0
+  version: 0.5.0
+platform: ruby
+authors: 
+- Tim Dysinger
+- Dave M
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-12-26 00:00:00 +00:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: uuidtools
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+description: Amazon SDB API
+email: dmarti21@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README
+- LICENSE
+files: 
+- LICENSE
+- README
+- Rakefile
+- lib/aws_sdb.rb
+- lib/aws_sdb/error.rb
+- lib/aws_sdb/service.rb
+- spec/spec_helper.rb
+- spec/aws_sdb/aws_sdb.log
+- spec/aws_sdb/service_spec.rb
+has_rdoc: true
+homepage: http://github.com/forforf/aws-sdb
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: forforf-aws-sdb
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Amazon SDB API
+test_files: []
+