amazon_sdb 0.5.5

data/History.txt

@@ -0,0 +1,11 @@
+== 0.5.5 / 2007-12-15
+
+* 2 major enhancements
+  * Updated for new Amazon API release
+  * Thorough tests
+
+== 0.5.0 / 2007-11-05
+
+* 1 major enhancement
+  * Initial release
+

data/Manifest.txt

@@ -0,0 +1,15 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/amazon_sdb.rb
+lib/amazon_sdb/base.rb
+lib/amazon_sdb/domain.rb
+lib/amazon_sdb/item.rb
+lib/amazon_sdb/multimap.rb
+lib/amazon_sdb/resultset.rb
+lib/amazon_sdb/exceptions.rb
+test/test_amazon_base.rb
+test/test_amazon_domain.rb
+test/test_sdb_harness.rb
+test/test_multimap.rb

data/README.txt

@@ -0,0 +1,84 @@
+amazon_sdb
+by Jacob Harris
+jharris@nytimes.com
+http://open.nytimes.com/
+
+== SOURCE CODE:
+
+http://code.nytimes.com/svn/ruby/gems/amazon_sdb
+
+== DESCRIPTION:
+  
+Amazon sdb is a Ruby wrapper to Amazon's new Simple Database Service. Amazon sdb is a different type of database:
+
+* Accessed over the network via RESTful calls
+* No schemas or types
+* Each sdb account can have up to 100 domains for data.
+* Domains can hold objects referenced by unique keys
+* Each object can hold up to 256 name/value attributes.
+* Only name/value pairs must be unique in an objects attributes, there can be multiple name/value attributes with the same name.
+* In addition to key-driven accessors, objects can also be searched with a basic query language.
+* All value are stored as strings and comparisons use lexical order. Thus, it is necessary to pad integers and floats with 0s and save dates in ISO 8601 format for query comparisons to work
+
+== FEATURES:
+  
+* A basic interface to Amazon sdb
+* Includes a class for representing attribute sets in sdb
+* Automatic conversion to/from sdb representations for integers and dates (for floats, it's suggested you use the Multimap#numeric function)
+* The beginnings of mock-based tests for methods derived from Amazon's docs
+
+== CAVEATS
+
+* Not all features are tested yet (sorry!)
+* Amazon has not actually opened up access to the 2007-11-07 API on which this gem is based (and my tests are for). Some things may work differently in real life.
+* Errors still need to be figured out / tested.
+* I don't process the data usage/costs info from Amazon yet.
+
+== FUTURE WORK:
+
+* Some sort of fake SQL-esque query language
+* Some sort of AR/Datamapper/Ambition connection layer fun (with schema overlays)
+
+== SYNOPSIS:
+
+  b = Amazon::sdb::Base.new(aws_public_key, aws_secret_key)
+	b.domains #=> list of domain
+	domain = b.create_domain 'my domain'
+	
+	m = Multimap.new {:first_name => "Jacob", :last_name => "Harris"}
+	domain.put_attributes "Jacob Harris", m
+	resultset = domain.query "['first_name' begins-with 'Harris']"
+
+== REQUIREMENTS:
+
+* An Amazon Web Services account
+* hpricot
+
+== INSTALL:
+
+* sudo gem install amazon_sdb
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2007 FIX
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,23 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+require './lib/amazon_sdb.rb'
+require 'rcov/rcovtask'
+Hoe.new('amazon_sdb', Amazon::SDB::VERSION) do |p|
+  p.rubyforge_name = 'nytimes'
+  p.author = 'Jacob Harris'
+  p.email = 'harrisj@nytimes.com'
+  p.summary = 'A ruby wrapper to Amazon\'s sdb service'
+  p.description = 'A ruby wrapper to Amazon\'s sdb service'
+  p.url = "http://nytimes.rubyforge.org/amazon_sdb"
+  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+  p.extra_deps << ['hpricot', '>= 0.6']
+end
+
+Rcov::RcovTask.new do |t|
+  t.test_files = FileList['test/test*.rb']
+  t.rcov_opts << "-Ilib:test"
+  t.verbose = true     # uncomment to see the executed command
+end
+# vim: syntax=Ruby

data/lib/amazon_sdb.rb

@@ -0,0 +1,39 @@
+require 'rubygems'
+require "net/http"
+require "uri"
+require "cgi"
+require "digest/md5"
+require "digest/sha1"
+
+# HMAC Digest doesn't require TLS/SSL, but we do need the OpenSSL::HMAC class
+require "openssl"
+require 'base64'
+require 'open-uri'
+require 'hpricot'
+
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+unless defined?(Amazon::SDS)
+  begin
+    $:.unshift(File.dirname(__FILE__) + "/../../amazon_sdb/lib")
+    require 'amazon_sdb'  
+  rescue LoadError
+    require 'rubygems'
+    gem 'amazon_sdb'
+  end
+end
+
+require 'amazon_sdb/multimap'
+require 'amazon_sdb/base'
+require 'amazon_sdb/domain'
+require 'amazon_sdb/item'
+require 'amazon_sdb/resultset'
+require 'amazon_sdb/exceptions'
+
+module Amazon
+  module SDB
+    VERSION = '0.5.5'    
+  end
+end
+

data/lib/amazon_sdb/base.rb

@@ -0,0 +1,156 @@
+module Amazon
+  module SDB
+    SIGNATURE_VERSION = '1'
+    API_VERSION = '2007-11-07'
+    
+    ##
+    # The Amazon::SDS::Base class is the top-level interface class for your SDS interactions. It allows you to set global
+    # configuration settings, to manage Domain objects. If you are working within a particular domain you can also just use
+    # the domain initializer directly for that domain.
+    class Base
+      ##
+      # The base is initialized with 2 parameters from your Amazon Web Services account:
+      # * +aws_access_key+ - Your AWS Access Key
+      # * +aws_secret_key+ - Your AWS Secret Key
+      def initialize(aws_access_key, aws_secret_key)
+        @access_key = aws_access_key
+        @secret_key = aws_secret_key
+      end
+
+      ##
+      # Since all SDS supports only lexical comparisons, it's necessary to pad numbers with extra digits when saving them to SDS. 
+      # Under lexical matching, 23 > 123. But if we pad it sufficiently 000000023 < 000000123. By default, this is set to the 
+      # ungodly large value of 32 digits, but you can adjust it lower if this is too much. On reading from SDS, such numbers are
+      # auto-coerced back, so it's probably not necessary to change.
+      def self.number_padding
+        return @@number_padding
+      end
+
+      ##
+      # Change the number padding
+      def self.number_padding=(num)
+        @@number_padding = num
+      end
+      
+      def self.float_precision
+        return @@float_precision
+      end
+      
+      def self.float_precision=(num)
+        return @@float_precision
+      end
+
+      ##
+      # Retrieves a list of domains in your SDS database. Each entry is a Domain object.
+      def domains
+        domains = []
+        nextToken = nil
+        base_options = {:Action => 'ListDomains'}
+        continue = true
+        
+        while continue
+          options = base_options.dup
+          options[:NextToken] = nextToken unless nextToken.nil?
+          
+          sdb_query(options) do |h|
+            h.search('//DomainName').each {|e| domains << Domain.new(@access_key, @secret_key, e.innerText)}
+            mt = h.at('//NextToken')
+            if mt
+              nextToken = mt.innerText
+            else
+              continue = false
+            end
+          end
+        end
+        
+        domains
+      end
+
+      ##
+      # Returns a domain object for SDS. Assumes the domain already exists, so errors might occur if you didn't create it.
+      def domain(name)
+        Domain.new(@access_key, @secret_key, name)
+      end
+
+      def raise_errors(hpricot)
+        errnode = hpricot.at('//Errors/Error')
+        return unless errnode
+        
+        msg = errnode.at('Message').innerText
+        case errnode.at('Code').innerText
+        when 'InvalidParameterValue'
+          raise InvalidParameterError, msg
+        when 'NumberDomainsExceeded'
+          raise DomainLimitError, msg
+        else
+          raise UnknownError, msg
+        end
+      end
+      
+      def create_domain(name)
+        sdb_query({:Action => 'CreateDomain', 'DomainName' => name}) do |h|
+          domain(name)
+        end
+      end
+
+      ##
+      # Deletes a domain. This operation is currently not supported by SDS.
+      def delete_domain!(name)
+        sdb_query({:Action => 'DeleteDomain', 'DomainName' => name})
+      end
+
+      def timestamp
+        Time.now.iso8601
+      end
+
+      def self.hmac(key, msg) 
+        Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest::Digest.new('sha1'), key, msg))
+      end
+
+      def cgi_encode(options) 
+        options.map do |k, v| 
+          case v
+          when Array
+            v.map{|i| Base.uriencode(k)+'='+Base.uriencode(i)}.join('&')
+          else
+            Base.uriencode(k)+'='+Base.uriencode(v)
+          end
+        end.join('&')
+      end
+
+      def sdb_query(options = {})
+        options.merge!({'AWSAccessKeyId' => @access_key,
+          'SignatureVersion'             => SIGNATURE_VERSION,
+          'Timestamp'                    => timestamp,
+          'Version'                      => API_VERSION })
+        options['Signature'] = Base.sign(@secret_key, options)
+
+        # send to S3
+        url = BASE_PATH + '?' + cgi_encode(options)
+
+        # puts "Requesting #{url}" #if $DEBUG
+        open(url) do |f|
+          h = Hpricot.XML(f)
+
+          raise_errors h
+          yield h if block_given?
+        end
+      end
+
+      def self.uriencode(str)
+        CGI.escape str.to_s
+      end
+
+      def self.sign(key, query_options)
+        option_array = query_options.to_a.map {|pair| [pair[0].to_s, pair[1].to_s]}.sort {|a, b| a[0].downcase <=> b[0].downcase }
+        return hmac(key, option_array.map {|pair| pair[0]+pair[1]}.join('')).chop
+      end
+
+    private
+      @@number_padding                 = 32
+      @@float_precision                = 8
+
+      BASE_PATH = 'http://sdb.amazonaws.com/'
+    end
+  end
+end

data/lib/amazon_sdb/domain.rb

@@ -0,0 +1,145 @@
+module Amazon
+  module SDB
+    
+    ##
+    # Each sdb account can have up to 100 domains. This class represents a single domain and may be instantiated either indirectly
+    # from the Amazon::sdb::Base class or via the Domain#initialize method. This class is what you can use to directly set attributes
+    # on domains. Be aware that the following limits apply:
+    # - 100 attributes per each call 
+    # - 256 total attribute name-value pairs per item 
+    # - 250 million attributes per domain 
+    # - 10 GB of total user data storage per domain 
+    class Domain < Base
+      attr_reader :name
+      
+      ##
+      # Creates a new Domain object. 
+      def initialize(aws_access_key, aws_secret_key, name)
+        super(aws_access_key, aws_secret_key)
+        @name = name
+      end
+      
+      ##
+      # Sets attributes for a given key in the domain. If there are no attributes supplied, it creates an empty set.
+      # Takes the following arguments:
+      # - key - a string key for the attribute set
+      # - multimap - an collection of attributes for the set in a Multimap object. If nothing, creates an empty set.
+      # - options - for put options
+      def put_attributes(key, multimap=nil, options = {})
+        req_options = {'Action' => 'PutAttributes', 'DomainName' => name, 'ItemName' => key}
+        
+        unless multimap.nil?
+          req_options.merge! case multimap
+          when Hash, Array
+            Multimap.new(multimap).to_sdb(options)
+          when Multimap
+            multimap.to_sdb(options)
+          else
+            raise ArgumentError, "The second argument must be a multimap, hash, or array"
+          end
+        end
+        
+        sdb_query(req_options) do |h|
+          # check for success?
+          if h.search('//Success').any?
+            return Item.new(self, key, multimap)
+          else
+           # error?
+          end
+        end
+      end
+      
+      ##
+      # Gets the attribute list for a key. Arguments:
+      # - <tt>key</tt> - the key for the attribute set
+      def get_attributes(key, *attr_list)
+        options = {'Action' => 'GetAttributes', 'DomainName' => name, 'ItemName' => key}
+        
+        unless attr_list.nil? or attr_list.empty?
+          options["AttributeName"] = attr_list.map {|x| x.to_s }
+        end
+        
+        sdb_query(options) do |h|
+          attr_nodes = h.search('//GetAttributesResult/Attribute')
+          attr_array = []
+          attr_nodes.each do |a|
+            attr_array << [a.at('Name').innerText, a.at('Value').innerText]
+          end
+          
+          if attr_array.any?
+            return Item.new(self, key, Multimap.new(attr_array))
+          else
+            raise RecordNotFoundError, "No record was found for key=#{key}"
+          end
+        end
+      end
+
+      ##
+      # Not implemented yet.
+      def delete_attributes(key, multimap=nil)
+        options = {'Action' => 'DeleteAttributes', 'DomainName' => name, 'ItemName' => key}
+        
+        unless multimap.nil?
+          case multimap
+          when String, Symbol
+            options.merge! "Attribute.0.Name" => multimap.to_s
+          when Array
+            multimap.each_with_index do |k, i|
+              options["Attribute.#{i}.Name"] = k
+            end
+          when Hash
+            options.merge! Multimap.new(multimap).to_sdb
+          when Multimap
+            options.merge! multimap.to_sdb
+          else
+            raise ArgumentError, "Bad input paramter for attributes"
+          end
+        end
+        
+        sdb_query(options) do |h|
+          if h.search('//Success').any?
+            return true
+          end
+        end
+      end
+      
+      ##
+      # Returns a list of matching items that match a filter
+      # Options include:
+      # - <tt>max_results</tt> = the max items to return for a listing (top/default is 100)
+      # - <tt>:more_token</tt> = to retrieve a second or more page of results, the more token should be provided
+      # - <tt>:load_attrs</tt> = this query normally returns just a list of names, the attributes have to be retrieved separately. To load the attributes for matching results automatically, set to true (normally false)
+      def query(query_options = {})
+        req_options = {'Action' => 'Query', 'DomainName' => name}
+        
+        unless query_options[:expr].nil?
+          req_options['QueryExpression'] = query_options[:expr]
+        end
+        
+        if query_options[:next_token]
+          req_options['NextToken'] = query_options[:next_token]
+        end
+        
+        if query_options[:max_results]
+          req_options['MaxNumberOfItems'] = query_options[:max_results]
+        end
+        
+        sdb_query(req_options) do |h|
+          more_token = nil
+          results = h.search('//QueryResponse/QueryResult/ItemName')
+
+          items = results.map {|n| Item.new(self, n.innerText) }
+  
+          if query_options[:load_attrs]
+            items.each {|i| i.reload! }
+          end
+          
+          mt = h.search('//NextToken')
+          more_token = mt.inner_text unless mt.nil?
+        
+          return ResultSet.new(self, items, more_token)
+        end
+      end
+    end
+  end
+end