right_aws 1.9.0 → 1.10.0

data/lib/sdb/active_sdb.rb

@@ -24,7 +24,7 @@
 begin
   require 'uuidtools'
 rescue LoadError => e
-  STDERR.puts("RightSDB Alpha requires the uuidtools gem.  Run \'gem install uuidtools\' and try again.")
+  STDERR.puts("RightSDB requires the uuidtools gem.  Run \'gem install uuidtools\' and try again.")
   exit
 end
 
@@ -294,6 +294,8 @@ module RightAws
         #    Client.find(:first)                      #=> #<Client:0xb77d0d40 @attributes={"id"=>"2937601a-e45d-11dc-a75f-001bfc466dd7"}, @new_record=false>
         #    Client.find(:first, :auto_load => true)  #=> #<Client:0xb77d0d40 @attributes={"id"=>"2937601a-e45d-11dc-a75f-001bfc466dd7", "name"=>["Cat"], "toys"=>["Jons socks", "clew", "mice"]}, @new_record=false>
         #
+        # see http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?UsingQuery.html
+        #
         def find(*args)
           options = args.last.is_a?(Hash) ? args.pop : {}
           case args.first
@@ -303,12 +305,137 @@ module RightAws
           end
         end
 
+        # Perform a SQL-like select request.
+        #
+        # Single record:
+        #
+        #  Client.select(:first)
+        #  Client.select(:first, :conditions=> [ "name=? AND wife=?", "Jon", "Sandy"])
+        #  Client.select(:first, :conditions=> { :name=>"Jon", :wife=>"Sandy" }, :select => :girfriends)
+        #
+        # Bunch of records:
+        #
+        #  Client.select(:all)
+        #  Client.select(:all, :limit => 10)
+        #  Client.select(:all, :conditions=> [ "name=? AND 'girlfriend'=?", "Jon", "Judy"])
+        #  Client.select(:all, :conditions=> { :name=>"Sandy" }, :limit => 3)
+        #
+        # Records by ids:
+        #
+        #  Client.select('1')
+        #  Client.select('1234987b4583475347523948')
+        #  Client.select('1','2','3','4', :conditions=> ["toys=?", "beer"])
+        #
+        # Find helpers: RightAws::ActiveSdb::Base.select_by_... and RightAws::ActiveSdb::Base.select_all_by_...
+        #
+        #  Client.select_by_name('Matias Rust')
+        #  Client.select_by_name_and_city('Putin','Moscow')
+        #  Client.select_by_name_and_city_and_post('Medvedev','Moscow','president')
+        #
+        #  Client.select_all_by_author('G.Bush jr')
+        #  Client.select_all_by_age_and_gender_and_ethnicity('34','male','russian')
+        #  Client.select_all_by_gender_and_country('male', 'Russia', :order => 'name')
+        #
+        # Continue listing:
+        #
+        #  # initial listing
+        #  Client.select(:all, :limit => 10)
+        #  # continue listing
+        #  begin
+        #    Client.select(:all, :limit => 10, :next_token => Client.next_token)
+        #  end while Client.next_token
+        #
+        #  Sort oder:
+        #  If :order=>'attribute' option is specified then result response (ordered by 'attribute') will contain only items where attribute is defined (is not null).
+        #  
+        #    Client.select(:all)                         # returns all records
+        #    Client.select(:all, :order => 'gender')     # returns all records ordered by gender where gender attribute exists
+        #    Client.select(:all, :order => 'name desc')  # returns all records ordered by name in desc order where name attribute exists
+        #
+        # see http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?UsingSelect.html
+        #
+        def select(*args)
+          options = args.last.is_a?(Hash) ? args.pop : {}
+          case args.first
+            when :all   then sql_select(options)
+            when :first then sql_select(options.merge(:limit => 1)).first
+            else             select_from_ids args, options
+          end
+        end
+
+        def generate_id # :nodoc:
+          UUID.timestamp_create().to_s
+        end
+
       protected
 
+        # Select
+
+        def select_from_ids(args, options) # :nodoc:
+          cond = []
+          # detect amount of records requested
+          bunch_of_records_requested = args.size > 1 || args.first.is_a?(Array)
+          # flatten ids
+          args = args.to_a.flatten
+          args.each { |id| cond << "id=#{self.connection.escape(id)}" }
+          ids_cond = "(#{cond.join(' OR ')})"
+          # user defined :conditions to string (if it was defined)
+          options[:conditions] = build_conditions(options[:conditions])
+          # join ids condition and user defined conditions
+          options[:conditions] = options[:conditions].blank? ? ids_cond : "(#{options[:conditions]}) AND #{ids_cond}"
+          result = sql_select(options)
+          # if one record was requested then return it
+          unless bunch_of_records_requested
+            record = result.first
+            # railse if nothing was found
+            raise ActiveSdbError.new("Couldn't find #{name} with ID #{args}") unless record
+            record
+          else
+            # if a bunch of records was requested then return check that we found all of them
+            # and return as an array
+            unless args.size == result.size
+              id_list = args.map{|i| "'#{i}'"}.join(',')
+              raise ActiveSdbError.new("Couldn't find all #{name} with IDs (#{id_list}) (found #{result.size} results, but was looking for #{args.size})")
+            else
+              result
+            end
+          end
+        end
+
+        def sql_select(options) # :nodoc:
+          @next_token = options[:next_token]
+          select_expression = build_select(options)
+          # request items
+          query_result = self.connection.select(select_expression, @next_token)
+          @next_token = query_result[:next_token]
+          items = query_result[:items].map do |hash|
+            id, attributes = hash.shift
+            new_item = self.new( attributes.merge({ 'id' => id }))
+            new_item.mark_as_old
+            new_item
+          end
+          items
+        end
+
+        # select_by helpers
+        def select_all_by_(format_str, args, options) # :nodoc:
+          fields = format_str.to_s.sub(/^select_(all_)?by_/,'').split('_and_')
+          conditions = fields.map { |field| "#{field}=?" }.join(' AND ')
+          options[:conditions] = [conditions, *args]
+          select(:all, options)
+        end
+
+        def select_by_(format_str, args, options) # :nodoc:
+          options[:limit] = 1
+          select_all_by_(format_str, args, options).first
+        end
+
+        # Query
+
         # Returns an array of query attributes.
         # Query_expression must be a well formated SDB query string:
         # query_attributes("['title' starts-with 'O\\'Reily'] intersection ['year' = '2007']") #=> ["title", "year"]
-        def query_attributes(query_expression)
+        def query_attributes(query_expression) # :nodoc:
           attrs = []
           array = query_expression.scan(/['"](.*?[^\\])['"]/).flatten
           until array.empty? do
@@ -334,8 +461,7 @@ module RightAws
         #
         def query(options) # :nodoc:
           @next_token = options[:next_token]
-          query_expression = options[:query_expression]
-          query_expression = connection.query_expression_from_array(query_expression) if query_expression.is_a?(Array)
+          query_expression = build_conditions(options[:query_expression])
           # add sort_options to the query_expression
           if options[:sort_option]
             sort_by, sort_order = sort_options(options[:sort_option])
@@ -358,14 +484,19 @@ module RightAws
           items = query_result[:items].map do |name| 
             new_item = self.new('id' => name)
             new_item.mark_as_old
-            new_item.reload if options[:auto_load]
+            reload_if_exists(record) if options[:auto_load]
             new_item
           end
           items
         end
 
+        # reload a record unless it is nil
+        def reload_if_exists(record) # :nodoc:
+          record && record.reload
+        end
+
         def reload_all_records(*list) # :nodoc:
-          list.flatten.each { |record| record.reload }
+          list.flatten.each { |record| reload_if_exists(record) }
         end
 
         def find_every(options) # :nodoc:
@@ -391,15 +522,16 @@ module RightAws
           args.each { |id| cond << "'id'=#{self.connection.escape(id)}" }
           ids_cond = "[#{cond.join(' OR ')}]"
           # user defined :conditions to string (if it was defined)
-          if options[:conditions].is_a?(Array)
-            options[:conditions] = connection.query_expression_from_array(options[:conditions])
-          end
+          options[:conditions] = build_conditions(options[:conditions])
           # join ids condition and user defined conditions
           options[:conditions] = options[:conditions].blank? ? ids_cond : "#{options[:conditions]} intersection #{ids_cond}"
           result = find_every(options)
           # if one record was requested then return it
           unless bunch_of_records_requested
-            options[:auto_load] ? reload_all_records(result.first).first : result.first
+            record = result.first
+            # railse if nothing was found
+            raise ActiveSdbError.new("Couldn't find #{name} with ID #{args}") unless record
+            options[:auto_load] ? reload_all_records(record).first : record
           else
             # if a bunch of records was requested then return check that we found all of them
             # and return as an array
@@ -425,22 +557,39 @@ module RightAws
           find_all_by_(format_str, args, options).first
         end
 
+        # Misc
+
         def method_missing(method, *args) # :nodoc:
-          if method.to_s[/^(find_all_by_|find_by_)/]
+          if method.to_s[/^(find_all_by_|find_by_|select_all_by_|select_by_)/]
             options = args.last.is_a?(Hash) ? args.pop : {}
-            if method.to_s[/^find_all_by_/]
-              find_all_by_(method, args, options)
-            else
-              find_by_(method, args, options)
-            end
+            __send__($1, method, args, options)
           else
             super(method, *args)
           end
         end
-      end
-      
-      def self.generate_id # :nodoc:
-        UUID.timestamp_create().to_s
+
+        def build_select(options) # :nodoc:
+          select     = options[:select]    || '*'
+          from       = options[:from]      || domain
+          conditions = options[:conditions] ? " WHERE #{build_conditions(options[:conditions])}" : ''
+          order      = options[:order]      ? " ORDER BY #{options[:order]}"                     : ''
+          limit      = options[:limit]      ? " LIMIT #{options[:limit]}"                        : ''
+          # mix sort by argument (it must present in response)
+          unless order.blank?
+            sort_by, sort_order = sort_options(options[:order])
+            conditions << (conditions.blank? ? " WHERE " : " AND ") << "(#{sort_by} IS NOT NULL)"
+          end
+          "SELECT #{select} FROM #{from}#{conditions}#{order}#{limit}"
+        end
+
+        def build_conditions(conditions) # :nodoc:
+          case
+          when conditions.is_a?(Array) then connection.query_expression_from_array(conditions)
+          when conditions.is_a?(Hash)  then connection.query_expression_from_hash(conditions)
+          else                              conditions
+          end
+        end
+
       end
       
       public
@@ -695,7 +844,15 @@ module RightAws
         attrs.delete('id')
         unless attrs.blank?
           connection.delete_attributes(domain, id, attrs)
-          attrs.each { |attribute, values| @attributes[attribute] -= values }
+          attrs.each do |attribute, values|
+            # remove the values from the attribute
+            if @attributes[attribute]
+              @attributes[attribute] -= values
+            else
+              # if the attribute is unknown remove it from a resulting list of fixed attributes
+              attrs.delete(attribute)
+            end
+          end
         end
         attrs
       end

data/lib/sdb/right_sdb_interface.rb

@@ -78,28 +78,25 @@ module RightAws
       params.delete_if {|key,value| value.nil? }
       #params_string  = params.to_a.collect{|key,val| key + "=#{CGI::escape(val.to_s)}" }.join("&")
       # prepare service data
-      service_hash = {"Action"            => action,
-                      "AWSAccessKeyId"    => @aws_access_key_id,
-                      "Version"           => API_VERSION,
-                      "Timestamp"         => Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z"),
-                      "SignatureVersion"  => signature_version }
+      service = '/'
+      service_hash = {"Action"         => action,
+                      "AWSAccessKeyId" => @aws_access_key_id,
+                      "Version"        => API_VERSION }
       service_hash.update(params)
-      # prepare string to sight
-      string_to_sign = case signature_version
-                       when '0' then service_hash["Action"] + service_hash["Timestamp"]
-                       when '1' then service_hash.sort{|a,b| (a[0].to_s.downcase)<=>(b[0].to_s.downcase)}.to_s
-                       end
-      service_hash.update('Signature' =>  AwsUtils::sign(@aws_secret_access_key, string_to_sign))
-      service_string = service_hash.to_a.collect{|key,val| key + "=#{CGI::escape(val.to_s)}" }.join("&")
+      service_params = signed_service_params(@aws_secret_access_key, service_hash, :get, @params[:server], service)
       #
       # use POST method if the length of the query string is too large
       # see http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/MakingRESTRequests.html
-      if service_string.size > 2000
-        request      = Net::HTTP::Post.new("/")
-        request.body = service_string
+      if service_params.size > 2000
+        if signature_version == '2'
+          # resign the request because HTTP verb is included into signature
+          service_params = signed_service_params(@aws_secret_access_key, service_hash, :post, @params[:server], service)
+        end
+        request      = Net::HTTP::Post.new(service)
+        request.body = service_params
+        request['Content-Type'] = 'application/x-www-form-urlencoded'
       else
-        #params_string = "&#{params_string}" unless params_string.blank?
-        request       = Net::HTTP::Get.new("/?#{service_string}")
+        request = Net::HTTP::Get.new("#{service}?#{service_params}")
       end
       # prepare output hash
       { :request  => request, 
@@ -163,23 +160,44 @@ module RightAws
     def sdb_to_ruby(value)
       value.eql?(@nil_rep) ? nil : value
     end
-    
+
+    # Convert select and query_with_attributes responses to a Ruby language values by replacing the user's chosen string representation of nil with Ruby nil.
+    # (This method affects on a passed response value)
+    def select_response_to_ruby(response) #:nodoc:
+      response[:items].each_with_index do |item, idx|
+        item.each do |key, attributes|
+          attributes.each do |name, values|
+            values.collect! { |value| sdb_to_ruby(value) }
+          end
+        end
+      end
+      response
+    end
+
     # Create query expression from an array.
     # (similar to ActiveRecord::Base#find using :conditions => ['query', param1, .., paramN])
     #
     def query_expression_from_array(params) #:nodoc:
-      unless params.blank?
-        query = params.shift.to_s
-        query.gsub(/(\\)?(\?)/) do
-          if $1 # if escaped '\?' is found - replace it by '?' without backslash
-            "?"
-          else  # well, if no backslash precedes '?' then replace it by next param from the list
-            escape(params.shift)
-          end
+      return '' if params.blank?
+      query = params.shift.to_s
+      query.gsub(/(\\)?(\?)/) do
+        if $1 # if escaped '\?' is found - replace it by '?' without backslash
+          "?"
+        else  # well, if no backslash precedes '?' then replace it by next param from the list
+          escape(params.shift)
         end
       end
     end
-    
+
+    def query_expression_from_hash(hash)
+      return '' if hash.blank?
+      expression = []
+      hash.each do |key, value|
+        expression << "#{key}=#{escape(value)}"
+      end
+      expression.join(' AND ')
+    end
+
     # Retrieve a list of SDB domains from Amazon.
     # 
     # Returns a hash:
@@ -440,6 +458,137 @@ module RightAws
       on_exception
     end
     
+    # Perform a query and fetch specified attributes.
+    # If attributes are not specified then fetches the whole list of attributes.
+    #
+    #
+    # Returns a hash:
+    #   { :box_usage  => string,
+    #     :request_id => string,
+    #     :next_token => string,
+    #     :items      => [ { ItemName1 => { attribute1 => value1, ...  attributeM => valueM } },
+    #                      { ItemName2 => {...}}, ... ]
+    #
+    # Example:
+    #
+    #   sdb.query_with_attributes(domain, ['hobby', 'country'], "['gender'='female'] intersection ['name' starts-with ''] sort 'name'") #=>
+    #     { :request_id => "06057228-70d0-4487-89fb-fd9c028580d3",
+    #       :items =>
+    #         [ { "035f1ba8-dbd8-11dd-80bd-001bfc466dd7"=>
+    #             { "hobby"   => ["cooking", "flowers", "cats"],
+    #               "country" => ["Russia"]}},
+    #           { "0327614a-dbd8-11dd-80bd-001bfc466dd7"=>
+    #             { "hobby"   => ["patchwork", "bundle jumping"],
+    #               "country" => ["USA"]}}, ... ],
+    #        :box_usage=>"0.0000504786"}
+    #
+    #   sdb.query_with_attributes(domain, [], "['gender'='female'] intersection ['name' starts-with ''] sort 'name'") #=>
+    #     { :request_id => "75bb19db-a529-4f69-b86f-5e3800f79a45",
+    #       :items =>
+    #       [ { "035f1ba8-dbd8-11dd-80bd-001bfc466dd7"=>
+    #           { "hobby"   => ["cooking", "flowers", "cats"],
+    #             "name"    => ["Mary"],
+    #             "country" => ["Russia"],
+    #             "gender"  => ["female"],
+    #             "id"      => ["035f1ba8-dbd8-11dd-80bd-001bfc466dd7"]}},
+    #         { "0327614a-dbd8-11dd-80bd-001bfc466dd7"=>
+    #           { "hobby"   => ["patchwork", "bundle jumping"],
+    #             "name"    => ["Mary"],
+    #             "country" => ["USA"],
+    #             "gender"  => ["female"],
+    #             "id"      => ["0327614a-dbd8-11dd-80bd-001bfc466dd7"]}}, ... ],
+    #      :box_usage=>"0.0000506668"}
+    #
+    # see: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?SDB_API_QueryWithAttributes.html
+    #
+    def query_with_attributes(domain_name, attributes=[], query_expression = nil, max_number_of_items = nil, next_token = nil)
+      attributes = attributes.to_a
+      query_expression = query_expression_from_array(query_expression) if query_expression.is_a?(Array)
+      @last_query_expression = query_expression
+      #
+      request_params = { 'DomainName'       => domain_name,
+                         'QueryExpression'  => query_expression,
+                         'MaxNumberOfItems' => max_number_of_items,
+                         'NextToken'        => next_token }
+      attributes.each_with_index do |attribute, idx|
+        request_params["AttributeName.#{idx+1}"] = attribute
+      end
+      link   = generate_request("QueryWithAttributes", request_params)
+      result = select_response_to_ruby(request_info( link, QSdbQueryWithAttributesParser.new ))
+      # return result if no block given
+      return result unless block_given?
+      # loop if block if given
+      begin
+        # the block must return true if it wanna continue
+        break unless yield(result) && result[:next_token]
+        # make new request
+        request_params['NextToken'] = result[:next_token]
+        link   = generate_request("QueryWithAttributes", request_params)
+        result = select_response_to_ruby(request_info( link, QSdbQueryWithAttributesParser.new ))
+      end while true
+    rescue Exception
+      on_exception
+    end
+
+    # Perform SQL-like select and fetch attributes.
+    # Attribute values must be quoted with a single or double quote. If a quote appears within the attribute value, it must be escaped with the same quote symbol as shown in the following example.
+    # (Use array to pass select_expression params to avoid manual escaping).
+    #
+    #  sdb.select(["select * from my_domain where gender=?", 'female']) #=>
+    #    {:request_id =>"8241b843-0fb9-4d66-9100-effae12249ec",
+    #     :items =>
+    #      [ { "035f1ba8-dbd8-11dd-80bd-001bfc466dd7"=>
+    #          {"hobby"   => ["cooking", "flowers", "cats"],
+    #           "name"    => ["Mary"],
+    #           "country" => ["Russia"],
+    #           "gender"  => ["female"],
+    #           "id"      => ["035f1ba8-dbd8-11dd-80bd-001bfc466dd7"]}},
+    #        { "0327614a-dbd8-11dd-80bd-001bfc466dd7"=>
+    #          {"hobby"   => ["patchwork", "bundle jumping"],
+    #           "name"    => ["Mary"],
+    #           "country" => ["USA"],
+    #           "gender"  => ["female"],
+    #           "id"      => ["0327614a-dbd8-11dd-80bd-001bfc466dd7"]}}, ... ]
+    #     :box_usage =>"0.0000506197"}
+    #
+    #   sdb.select('select country, name from my_domain') #=>
+    #    {:request_id=>"b1600198-c317-413f-a8dc-4e7f864a940a",
+    #     :items=>
+    #      [ { "035f1ba8-dbd8-11dd-80bd-001bfc466dd7"=> {"name"=>["Mary"],     "country"=>["Russia"]} },
+    #        { "376d2e00-75b0-11dd-9557-001bfc466dd7"=> {"name"=>["Putin"],    "country"=>["Russia"]} },
+    #        { "0327614a-dbd8-11dd-80bd-001bfc466dd7"=> {"name"=>["Mary"],     "country"=>["USA"]}    },
+    #        { "372ebbd4-75b0-11dd-9557-001bfc466dd7"=> {"name"=>["Bush"],     "country"=>["USA"]}    },
+    #        { "37a4e552-75b0-11dd-9557-001bfc466dd7"=> {"name"=>["Medvedev"], "country"=>["Russia"]} },
+    #        { "38278dfe-75b0-11dd-9557-001bfc466dd7"=> {"name"=>["Mary"],     "country"=>["Russia"]} },
+    #        { "37df6c36-75b0-11dd-9557-001bfc466dd7"=> {"name"=>["Mary"],     "country"=>["USA"]}    } ],
+    #     :box_usage=>"0.0000777663"}
+    #
+    # see: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?SDB_API_Select.html
+    #      http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?UsingSelect.html
+    #      http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?SDBLimits.html
+    #
+    def select(select_expression, next_token = nil)
+      select_expression      = query_expression_from_array(select_expression) if select_expression.is_a?(Array)
+      @last_query_expression = select_expression
+      #
+      request_params = { 'SelectExpression' => select_expression,
+                         'NextToken'        => next_token }
+      link   = generate_request("Select", request_params)
+      result = select_response_to_ruby(request_info( link, QSdbSelectParser.new ))
+      return result unless block_given?
+      # loop if block if given
+      begin
+        # the block must return true if it wanna continue
+        break unless yield(result) && result[:next_token]
+        # make new request
+        request_params['NextToken'] = result[:next_token]
+        link   = generate_request("Select", request_params)
+        result = select_response_to_ruby(request_info( link, QSdbSelectParser.new ))
+      end while true
+    rescue Exception
+      on_exception
+    end
+
     #-----------------------------------------------------------------
     #      PARSERS:
     #-----------------------------------------------------------------
@@ -498,6 +647,52 @@ module RightAws
       end
     end
 
+    class QSdbQueryWithAttributesParser < RightAWSParser #:nodoc:
+      def reset
+        @result = { :items => [] }
+      end
+      def tagend(name)
+        case name
+        when 'Name'
+          case @xmlpath
+          when 'QueryWithAttributesResponse/QueryWithAttributesResult/Item'
+            @item = @text
+            @result[:items] << { @item => {} }
+          when 'QueryWithAttributesResponse/QueryWithAttributesResult/Item/Attribute'
+            @attribute = @text
+            @result[:items].last[@item][@attribute] ||= []
+          end
+        when 'RequestId' then @result[:request_id] = @text
+        when 'BoxUsage'  then @result[:box_usage]  = @text
+        when 'NextToken' then @result[:next_token] = @text
+        when 'Value'     then @result[:items].last[@item][@attribute] << @text
+        end
+      end
+    end
+
+    class QSdbSelectParser < RightAWSParser #:nodoc:
+      def reset
+        @result = { :items => [] }
+      end
+      def tagend(name)
+        case name
+        when 'Name'
+          case @xmlpath
+          when 'SelectResponse/SelectResult/Item'
+            @item = @text
+            @result[:items] << { @item => {} }
+          when 'SelectResponse/SelectResult/Item/Attribute'
+            @attribute = @text
+            @result[:items].last[@item][@attribute] ||= []
+          end
+        when 'RequestId' then @result[:request_id] = @text
+        when 'BoxUsage'  then @result[:box_usage]  = @text
+        when 'NextToken' then @result[:next_token] = @text
+        when 'Value'     then @result[:items].last[@item][@attribute] << @text
+        end
+      end
+    end
+
   end
   
 end