staugaard-cloudmaster 0.1.1

data/lib/OriginalAWS/SQS.rb

@@ -0,0 +1,159 @@
+# Sample Ruby code for the O'Reilly book "Programming Amazon Web
+# Services" by James Murty.
+#
+# This code was written for Ruby version 1.8.6 or greater.
+#
+# The SQS module implements the Query API of the Amazon Simple Queue
+# Service.
+require 'AWS'
+
+class SQS
+  include AWS # Include the AWS module as a mixin
+
+  ENDPOINT_URI = URI.parse("https://queue.amazonaws.com/")
+  API_VERSION = '2008-01-01'
+  SIGNATURE_VERSION = '1'
+
+  HTTP_METHOD = 'POST' # 'GET'
+
+
+  def list_queues(queue_name_prefix=nil)
+    parameters = build_query_params(API_VERSION, SIGNATURE_VERSION,
+      {
+      'Action' => 'ListQueues',
+      'QueueNamePrefix' => queue_name_prefix
+      })
+
+    response = do_query(HTTP_METHOD, ENDPOINT_URI, parameters)
+
+    queue_names = []
+    xml_doc = REXML::Document.new(response.body)
+
+    xml_doc.elements.each('//QueueUrl') do |queue_url|
+      queue_names << queue_url.text
+    end
+
+    return queue_names
+  end
+
+  def create_queue(queue_name, visibility_timeout_secs=nil)
+    parameters = build_query_params(API_VERSION, SIGNATURE_VERSION,
+      {
+      'Action' => 'CreateQueue',
+      'QueueName' => queue_name,
+      'DefaultVisibilityTimeout' => visibility_timeout_secs
+      })
+
+    response = do_query(HTTP_METHOD, ENDPOINT_URI, parameters)
+
+    xml_doc = REXML::Document.new(response.body)
+    return xml_doc.elements['//QueueUrl'].text
+  end
+
+  def delete_queue(queue_url)
+    parameters = build_query_params(API_VERSION, SIGNATURE_VERSION,
+      {
+      'Action' => 'DeleteQueue'
+      })
+
+    response = do_query(HTTP_METHOD, URI.parse(queue_url), parameters)
+
+    return true
+  end
+
+  def get_queue_attributes(queue_url, attribute='All')
+    parameters = build_query_params(API_VERSION, SIGNATURE_VERSION,
+      {
+      'Action' => 'GetQueueAttributes',
+      'AttributeName' => attribute
+      })
+
+    response = do_query(HTTP_METHOD, URI.parse(queue_url), parameters)
+
+    attributes = {}
+    xml_doc = REXML::Document.new(response.body)
+
+    xml_doc.elements.each('//Attribute') do |att|
+      name = att.elements['Name'].text
+      # All currently supported attributes have integer values
+      value = att.elements['Value'].text.to_i
+
+      attributes[name] = value
+    end
+
+    return attributes
+  end
+
+  def set_queue_attribute(queue_url, value, attribute='VisibilityTimeout')
+    parameters = build_query_params(API_VERSION, SIGNATURE_VERSION,
+      {
+      'Action' => 'SetQueueAttributes',
+      'Attribute.Name' => attribute,
+      'Attribute.Value' => value
+      })
+
+    response = do_query(HTTP_METHOD, URI.parse(queue_url), parameters)
+    return true
+  end
+
+  def send_message(queue_url, message_body, encode=false)
+    message_body = encode_base64(message_body) if encode
+
+    parameters = build_query_params(API_VERSION, SIGNATURE_VERSION,
+      {
+      'Action' => 'SendMessage',
+      'MessageBody' => message_body
+      })
+
+    response = do_query(HTTP_METHOD, URI.parse(queue_url), parameters)
+
+    xml_doc = REXML::Document.new(response.body)
+    return xml_doc.elements['//MessageId'].text
+  end
+
+  def receive_messages(queue_url, maximum=1, visibility_timeout=nil)
+    parameters = build_query_params(API_VERSION, SIGNATURE_VERSION,
+      {
+      'Action' => 'ReceiveMessage',
+      'MaxNumberOfMessages' => maximum,
+      'VisibilityTimeout' => visibility_timeout
+      })
+
+    response = do_query(HTTP_METHOD, URI.parse(queue_url), parameters)
+    msgs = []
+
+    xml_doc = REXML::Document.new(response.body)
+
+    xml_doc.elements.each('//Message') do |msg|
+      msgs << {
+        :id => msg.elements['MessageId'].text,
+        :receipt_handle => msg.elements['ReceiptHandle'].text,
+        :md5_of_body => msg.elements['MD5OfBody'].text,
+        :body => msg.elements['Body'].text
+      }
+    end
+
+    return msgs
+  end
+
+  def delete_message(queue_url, receipt_handle)
+    parameters = build_query_params(API_VERSION, SIGNATURE_VERSION,
+      {
+      'Action' => 'DeleteMessage',
+      'ReceiptHandle' => receipt_handle
+      })
+
+    response = do_query(HTTP_METHOD, URI.parse(queue_url), parameters)
+
+    return true
+  end
+
+  def receive_new_messages(msgs, queue_url, max_count=1)
+    new_msgs = receive_messages(queue_url, max_count)
+    new_msgs.each do |new|
+      msgs << new unless msgs.find{|old| old[:id] == new[:id]}
+    end
+    return msgs
+  end
+
+end

data/lib/OriginalAWS/SimpleDB.rb

@@ -0,0 +1,460 @@
+# Sample Ruby code for the O'Reilly book "Using AWS Infrastructure
+# Services" by James Murty.
+#
+# This code was written for Ruby version 1.8.6 or greater.
+#
+# The SimpleDB module implements the Query API of the Amazon SimpleDB
+# Service.
+
+require 'AWS'
+require 'bigdecimal'
+
+class SimpleDB
+  include AWS # Include the AWS module as a mixin
+
+  ENDPOINT_URI = URI.parse("https://sdb.amazonaws.com/")
+  API_VERSION = '2007-11-07'
+  SIGNATURE_VERSION = '1'
+
+  HTTP_METHOD = 'POST' # 'GET'
+
+  attr_reader :prior_box_usage
+  attr_reader :total_box_usage
+
+
+  def do_sdb_query(parameters)
+    response = do_query(HTTP_METHOD, ENDPOINT_URI, parameters)
+    xml_doc = REXML::Document.new(response.body)
+
+    @total_box_usage = 0 if @total_box_usage.nil?
+
+    @prior_box_usage = xml_doc.elements['//BoxUsage'].text.to_f
+    @total_box_usage += @prior_box_usage
+
+    return xml_doc
+  end
+
+
+  def encode_boolean(value)
+    if value
+      return '!b'
+    else
+      return '!B'
+    end
+  end
+
+
+  def decode_boolean(value_str)
+    if value_str == '!B'
+      return false
+    elsif value_str == '!b'
+      return true
+    else
+      raise "Cannot decode boolean from string: #{value_str}"
+    end
+  end
+
+
+  def encode_date(value)
+    return "!d" + value.getutc.iso8601
+  end
+
+
+  def decode_date(value_str)
+    if value_str[0..1] == '!d'
+      return Time.parse(value_str[2..-1])
+    else
+      raise "Cannot decode date from string: #{value_str}"
+    end
+  end
+
+
+  def encode_integer(value, max_digits=18)
+    upper_bound = (10 ** max_digits)
+
+    if value >= upper_bound or value < -upper_bound
+      raise "Integer #{value} is outside encoding range (-#{upper_bound} " +
+        "to #{upper_bound - 1})"
+    end
+
+    if value < 0
+      return "!I" + format("%0#{max_digits}d", upper_bound + value)
+    else
+      return "!i" + format("%0#{max_digits}d", value)
+    end
+  end
+
+
+  def decode_integer(value_str)
+    if value_str[0..1] == '!I'
+      # Encoded value is a negative integer
+      max_digits = value_str.size - 2
+      upper_bound = (10 ** max_digits)
+
+      return value_str[2..-1].to_i - upper_bound
+    elsif value_str[0..1] == '!i'
+      # Encoded value is a positive integer
+      return value_str[2..-1].to_i
+    else
+      raise "Cannot decode integer from string: #{value_str}"
+    end
+  end
+
+
+  def encode_float(value, max_exp_digits=2, max_precision_digits=15)
+    exp_midpoint = (10 ** max_exp_digits) / 2
+
+    sign, fraction, base, exponent = BigDecimal(value.to_s).split
+
+    if exponent >= exp_midpoint or exponent < -exp_midpoint
+      raise "Exponent #{exponent} is outside encoding range " +
+        "(-#{exp_midpoint} " + "to #{exp_midpoint - 1})"
+    end
+
+    if fraction.size > max_precision_digits
+      # Round fraction value if it exceeds allowed precision.
+      fraction_str = fraction[0...max_precision_digits] + '.' +
+                     fraction[max_precision_digits..-1]
+      fraction = BigDecimal(fraction_str).round(0).split[1]
+    elsif fraction.size < max_precision_digits
+      # Right-pad fraction with zeros if it is too short.
+      fraction = fraction + ('0' * (max_precision_digits - fraction.size))
+    end
+
+    # The zero value is a special case, for which the exponent must be 0
+    exponent = -exp_midpoint if value == 0
+
+    if sign == 1
+      return format("!f%0#{max_exp_digits}d", exp_midpoint + exponent) +
+        format("!%0#{max_precision_digits}d", fraction.to_i)
+    else
+      fraction_upper_bound = (10 ** max_precision_digits)
+      diff_fraction = fraction_upper_bound - BigDecimal(fraction)
+      return format("!F%0#{max_exp_digits}d", exp_midpoint - exponent) +
+        format("!%0#{max_precision_digits}d", diff_fraction)
+    end
+  end
+
+
+  def decode_float(value_str)
+    prefix = value_str[0..1]
+
+    if prefix != '!f' and prefix != '!F'
+      raise "Cannot decode float from string: #{value_str}"
+    end
+
+    value_str =~ /![fF]([0-9]+)!([0-9]+)/
+    exp_str = $1
+    fraction_str = $2
+
+    max_exp_digits = exp_str.size
+    exp_midpoint = (10 ** max_exp_digits) / 2
+    max_precision_digits = fraction_str.size
+
+    if prefix == '!F'
+      sign = -1
+      exp = exp_midpoint - exp_str.to_i
+
+      fraction_upper_bound = (10 ** max_precision_digits)
+      fraction = fraction_upper_bound - BigDecimal(fraction_str)
+    else
+      sign = 1
+      exp = exp_str.to_i - exp_midpoint
+
+      fraction = BigDecimal(fraction_str)
+    end
+
+    return sign * "0.#{fraction.to_i}".to_f * (10 ** exp)
+  end
+
+
+  def encode_attribute_value(value)
+    if value == true or value == false
+      return encode_boolean(value)
+    elsif value.is_a? Time
+      return encode_date(value)
+    elsif value.is_a? Integer
+      return encode_integer(value)
+    elsif value.is_a? Numeric
+      return encode_float(value)
+    else
+      # No type-specific encoding is available, so we simply convert
+      # the value to a string.
+      return value.to_s
+    end
+  end
+
+
+  def decode_attribute_value(value_str)
+    return '' if value_str.nil?
+
+    # Check whether the '!' flag is present to indicate an encoded value
+    return value_str if value_str[0..0] != '!'
+
+    prefix = value_str[0..1].downcase
+    if prefix == '!b'
+      return decode_boolean(value_str)
+    elsif prefix == '!d'
+      return decode_date(value_str)
+    elsif prefix == '!i'
+      return decode_integer(value_str)
+    elsif prefix == '!f'
+      return decode_float(value_str)
+    else
+      return value_str
+    end
+  end
+
+
+  def list_domains(max_domains=100)
+    more_domains = true
+    next_token = nil
+    domain_names = []
+
+    while more_domains
+      parameters = build_query_params(API_VERSION, SIGNATURE_VERSION,
+        {
+        'Action' => 'ListDomains',
+        'MaxNumberOfDomains' => max_domains,
+        'NextToken' => next_token
+        })
+
+      xml_doc = do_sdb_query(parameters)
+
+      xml_doc.elements.each('//DomainName') do |name|
+        domain_names << name.text
+      end
+
+      # If we receive a NextToken element, perform a follow-up operation
+      # to retrieve the next set of domain names.
+      next_token = xml_doc.elements['//NextToken/text()']
+      more_domains = !next_token.nil?
+    end
+
+    return domain_names
+  end
+
+
+  def create_domain(domain_name)
+    parameters = build_query_params(API_VERSION, SIGNATURE_VERSION,
+      {
+      'Action' => 'CreateDomain',
+      'DomainName' => domain_name
+      })
+
+    do_sdb_query(parameters)
+    return true
+  end
+
+
+  def delete_domain(domain_name)
+    parameters = build_query_params(API_VERSION, SIGNATURE_VERSION,
+      {
+      'Action' => 'DeleteDomain',
+      'DomainName' => domain_name
+      })
+
+    do_sdb_query(parameters)
+    return true
+  end
+
+
+  def build_attribute_params(attributes={}, replace=false)
+    attribute_params = {}
+    index = 0
+
+    attributes.each do |attrib_name, attrib_value|
+      attrib_value = [attrib_value] if not attrib_value.is_a? Array
+
+      attrib_value.each do |value|
+        attribute_params["Attribute.#{index}.Name"] = attrib_name
+        if not value.nil?
+          if respond_to? :encode_attribute_value
+            # Automatically encode attribute values if the method
+            # encode_attribute_value is available in this class
+            value = encode_attribute_value(value)
+          end
+          attribute_params["Attribute.#{index}.Value"] = value
+        end
+        # Add a Replace parameter for the attribute if the replace flag is set
+        attribute_params["Attribute.#{index}.Replace"] = 'true' if replace
+        index += 1
+      end if attrib_value
+    end
+
+    return attribute_params
+  end
+
+
+  def put_attributes(domain_name, item_name, attributes, replace=false)
+    parameters = build_query_params(API_VERSION, SIGNATURE_VERSION,
+      {
+      'Action' => 'PutAttributes',
+      'DomainName' => domain_name,
+      'ItemName' => item_name
+      })
+
+    parameters.merge!(build_attribute_params(attributes, replace))
+
+    do_sdb_query(parameters)
+    return true
+  end
+
+
+  def delete_attributes(domain_name, item_name, attributes={})
+    parameters = build_query_params(API_VERSION, SIGNATURE_VERSION,
+      {
+      'Action' => 'DeleteAttributes',
+      'DomainName' => domain_name,
+      'ItemName' => item_name
+      })
+
+    parameters.merge!(build_attribute_params(attributes))
+
+    do_sdb_query(parameters)
+    return true
+  end
+
+
+  def get_attributes(domain_name, item_name, attribute_name=nil)
+    parameters = build_query_params(API_VERSION, SIGNATURE_VERSION,
+      {
+      'Action' => 'GetAttributes',
+      'DomainName' => domain_name,
+      'ItemName' => item_name,
+      'AttributeName' => attribute_name
+      })
+
+    xml_doc = do_sdb_query(parameters)
+
+    attributes = {}
+    xml_doc.elements.each('//Attribute') do |attribute_node|
+      attr_name = attribute_node.elements['Name'].text
+      value = attribute_node.elements['Value'].text
+
+      if respond_to? :decode_attribute_value
+        # Automatically decode attribute values if the method
+        # decode_attribute_value is available in this class
+        value = decode_attribute_value(value)
+      end
+
+      # An empty attribute value is an empty string, not nil.
+      value = '' if value.nil?
+
+      if attributes.has_key?(attr_name)
+        attributes[attr_name] << value
+      else
+        attributes[attr_name] = [value]
+      end
+    end
+
+    if not attribute_name.nil?
+      # If a specific attribute was requested, return only the values array
+      # for this attribute.
+      if not attributes[attribute_name]
+        return []
+      else
+        return attributes[attribute_name]
+      end
+    else
+      return attributes
+    end
+  end
+
+
+  def query(domain_name, query_expression=nil, options={:fetch_all=>true})
+    more_items = true
+    next_token = nil
+    item_names = []
+
+    while more_items
+      parameters = build_query_params(API_VERSION, SIGNATURE_VERSION,
+        {
+        'Action' => 'Query',
+        'DomainName' => domain_name,
+        'QueryExpression' => query_expression,
+        'MaxNumberOfItems' => options[:max_items],
+        'NextToken' => next_token
+        })
+
+      xml_doc = do_sdb_query(parameters)
+
+      xml_doc.elements.each('//ItemName') do |item_name|
+        item_names << item_name.text
+      end
+
+      if xml_doc.elements['//NextToken']
+        next_token = xml_doc.elements['//NextToken'].text.gsub("\n","")
+        more_items = options[:fetch_all]
+      else
+        more_items = false
+      end
+    end
+
+    return item_names
+  end
+
+  # The query with attributes feature was added to the S3 API after the 
+  # release of "Programming Amazon Web Services" so it is not discussed in 
+  # the book's text. For more details, see: 
+  # http://www.jamesmurty.com/2008/09/07/samples-for-simpledb-querywithattributes/
+  def query_with_attributes(domain_name, query_expression=nil, 
+                            attribute_names=[], options={:fetch_all=>true})
+    more_items = true
+    next_token = nil
+    items = []
+
+    while more_items
+      parameters = build_query_params(API_VERSION, SIGNATURE_VERSION,
+        {
+        'Action' => 'QueryWithAttributes',
+        'DomainName' => domain_name,
+        'QueryExpression' => query_expression,
+        'MaxNumberOfItems' => options[:max_items],
+        'NextToken' => next_token
+        },{
+        'AttributeName' => attribute_names,
+        })
+
+      xml_doc = do_sdb_query(parameters)
+
+      xml_doc.elements.each('//Item') do |item_node|
+        item = {'name' => item_node.elements['Name'].text}
+            
+        attributes = {}
+        item_node.elements.each('Attribute') do |attribute_node|
+          attr_name = attribute_node.elements['Name'].text
+          value = attribute_node.elements['Value'].text
+
+          if respond_to? :decode_attribute_value
+            # Automatically decode attribute values if the method
+            # decode_attribute_value is available in this class
+            value = decode_attribute_value(value)
+          end
+
+          # An empty attribute value is an empty string, not nil.
+          value = '' if value.nil?
+
+          if attributes.has_key?(attr_name)
+            attributes[attr_name] << value
+          else
+            attributes[attr_name] = [value]
+          end
+        end
+    
+        item['attributes'] = attributes
+        items << item
+      end
+
+      if xml_doc.elements['//NextToken']
+        next_token = xml_doc.elements['//NextToken'].text.gsub("\n","")
+        more_items = options[:fetch_all]
+      else
+        more_items = false
+      end
+    end
+
+    return items
+  end
+
+end

data/lib/RetryAWS/EC2.rb

@@ -0,0 +1,88 @@
+require 'AWS/EC2'
+
+# RetryAWS wraps the AWS module in exception catcher blocks, so that any
+# exceptions that are thrown do not affect the caller.
+#
+# The RetryEC2, RetrySQS, and RetryS3 log any errors that they encounter, so
+# that they can be examined later.
+module RetryAWS
+  # Wrap EC2 functions that we use.
+  # Catch errors and do something reasonable.
+  class EC2
+    def initialize(access_key, secret_key)
+      @ec2 = AWS::EC2.new(access_key, secret_key)
+      @@log = STDOUT
+      @retry_limit = 16
+    end
+
+    def logger=(logger)
+      @@log = logger
+    end
+
+    private
+
+    def report_error(res)
+      @@log.puts "error #{$!}"
+      $@.each {|line| @@log.puts "  #{line}"}
+      res
+    end
+
+    def retry?(err, retry_time)
+      if err.response.code.to_i >= 500 && retry_time < @retry_limit
+        sleep retry_time
+	return retry_time * 2
+      end
+      nil
+    end
+
+    public 
+
+    def describe_images(options={})
+      retry_time = 1
+      begin
+        @ec2.describe_images(options)
+      rescue AWS::ServiceError => err
+        retry if retry_time = retry?(err, retry_time)
+        report_error []
+      rescue
+        report_error []
+      end
+    end
+    
+    def describe_instances(*instance_ids)
+      retry_time = 1
+      begin
+        @ec2.describe_instances(*instance_ids)
+      rescue AWS::ServiceError => err
+        retry if retry_time = retry?(err, retry_time)
+        report_error []
+      rescue
+        report_error []
+      end
+    end
+
+    def run_instances(image_id, min_count=1, max_count=min_count, options={})
+      retry_time = 1
+      begin
+        @ec2.run_instances(image_id, min_count, max_count, options)
+      rescue AWS::ServiceError => err
+        retry if retry_time = retry?(err, retry_time)
+        report_error []
+      rescue
+        report_error []
+      end
+    end
+
+    def terminate_instances(*instance_ids)
+      retry_time = 1
+      begin
+        @ec2.terminate_instances(*instance_ids)
+      rescue AWS::ServiceError => err
+        retry if retry_time = retry?(err, retry_time)
+        report_error []
+      rescue
+        report_error []
+      end
+    end
+  end
+end