rubber 1.15.0 → 2.0.0.pre1

data/lib/rubber/cloud/aws_table_store.rb

@@ -0,0 +1,127 @@
+require 'json'
+
+module Rubber
+  module Cloud
+    
+    class AwsTableStore
+
+      RETRYABLE_EXCEPTIONS = [Excon::Errors::Error]
+
+      attr_reader :metadata
+
+      def logger
+        Rubber.logger
+      end
+      
+      def initialize(provider, table_key)
+        raise "table_key required" unless table_key && table_key.size > 0
+        
+        @table_provider = provider
+        @table_key = table_key
+        
+        ensure_table_key
+      end
+  
+      # create the table if needed
+      def ensure_table_key()
+        Rubber::Util.retry_on_failure(*RETRYABLE_EXCEPTIONS) do
+          begin
+            @metadata = @table_provider.domain_metadata(@table_key)
+          rescue Excon::Errors::BadRequest
+            @table_provider.create_domain(@table_key)
+            @metadata = @table_provider.domain_metadata(@table_key)
+          end
+        end
+      end
+      
+      def encode(v)
+        [v].to_json
+      end
+      
+      def decode(v)
+        JSON.parse(v).first
+      end
+      
+      def decode_attributes(data)
+        Hash[data.collect {|k, v| [k, decode(v.first)] }]
+      end
+      
+      def put(key, attributes)
+        data = Hash[attributes.collect {|k, v| [k, encode(v)] }]
+        @table_provider.put_attributes(@table_key, key, data, :replace => attributes.keys)
+        return true
+      end
+      
+      def get(key, attributes=[])
+        response = @table_provider.get_attributes(@table_key, key, 'AttributeName' => attributes)
+        data = response.body['Attributes']
+        return decode_attributes(data)
+      end
+  
+      def delete(key, attributes=nil)
+        @table_provider.delete_attributes(@table_key, key, attributes)
+        return true
+      end
+      
+      def find(key=nil, attributes=nil, opts={})
+        query = "select"
+        query << " " + (attributes ? attributes.join(", ") : '*')
+        query << " from `#{@table_key}`"
+        query << " where ItemName = '#{key}'" if key
+        query << " limit " + (opts[:limit] ? opts[:limit].to_s : "200")
+        
+        query_opts = {}
+        query_opts["NextToken"] = opts[:offset].to_s if opts[:offset]
+        
+        response = @table_provider.select(query, query_opts)
+
+        data = response.body['Items']
+        result = TableResponse.new
+        data.each do |name, attribs|
+          result[name] = decode_attributes(attribs)
+        end 
+            
+        result.next_offset = response.body['NextToken']
+        
+        return result
+      end
+      
+      class TableResponse < Hash
+        attr_accessor :next_offset
+        
+        def initialize(*args)
+          super
+        end
+      end
+      
+    end
+
+  end
+end
+
+=begin
+  simpledb limits:
+
+  Domain size	 10 GB per domain
+  Domain size	 1 billion attributes per domain
+  Domain name	 3-255 characters (a-z, A-Z, 0-9, '_', '-', and '.')
+  Domains per account	 250
+  Attribute name-value pairs per item	 256
+  Attribute name length	 1024 bytes
+  Attribute value length	 1024 bytes
+  Item name length	 1024 bytes
+  Attribute name, attribute value, and item name allowed characters	
+  All UTF-8 characters that are valid in XML documents.
+  
+  Control characters and any sequences that are not valid in XML are returned Base64-encoded. For more information, see Working with XML-Restricted Characters.
+  Attributes per PutAttributes operation	256
+  Attributes requested per Select operation	 256
+  Items per BatchPutAttributes operation	25
+  Maximum items in Select response	2500
+  Maximum query execution time	5 seconds
+  Maximum number of unique attributes per Select expression	
+  20
+  Maximum number of comparisons per Select expression	
+  20
+  Maximum response size for Select	1MB
+=end

data/lib/rubber/cloud/fog.rb

@@ -0,0 +1,313 @@
+require 'fog'
+require 'rubber/cloud/fog_storage'
+
+module Rubber
+  module Cloud
+  
+    class Fog < Base
+      
+      attr_reader :compute_provider, :storage_provider
+
+      def initialize(env, capistrano)
+        super(env, capistrano)
+        credentials = Rubber::Util.symbolize_keys(env.credentials)
+        @compute_provider = ::Fog::Compute.new(credentials)
+        @storage_provider = ::Fog::Storage.new(credentials)
+      end
+      
+      def storage(bucket)
+        return Rubber::Cloud::FogStorage.new(@storage_provider, bucket)
+      end
+
+      def table_store(table_key)
+        raise NotImplementedError, "No table store available for generic fog adapter"
+      end
+
+      def create_instance(ami, ami_type, security_groups, availability_zone)
+        response = @compute_provider.servers.create(:image_id => ami,
+                                                    :flavor_id => ami_type,
+                                                    :groups => security_groups,
+                                                    :availability_zone => availability_zone,
+                                                    :key_name => env.key_name)
+        instance_id = response.id
+        return instance_id
+      end
+
+      def create_spot_instance_request(spot_price, ami, ami_type, security_groups, availability_zone)
+        response = @compute_provider.spot_requests.create(:spot_price => spot_price,
+                                                          :image_id => ami,
+                                                          :flavor_id => ami_type,
+                                                          :groups => security_groups,
+                                                          :availability_zone => availability_zone,
+                                                          :key_name => env.key_name)
+        request_id = response.id
+        return request_id
+      end
+
+      def describe_instances(instance_id=nil)
+        instances = []
+        opts = {}
+        opts["instance-id"] = instance_id if instance_id
+
+        response = @compute_provider.servers.all(opts)
+        response.each do |item|
+          instance = {}
+          instance[:id] = item.id
+          instance[:type] = item.flavor_id
+          instance[:external_host] = item.dns_name
+          instance[:external_ip] = item.public_ip_address
+          instance[:internal_host] = item.private_dns_name
+          instance[:internal_ip] = item.private_ip_address
+          instance[:state] = item.state
+          instance[:zone] = item.availability_zone
+          instance[:platform] = item.platform || 'linux'
+          instance[:root_device_type] = item.root_device_type
+          instances << instance
+        end
+
+        return instances
+      end
+
+      def describe_spot_instance_requests(request_id=nil)
+        requests = []
+        opts = {}
+        opts["spot-instance-request-id"] = request_id if request_id
+        response = @compute_provider.spot_requests.all(opts)
+        response.each do |item|
+          request = {}
+          request[:id] = item.id
+          request[:spot_price] = item.price
+          request[:state] = item.state
+          request[:created_at] = item.created_at
+          request[:type] = item.flavor_id
+          request[:image_id] = item.image_id
+          request[:instance_id] = item.instance_id
+          requests << request
+        end
+        return requests
+      end
+
+
+      def destroy_instance(instance_id)
+        response = @compute_provider.servers.get(instance_id).destroy()
+      end
+
+      def destroy_spot_instance_request(request_id)
+        @compute_provider.spot_requests.get(request_id).destroy
+      end
+  
+      def reboot_instance(instance_id)
+        response = @compute_provider.servers.get(instance_id).reboot()
+      end
+
+      def stop_instance(instance_id, force=false)
+        # Don't force the stop process. I.e., allow the instance to flush its file system operations.
+        response = @compute_provider.servers.get(instance_id).stop(force)
+      end
+
+      def start_instance(instance_id)
+        response = @compute_provider.servers.get(instance_id).start()
+      end
+
+      def describe_availability_zones
+        zones = []
+        response = @compute_provider.describe_availability_zones()
+        items = response.body["availabilityZoneInfo"] 
+        items.each do |item|
+          zone = {}
+          zone[:name] = item["zoneName"]
+          zone[:state] =item["zoneState"]
+          zones << zone
+        end
+        return zones
+      end
+
+      def create_security_group(group_name, group_description)
+        @compute_provider.security_groups.create(:name => group_name, :description => group_description)
+      end
+
+      def describe_security_groups(group_name=nil)
+        groups = []
+
+        opts = {}
+        opts["group-name"] = group_name if group_name
+        response = @compute_provider.security_groups.all(opts)
+
+        response.each do |item|
+          group = {}
+          group[:name] = item.name
+          group[:description] = item.description
+
+          item.ip_permissions.each do |ip_item|
+            group[:permissions] ||= []
+            rule = {}
+
+            rule[:protocol] = ip_item["ipProtocol"]
+            rule[:from_port] = ip_item["fromPort"]
+            rule[:to_port] = ip_item["toPort"]
+
+            ip_item["groups"].each do |rule_group|
+              rule[:source_groups] ||= []
+              source_group = {}
+              source_group[:account] = rule_group["userId"]
+              source_group[:name] = rule_group["groupName"]
+              rule[:source_groups] << source_group
+            end if ip_item["groups"]
+
+            ip_item["ipRanges"].each do |ip_range|
+              rule[:source_ips] ||= []
+              rule[:source_ips] << ip_range["cidrIp"]
+            end if ip_item["ipRanges"]
+
+            group[:permissions] << rule
+          end
+
+          groups << group
+          
+        end
+
+        return groups
+      end
+
+      def add_security_group_rule(group_name, protocol, from_port, to_port, source)
+        group = @compute_provider.security_groups.get(group_name)
+        if source.instance_of? Hash
+          group.authorize_group_and_owner(source[:name], source[:account])
+        else
+          group.authorize_port_range(from_port.to_i..to_port.to_i, :ip_protocol => protocol, :cidr_ip => source)
+        end
+      end
+
+      def remove_security_group_rule(group_name, protocol, from_port, to_port, source)
+        group = @compute_provider.security_groups.get(group_name)
+        if source.instance_of? Hash
+          group.revoke_group_and_owner(source[:name], source[:account])
+        else
+          group.revoke_port_range(from_port.to_i..to_port.to_i, :ip_protocol => protocol, :cidr_ip => source)
+        end
+      end
+
+      def destroy_security_group(group_name)
+        @compute_provider.security_groups.get(group_name).destroy
+      end
+
+      def create_static_ip
+        address = @compute_provider.addresses.create()
+        return address.public_ip
+      end
+
+      def attach_static_ip(ip, instance_id)
+        address = @compute_provider.addresses.get(ip)
+        server = @compute_provider.servers.get(instance_id)
+        response = (address.server = server)
+        return ! response.nil?
+      end
+
+      def detach_static_ip(ip)
+        address = @compute_provider.addresses.get(ip)
+        response = (address.server = nil)
+        return ! response.nil?
+      end
+
+      def describe_static_ips(ip=nil)
+        ips = []
+        opts = {}
+        opts["public-ip"] = ip if ip
+        response = @compute_provider.addresses.all(opts)
+        response.each do |item|
+          ip = {}
+          ip[:instance_id] = item.server_id
+          ip[:ip] = item.public_ip
+          ips << ip
+        end
+        return ips
+      end
+
+      def destroy_static_ip(ip)
+        address = @compute_provider.addresses.get(ip)
+        return address.destroy
+      end
+
+      def create_volume(size, zone)
+        volume = @compute_provider.volumes.create(:size => size.to_s, :availability_zone => zone)
+        return volume.id
+      end
+
+      def attach_volume(volume_id, instance_id, device)
+        volume = @compute_provider.volumes.get(volume_id)
+        server = @compute_provider.servers.get(instance_id)
+        volume.device = device
+        volume.server = server
+        return volume.status
+      end
+
+      def detach_volume(volume_id, force=true)
+        volume = @compute_provider.volumes.get(volume_id)
+        force ? volume.force_detach : (volume.server = nil)
+        return volume.status
+      end
+
+      def describe_volumes(volume_id=nil)
+        volumes = []
+        opts = {}
+        opts[:volume-id] = volume_id if volume_id
+        response = @compute_provider.volumes.all(opts)
+        response.each do |item|
+          volume = {}
+          volume[:id] = item.id
+          volume[:status] = item.state
+          if item.server_id
+            attach = item.attachmentSet.item[0]
+            volume[:attachment_instance_id] = item.server_id
+            volume[:attachment_status] = item.attached_at ? "attached" : "waiting"
+          end
+          volumes << volume
+        end
+        return volumes
+      end
+
+      def destroy_volume(volume_id)
+        @compute_provider.volumes.get(volume_id).destroy
+      end
+
+      def create_image(image_name)
+        raise NotImplementedError, "create_image not implemented in generic fog adapter"
+      end
+
+      def describe_images(image_id=nil)
+        images = []
+        opts = {"Owner" => "self"}
+        opts["image-id"] = image_id if image_id
+        response = @compute_provider.images.all(opts)
+        response.each do |item|
+          image = {}
+          image[:id] = item.id
+          image[:location] = item.location
+          image[:root_device_type] = item.root_device_type
+          images << image
+        end
+        return images
+      end
+
+      def destroy_image(image_id)
+        raise NotImplementedError, "destroy_image not implemented in generic fog adapter"
+      end
+
+      def describe_load_balancers(name=nil)
+        raise NotImplementedError, "describe_load_balancers not implemented in generic fog adapter"
+      end
+
+      # resource_id is any Amazon resource ID (e.g., instance ID or volume ID)
+      # tags is a hash of tag_name => tag_value pairs
+      def create_tags(resource_id, tags)
+        # Tags need to be created individually in fog
+        tags.each do |k, v|
+          @compute_provider.tags.create(:resource_id => resource_id,
+                                        :key => k.to_s, :value => v.to_s)
+        end
+      end
+      
+    end
+
+  end
+end

data/lib/rubber/cloud/fog_storage.rb

@@ -0,0 +1,143 @@
+module Rubber
+  module Cloud
+    
+    class FogStorage
+
+      RETRYABLE_EXCEPTIONS = [Excon::Errors::Error]
+
+      def logger
+        Rubber.logger
+      end
+      
+      def initialize(storage_provider, bucket)
+        raise "bucket required" unless bucket && bucket.size > 0
+        
+        @storage_provider = storage_provider
+        @bucket = bucket
+        @directory = @storage_provider.directories.get(@bucket)
+      end
+  
+      # create the bucket if needed
+      def ensure_bucket()
+        Rubber::Util.retry_on_failure(*RETRYABLE_EXCEPTIONS) do
+          @directory = @storage_provider.directories.create(:key => @bucket) unless @directory
+        end
+        return self
+      end
+    
+      # data can be a string or io handle
+      def store(key, data, opts={})
+        if data.respond_to?(:read)
+          multipart_store(key, data, opts)
+        else
+          singlepart_store(key, data, opts)
+        end
+      end
+  
+      def singlepart_store(key, data, opts={})
+        raise "a key is required" unless key && key.size > 0
+        
+        file = nil
+  
+        # store the object
+        logger.debug "Storing object: #{key}"
+  
+        ensure_bucket()
+        
+        Rubber::Util.retry_on_failure(*RETRYABLE_EXCEPTIONS) do
+          file = @directory.files.new(opts.merge(:key => key))
+          file.body = data
+          file.save
+        end
+  
+        file
+      end
+  
+      def multipart_store(key, data, opts={})
+        raise "a key is required" unless key && key.size > 0
+        
+        opts = {:chunk_size => (5 * 10**6)}.merge(opts)
+  
+        chunk = data.read(opts[:chunk_size])
+  
+        if chunk.size < opts[:chunk_size]
+          singlepart_store(key, chunk, opts)
+        else
+          logger.info "Multipart uploading #{key}"
+  
+          part_ids = []
+  
+          ensure_bucket()
+          
+          response = @storage_provider.initiate_multipart_upload(@bucket, key)
+          begin
+            upload_id = response.body['UploadId']
+  
+            while chunk
+              next_chunk = data.read(opts[:chunk_size])
+              if data.eof?
+                chunk << next_chunk
+                next_chunk = nil
+              end
+              part_number = part_ids.size + 1
+              logger.info("Uploading part #{part_number}")
+              Rubber::Util.retry_on_failure(*RETRYABLE_EXCEPTIONS) do
+                response = @storage_provider.upload_part(@bucket, key, upload_id, part_number, chunk)
+                part_ids << response.headers['ETag']
+              end
+              chunk = next_chunk
+            end
+  
+            @storage_provider.complete_multipart_upload(@bucket, key, upload_id, part_ids)
+            logger.info("Completed multipart upload: #{upload_id}")
+  
+          rescue Exception => e
+            logger.error("Aborting multipart upload: #{upload_id}")
+            @storage_provider.abort_multipart_upload(@bucket, key, upload_id)
+            raise
+          end
+        end
+      end
+  
+      def fetch(key, opts={}, &block)
+        raise "a key is required" unless key && key.size > 0
+        
+        if block_given?
+          @directory.files.get(key, opts, &block)
+        else
+          Rubber::Util.retry_on_failure(*RETRYABLE_EXCEPTIONS) do
+            begin
+              file = @directory.files.get(key, opts)
+              file.body if file
+            rescue Excon::Errors::NotFound
+              nil
+            end
+          end
+        end
+      end
+      
+      def delete(key, opts={})
+        raise "a key is required" unless key && key.size > 0
+        
+        # store the object
+        logger.debug "Deleting object: #{key}"
+        Rubber::Util.retry_on_failure(*RETRYABLE_EXCEPTIONS) do
+          file = @directory.files.get(key, opts);
+          file.destroy if file
+        end
+      end
+  
+      def walk_tree(prefix=nil)
+        # Grab a new bucket connection since this method destroys
+        # the cached copy of files once the prefix is applied.
+        Rubber::Util.retry_on_failure(*RETRYABLE_EXCEPTIONS) do
+          @storage_provider.directories.get(@bucket, :prefix => prefix).files.each do |f|
+            yield f
+          end
+        end
+      end
+  
+    end
+
+  end
+end