fog-softlayer 0.0.5 → 0.0.7

data/lib/fog/softlayer/requests/storage/head_container.rb

@@ -0,0 +1,28 @@
+module Fog
+  module Storage
+    class Softlayer
+      class Real
+
+        # List number of objects and total bytes stored
+        #
+        # ==== Parameters
+        # * container<~String> - Name of container to retrieve info for
+        #
+        # ==== Returns
+        # * response<~Excon::Response>:
+        #   * headers<~Hash>:
+        #     * 'X-Container-Object-Count'<~String> - Count of containers
+        #     * 'X-Container-Bytes-Used'<~String>   - Bytes used
+        def head_container(container)
+          request(
+            :expects  => 204,
+            :method   => 'HEAD',
+            :path     => Fog::Softlayer.escape(container),
+            :query    => {'format' => 'json'}
+          )
+        end
+
+      end
+    end
+  end
+end

data/lib/fog/softlayer/requests/storage/head_containers.rb

@@ -0,0 +1,25 @@
+module Fog
+  module Storage
+    class Softlayer
+      class Real
+
+        # List number of containers and total bytes stored
+        #
+        # ==== Returns
+        # * response<~Excon::Response>:
+        #   * headers<~Hash>:
+        #     * 'X-Account-Container-Count'<~String> - Count of containers
+        #     * 'X-Account-Bytes-Used'<~String> - Bytes used
+        def head_containers
+          request(
+            :expects  => 204,
+            :method   => 'HEAD',
+            :path     => '',
+            :query    => {'format' => 'json'}
+          )
+        end
+
+      end
+    end
+  end
+end

data/lib/fog/softlayer/requests/storage/head_object.rb

@@ -0,0 +1,23 @@
+module Fog
+  module Storage
+    class Softlayer
+      class Real
+
+        # Get headers for object
+        #
+        # ==== Parameters
+        # * container<~String> - Name of container to look in
+        # * object<~String> - Name of object to look for
+        #
+        def head_object(container, object)
+          request({
+            :expects  => 200,
+            :method   => 'HEAD',
+            :path     => "#{Fog::Softlayer.escape(container)}/#{Fog::Softlayer.escape(object)}"
+          }, false)
+        end
+
+      end
+    end
+  end
+end

data/lib/fog/softlayer/requests/storage/post_set_meta_temp_url_key.rb

@@ -0,0 +1,37 @@
+module Fog
+  module Storage
+    class Softlayer
+
+      class Real
+
+        # Set the account wide Temp URL Key. This is a secret key that's
+        # used to generate signed expiring URLs.
+        #
+        # Once the key has been set with this request you should create new
+        # Storage objects with the :temp_url_key option then use
+        # the get_object_https_url method to generate expiring URLs.
+        #
+        # *** CAUTION *** changing this secret key will invalidate any expiring
+        # URLS generated with old keys.
+        #
+        # ==== Parameters
+        # * key<~String> - The new Temp URL Key
+        #
+        # ==== Returns
+        # * response<~Excon::Response>
+        #
+        # ==== See Also
+        # http://docs.rackspace.com/files/api/v1/cf-devguide/content/Set_Account_Metadata-d1a4460.html
+        def post_set_meta_temp_url_key(key)
+          request(
+            :expects  => [201, 202, 204],
+            :method   => 'POST',
+            :headers  => {'X-Account-Meta-Temp-Url-Key' => key}
+          )
+        end
+
+      end
+
+    end
+  end
+end

data/lib/fog/softlayer/requests/storage/put_container.rb

@@ -0,0 +1,32 @@
+module Fog
+  module Storage
+    class Softlayer
+      class Mock
+        def put_container(name)
+          @containers[name] = {} unless @containers[name]
+          response = Excon::Response.new
+          response.body = ''
+          response.status = 201
+          response
+        end
+      end
+
+      class Real
+
+        # Create a new container
+        #
+        # ==== Parameters
+        # * name<~String> - Name for container, should be < 256 bytes and must not contain '/'
+        #
+        def put_container(name)
+          request(
+            :expects  => [201, 202],
+            :method   => 'PUT',
+            :path     => Fog::Softlayer.escape(name)
+          )
+        end
+
+      end
+    end
+  end
+end

data/lib/fog/softlayer/requests/storage/put_dynamic_obj_manifest.rb

@@ -0,0 +1,43 @@
+module Fog
+  module Storage
+    class Softlayer
+      class Real
+
+        # Create a new dynamic large object manifest
+        #
+        # Creates an object with a +X-Object-Manifest+ header that specifies the common prefix ("<container>/<prefix>")
+        # for all uploaded segments. Retrieving the manifest object streams all segments matching this prefix.
+        # Segments must sort in the order they should be concatenated. Note that any future objects stored in the container
+        # along with the segments that match the prefix will be included when retrieving the manifest object.
+        #
+        # All segments must be stored in the same container, but may be in a different container than the manifest object.
+        # The default +X-Object-Manifest+ header is set to "+container+/+object+", but may be overridden in +options+
+        # to specify the prefix and/or the container where segments were stored.
+        # If overridden, names should be CGI escaped (excluding spaces) if needed (see {Fog::Softlayer.escape}).
+        #
+        # @param container [String] Name for container where +object+ will be stored. Should be < 256 bytes and must not contain '/'
+        # @param object [String] Name for manifest object.
+        # @param options [Hash] Config headers for +object+.
+        # @option options [String] 'X-Object-Manifest' ("container/object") "<container>/<prefix>" for segment objects.
+        #
+        # @raise [Fog::Storage::Softlayer::NotFound] HTTP 404
+        # @raise [Excon::Errors::BadRequest] HTTP 400
+        # @raise [Excon::Errors::Unauthorized] HTTP 401
+        # @raise [Excon::Errors::HTTPStatusError]
+        #
+        # @see http://docs.openstack.org/api/openstack-object-storage/1.0/content/dynamic-large-object-creation.html
+        def put_dynamic_obj_manifest(container, object, options = {})
+          path = "#{Fog::Softlayer.escape(container)}/#{Fog::Softlayer.escape(object)}"
+          headers = {'X-Object-Manifest' => path}.merge(options)
+          request(
+            :expects  => 201,
+            :headers  => headers,
+            :method   => 'PUT',
+            :path     => path
+          )
+        end
+
+      end
+    end
+  end
+end

data/lib/fog/softlayer/requests/storage/put_object.rb

@@ -0,0 +1,61 @@
+module Fog
+  module Storage
+    class Softlayer
+      class Mock
+        def put_object(container, object, data, options = {}, &block)
+          if @containers[container]
+            @containers[container][object] = data
+            response = Excon::Response.new
+            response.body = ''
+            response.status = 201
+            response.headers = {"Last-Modified"=>Time.now, "Content-Length"=>0}
+            response
+          else
+            response = Excon::Response.new
+            response.body = '<html><h1>Not Found</h1><p>The resource could not be found.</p></html>'
+            response.status = 404
+            response.headers = {"Content-Length"=>"70", "Content-Type"=>"text/html; charset=UTF-8", "X-Trans-Id"=>"abcdefghijklmnopqrstuvwx-0123456789", "Date"=>Time.now}
+            response
+          end
+        end
+      end
+
+      class Real
+
+        # Create a new object
+        #
+        # When passed a block, it will make a chunked request, calling
+        # the block for chunks until it returns an empty string.
+        # In this case the data parameter is ignored.
+        #
+        # ==== Parameters
+        # * container<~String> - Name for container, should be < 256 bytes and must not contain '/'
+        # * object<~String> - Name for object
+        # * data<~String|File> - data to upload
+        # * options<~Hash> - config headers for object. Defaults to {}.
+        # * block<~Proc> - chunker
+        #
+        def put_object(container, object, data, options = {}, &block)
+          if block_given?
+            params = { :request_block => block }
+            headers = options
+          else
+            data = Fog::Storage.parse_data(data)
+            headers = data[:headers].merge!(options)
+            params = { :body => data[:body] }
+          end
+
+          params.merge!(
+            :expects    => 201,
+            :idempotent => !params[:request_block],
+            :headers    => headers,
+            :method     => 'PUT',
+            :path       => "#{Fog::Softlayer.escape(container)}/#{Fog::Softlayer.escape(object)}"
+          )
+
+          request(params)
+        end
+      end
+    end
+  end
+end

data/lib/fog/softlayer/requests/storage/put_object_manifest.rb

@@ -0,0 +1,16 @@
+module Fog
+  module Storage
+    class Softlayer
+      class Real
+
+        # Create a new dynamic large object manifest
+        #
+        # This is an alias for {#put_dynamic_obj_manifest} for backward compatibility.
+        def put_object_manifest(container, object, options = {})
+          put_dynamic_obj_manifest(container, object, options)
+        end
+
+      end
+    end
+  end
+end

data/lib/fog/softlayer/requests/storage/put_static_obj_manifest.rb

@@ -0,0 +1,57 @@
+module Fog
+  module Storage
+    class Softlayer
+      class Real
+
+        # Create a new static large object manifest.
+        #
+        # A static large object is similar to a dynamic large object. Whereas a GET for a dynamic large object manifest
+        # will stream segments based on the manifest's +X-Object-Manifest+ object name prefix, a static large object
+        # manifest streams segments which are defined by the user within the manifest. Information about each segment is
+        # provided in +segments+ as an Array of Hash objects, ordered in the sequence which the segments should be streamed.
+        #
+        # When the SLO manifest is received, each segment's +etag+ and +size_bytes+ will be verified.
+        # The +etag+ for each segment is returned in the response to {#put_object}, but may also be calculated.
+        # e.g. +Digest::MD5.hexdigest(segment_data)+
+        #
+        # The maximum number of segments for a static large object is 1000, and all segments (except the last) must be
+        # at least 1 MiB in size. Unlike a dynamic large object, segments are not required to be in the same container.
+        #
+        # @example
+        #   segments = [
+        #     { :path => 'segments_container/first_segment',
+        #       :etag => 'md5 for first_segment',
+        #       :size_bytes => 'byte size of first_segment' },
+        #     { :path => 'segments_container/second_segment',
+        #       :etag => 'md5 for second_segment',
+        #       :size_bytes => 'byte size of second_segment' }
+        #   ]
+        #   put_static_obj_manifest('my_container', 'my_large_object', segments)
+        #
+        # @param container [String] Name for container where +object+ will be stored.
+        #     Should be < 256 bytes and must not contain '/'
+        # @param object [String] Name for manifest object.
+        # @param segments [Array<Hash>] Segment data for the object.
+        # @param options [Hash] Config headers for +object+.
+        #
+        # @raise [Fog::Storage::Softlayer::NotFound] HTTP 404
+        # @raise [Excon::Errors::BadRequest] HTTP 400
+        # @raise [Excon::Errors::Unauthorized] HTTP 401
+        # @raise [Excon::Errors::HTTPStatusError]
+        #
+        # @see http://docs.openstack.org/api/openstack-object-storage/1.0/content/static-large-objects.html
+        def put_static_obj_manifest(container, object, segments, options = {})
+          request(
+            :expects  => 201,
+            :method   => 'PUT',
+            :headers  => options,
+            :body     => Fog::JSON.encode(segments),
+            :path     => "#{Fog::Softlayer.escape(container)}/#{Fog::Softlayer.escape(object)}",
+            :query    => { 'multipart-manifest' => 'put' }
+          )
+        end
+
+      end
+    end
+  end
+end

data/lib/fog/softlayer/storage.rb

@@ -0,0 +1,283 @@
+require 'fog/softlayer/core'
+
+module Fog
+  module Storage
+    class Softlayer < Fog::Service
+      requires :softlayer_username, :softlayer_api_key, :softlayer_cluster
+      recognizes :persistent, :softlayer_storage_account, :softlayer_temp_url_key
+
+      model_path 'fog/softlayer/models/storage'
+      model       :directory
+      collection  :directories
+      model       :file
+      collection  :files
+
+      request_path 'fog/softlayer/requests/storage'
+      request :copy_object
+      request :delete_container
+      request :delete_object
+      request :delete_multiple_objects
+      request :delete_static_large_object
+      request :get_container
+      request :get_containers
+      request :get_object
+      request :get_object_https_url
+      request :head_container
+      request :head_containers
+      request :head_object
+      request :put_container
+      request :put_object
+      request :put_object_manifest
+      request :put_dynamic_obj_manifest
+      request :put_static_obj_manifest
+      request :post_set_meta_temp_url_key
+
+      class Mock
+
+        def self.data
+          @data ||= Hash.new do |hash, key|
+            hash[key] = {}
+          end
+        end
+
+        def self.reset
+          @data = nil
+        end
+
+        def initialize(options={})
+          @softlayer_api_key = options[:softlayer_api_key]
+          @softlayer_username = options[:softlayer_username]
+          @path = '/v1/AUTH_1234'
+          @containers = {}
+        end
+
+        def data
+          self.class.data[@softlayer_username]
+        end
+
+        def reset_data
+          self.class.data.delete(@softlayer_username)
+        end
+
+        def change_account(account)
+          @original_path ||= @path
+          version_string = @original_path.split('/')[1]
+          @path = "/#{version_string}/#{account}"
+        end
+
+        def reset_account_name
+          @path = @original_path
+        end
+
+      end
+
+      class Real
+        attr_reader :auth_url
+        attr_accessor :auth_token, :auth_expires
+
+        def initialize(options={})
+          @api_key = options[:softlayer_api_key]
+          @username = options[:softlayer_username]
+          @cluster = options[:softlayer_cluster]
+          @storage_account = options[:softlayer_storage_account] || default_storage_account
+          @connection_options     = options[:connection_options] || {}
+          authenticate
+          @persistent = options[:persistent] || false
+          @connection = Fog::Core::Connection.new("#{@scheme}://#{@host}:#{@port}", @persistent, @connection_options)
+          @temp_url_key = options[:softlayer_temp_url_key] || get_temp_url_key_for_account
+        end
+
+        def auth_url
+          "https://#{@cluster}.#{Fog::Softlayer::SL_STORAGE_AUTH_URL}"
+        end
+
+        def reload
+          @connection.reset
+        end
+
+        def request(params = {}, parse_json = true)
+          begin
+            params.is_a?(Hash) or raise ArgumentError, "#{self.class}#request params must be a Hash"
+            params = _build_params(params)
+            response = @connection.request(params)
+
+            if response.status == 401 && !!@auth_token
+              @auth_token = nil; @auth_expires = nil
+              authenticate
+              response = @connection.request(params)
+            end
+
+            if !response.body.empty? && parse_json && response.get_header('Content-Type') =~ %r{application/json}
+              response.body = Fog::JSON.decode(response.body)
+            end
+
+            response
+          rescue Excon::Errors::HTTPStatusError => error
+            raise case error
+              when Excon::Errors::NotFound
+                Fog::Storage::Softlayer::NotFound.slurp(error)
+              else
+                error
+            end
+          end
+        end
+
+        private
+
+        def _auth_headers
+          {
+              :headers => {
+                  'User-Agent' => "Fog SoftLayer Adapter #{Fog::Softlayer::VERSION}",
+                  'X-Auth-User' => "#{@storage_account}:#{@username}",
+                  'X-Auth-Key'  => @api_key
+              }
+          }
+        end
+
+        def _build_params(params)
+          output = {
+              :method => params.delete(:method) || :get
+          }
+
+          output[:path] = params[:path] ? "#{@path}/#{params.delete(:path)}".sub(/\/$/, '') : @path
+
+          output = output.deep_merge(params)
+          output.deep_merge(_headers)
+        end
+
+        def _headers
+          {
+            :headers => {
+              'Content-Type' => 'application/json',
+              'Accept' => 'application/json',
+              'X-Auth-Token' => @auth_token
+            }
+          }
+        end
+
+        def authenticate
+          if requires_auth?
+            connection = Fog::Core::Connection.new(auth_url, false, _auth_headers)
+            response = connection.request(:method => :get)
+
+            raise Fog::Errors::Error.new("Could not authenticate Object Storage User.") unless response.status.between?(200, 208)
+
+            @auth_token = response.headers['X-Auth-Token']
+            @auth_expires = Time.now + response.headers['X-Auth-Token-Expires'].to_i
+            @storage_token = response.headers['X-Storage-Token']
+
+            uri = URI.parse(response.headers['X-Storage-Url'])
+            @host   = uri.host
+            @path   = uri.path
+            @path.sub!(/\/$/, '')
+            @port   = uri.port
+            @scheme = uri.scheme
+          end
+          true
+        end
+
+        def default_storage_account
+          slapi = Fog::Compute[:softlayer].request(:account, :get_hub_network_storage)
+          slapi.body.map { |store| store['username'] }.first if slapi.body and slapi.body.instance_of? Array
+        end
+
+        def get_temp_url_key_for_account
+          request.headers['X-Account-Meta-Temp-Url-Key']
+        end
+
+        def requires_auth?
+          !@auth_token || !@auth_expires || (@auth_expires.to_i - Time.now.to_i) < 30
+        end
+
+      end
+
+
+
+      # Thanks to @camertron! https://gist.github.com/camertron/2939093
+      module Memory
+        # sizes are a guess, close enough for Mocks
+        REF_SIZE = 4 # ?
+        OBJ_OVERHEAD = 4 # ?
+        FIXNUM_SIZE = 4 # ?
+
+        # informational output from analysis
+        MemoryInfo = Struct.new :roots, :objects, :bytes, :loops
+
+        def self.analyze(*roots)
+          an = Analyzer.new
+          an.roots = roots
+          an.analyze
+        end
+
+        class Analyzer
+          attr_accessor :roots
+          attr_reader   :result
+
+          def analyze
+            @result = MemoryInfo.new roots, 0, 0, 0
+            @objs = {}
+
+            queue = roots.dup
+
+            until queue.empty?
+              obj = queue.shift
+
+              case obj
+                when IO
+                  visit(obj)
+                when String
+                  visit(obj) { @result.bytes += obj.size }
+                when Fixnum
+                  @result.bytes += FIXNUM_SIZE
+                when Array
+                  visit(obj) do
+                    @result.bytes += obj.size * REF_SIZE
+                    queue.concat(obj)
+                  end
+                when Hash
+                  visit(obj) do
+                    @result.bytes += obj.size * REF_SIZE * 2
+                    obj.each {|k,v| queue.push(k).push(v)}
+                  end
+                when Enumerable
+                  visit(obj) do
+                    obj.each do |o|
+                      @result.bytes += REF_SIZE
+                      queue.push(o)
+                    end
+                  end
+                else
+                  visit(obj) do
+                    obj.instance_variables.each do |var|
+                      @result.bytes += REF_SIZE
+                      queue.push(obj.instance_variable_get(var))
+                    end
+                  end
+              end
+            end
+
+            @result
+          end
+
+          private
+          def visit(obj)
+            id = obj.object_id
+
+            if @objs.has_key? id
+              @result.loops += 1
+              false
+            else
+              @objs[id] = true
+              @result.bytes += OBJ_OVERHEAD
+              @result.objects += 1
+              yield obj if block_given?
+              true
+            end
+          end
+        end
+      end
+
+    end
+  end
+end
+