fog-openstack 0.1.0

data/tests/openstack/requests/storage/large_object_tests.rb

@@ -0,0 +1,372 @@
+Shindo.tests('Fog::Storage[:openstack] | large object requests', ['openstack']) do
+
+  unless Fog.mocking?
+    @directory  = Fog::Storage[:openstack].directories.create(:key => 'foglargeobjecttests')
+    @directory2 = Fog::Storage[:openstack].directories.create(:key => 'foglargeobjecttests2')
+    @segments = {
+      :a => {
+        :container => @directory.identity,
+        :name      => 'fog_large_object/a',
+        :data      => 'a' * (1024**2 + 10),
+        :size      => 1024**2 + 10,
+        :etag      => 'c2e97007d59f0c19b850debdcb80cca5'
+      },
+      :b => {
+        :container => @directory.identity,
+        :name      => 'fog_large_object/b',
+        :data      => 'b' * (1024**2 + 20),
+        :size      => 1024**2 + 20,
+        :etag      => 'd35f50622a1259daad75ff7d5512c7ef'
+      },
+      :c => {
+        :container => @directory.identity,
+        :name      => 'fog_large_object2/a',
+        :data      => 'c' * (1024**2 + 30),
+        :size      => 1024**2 + 30,
+        :etag      => '901d3531a87d188041d4d5b43cb464c1'
+      },
+      :d => {
+        :container => @directory2.identity,
+        :name      => 'fog_large_object2/b',
+        :data      => 'd' * (1024**2 + 40),
+        :size      => 1024**2 + 40,
+        :etag      => '350c0e00525198813920a157df185c8d'
+      }
+    }
+  end
+
+  tests('success') do
+
+    tests('upload test segments').succeeds do
+      pending if Fog.mocking?
+
+      @segments.each_value do |segment|
+        Fog::Storage[:openstack].put_object(segment[:container], segment[:name], segment[:data])
+      end
+    end
+
+    tests('dynamic large object requests') do
+      pending if Fog.mocking?
+
+      tests('#put_object_manifest alias').succeeds do
+        Fog::Storage[:openstack].put_object_manifest(@directory.identity, 'fog_large_object')
+      end
+
+      tests('using default X-Object-Manifest header') do
+
+        tests('#put_dynamic_obj_manifest').succeeds do
+          Fog::Storage[:openstack].put_dynamic_obj_manifest(@directory.identity, 'fog_large_object')
+        end
+
+        tests('#get_object streams all segments matching the default prefix').succeeds do
+          expected = @segments[:a][:data] + @segments[:b][:data] + @segments[:c][:data]
+          Fog::Storage[:openstack].get_object(@directory.identity, 'fog_large_object').body == expected
+        end
+
+        # When the manifest object name is equal to the segment prefix, OpenStack treats it as if it's the first segment.
+        # So you must prepend the manifest object's Etag - Digest::MD5.hexdigest('')
+        tests('#head_object returns Etag that includes manifest object in calculation').succeeds do
+          etags = ['d41d8cd98f00b204e9800998ecf8427e', @segments[:a][:etag], @segments[:b][:etag], @segments[:c][:etag]]
+          expected = "\"#{ Digest::MD5.hexdigest(etags.join) }\"" # returned in quotes "\"2577f38428e895c50de6ea78ccc7da2a"\"
+          Fog::Storage[:openstack].head_object(@directory.identity, 'fog_large_object').headers['Etag'] == expected
+        end
+
+      end
+
+      tests('specifying X-Object-Manifest segment prefix') do
+
+        tests('#put_dynamic_obj_manifest').succeeds do
+          options = { 'X-Object-Manifest' => "#{ @directory.identity }/fog_large_object/" }
+          Fog::Storage[:openstack].put_dynamic_obj_manifest(@directory.identity, 'fog_large_object', options)
+        end
+
+        tests('#get_object streams segments only matching the specified prefix').succeeds do
+          expected = @segments[:a][:data] + @segments[:b][:data]
+          Fog::Storage[:openstack].get_object(@directory.identity, 'fog_large_object').body == expected
+        end
+
+        tests('#head_object returns Etag that does not include manifest object in calculation').succeeds do
+          etags = [@segments[:a][:etag], @segments[:b][:etag]]
+          expected = "\"#{ Digest::MD5.hexdigest(etags.join) }\"" # returned in quotes "\"0f035ed3cc38aa0ef46dda3478fad44d"\"
+          Fog::Storage[:openstack].head_object(@directory.identity, 'fog_large_object').headers['Etag'] == expected
+        end
+
+      end
+
+      tests('storing manifest in a different container than the segments') do
+
+        tests('#put_dynamic_obj_manifest').succeeds do
+          options = { 'X-Object-Manifest' => "#{ @directory.identity }/fog_large_object/" }
+          Fog::Storage[:openstack].put_dynamic_obj_manifest(@directory2.identity, 'fog_large_object', options)
+        end
+
+        tests('#get_object').succeeds do
+          expected = @segments[:a][:data] + @segments[:b][:data]
+          Fog::Storage[:openstack].get_object(@directory2.identity, 'fog_large_object').body == expected
+        end
+
+      end
+
+    end
+
+    tests('static large object requests') do
+      pending if Fog.mocking?
+
+      tests('single container') do
+
+        tests('#put_static_obj_manifest').succeeds do
+          segments = [
+            { :path => "#{ @segments[:a][:container] }/#{ @segments[:a][:name] }",
+              :etag => @segments[:a][:etag],
+              :size_bytes => @segments[:a][:size] },
+            { :path => "#{ @segments[:c][:container] }/#{ @segments[:c][:name] }",
+              :etag => @segments[:c][:etag],
+              :size_bytes => @segments[:c][:size] }
+          ]
+          Fog::Storage[:openstack].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments)
+        end
+
+        tests('#head_object') do
+          etags = [@segments[:a][:etag], @segments[:c][:etag]]
+          etag = "\"#{ Digest::MD5.hexdigest(etags.join) }\"" # "\"ad7e633a12e8a4915b45e6dd1d4b0b4b\""
+          content_length = (@segments[:a][:size] + @segments[:c][:size]).to_s
+          response = Fog::Storage[:openstack].head_object(@directory.identity, 'fog_large_object')
+
+          returns(etag, 'returns ETag computed from segments') { response.headers['Etag'] }
+          returns(content_length , 'returns Content-Length for all segments') { response.headers['Content-Length'] }
+          returns('True', 'returns X-Static-Large-Object header') { response.headers['X-Static-Large-Object'] }
+        end
+
+        tests('#get_object').succeeds do
+          expected = @segments[:a][:data] + @segments[:c][:data]
+          Fog::Storage[:openstack].get_object(@directory.identity, 'fog_large_object').body == expected
+        end
+
+        tests('#delete_static_large_object') do
+          expected = {
+            'Number Not Found'  => 0,
+            'Response Status'   => '200 OK',
+            'Errors'            => [],
+            'Number Deleted'    => 3,
+            'Response Body'     => ''
+          }
+          returns(expected, 'deletes manifest and segments') do
+            Fog::Storage[:openstack].delete_static_large_object(@directory.identity, 'fog_large_object').body
+          end
+        end
+
+      end
+
+      tests('multiple containers') do
+
+        tests('#put_static_obj_manifest').succeeds do
+          segments = [
+            { :path => "#{ @segments[:b][:container] }/#{ @segments[:b][:name] }",
+              :etag => @segments[:b][:etag],
+              :size_bytes => @segments[:b][:size] },
+            { :path => "#{ @segments[:d][:container] }/#{ @segments[:d][:name] }",
+              :etag => @segments[:d][:etag],
+              :size_bytes => @segments[:d][:size] }
+          ]
+          Fog::Storage[:openstack].put_static_obj_manifest(@directory2.identity, 'fog_large_object', segments)
+        end
+
+        tests('#head_object') do
+          etags = [@segments[:b][:etag], @segments[:d][:etag]]
+          etag = "\"#{ Digest::MD5.hexdigest(etags.join) }\"" # "\"9801a4cc4472896a1e975d03f0d2c3f8\""
+          content_length = (@segments[:b][:size] + @segments[:d][:size]).to_s
+          response = Fog::Storage[:openstack].head_object(@directory2.identity, 'fog_large_object')
+
+          returns(etag, 'returns ETag computed from segments') { response.headers['Etag'] }
+          returns(content_length , 'returns Content-Length for all segments') { response.headers['Content-Length'] }
+          returns('True', 'returns X-Static-Large-Object header') { response.headers['X-Static-Large-Object'] }
+        end
+
+        tests('#get_object').succeeds do
+          expected = @segments[:b][:data] + @segments[:d][:data]
+          Fog::Storage[:openstack].get_object(@directory2.identity, 'fog_large_object').body == expected
+        end
+
+        tests('#delete_static_large_object') do
+          expected = {
+            'Number Not Found'  => 0,
+            'Response Status'   => '200 OK',
+            'Errors'            => [],
+            'Number Deleted'    => 3,
+            'Response Body'     => ''
+          }
+          returns(expected, 'deletes manifest and segments') do
+            Fog::Storage[:openstack].delete_static_large_object(@directory2.identity, 'fog_large_object').body
+          end
+        end
+
+      end
+
+    end
+
+  end
+
+  tests('failure') do
+
+    tests('dynamic large object requests') do
+      pending if Fog.mocking?
+
+      tests('#put_dynamic_obj_manifest with missing container').raises(Fog::Storage::OpenStack::NotFound) do
+        Fog::Storage[:openstack].put_dynamic_obj_manifest('fognoncontainer', 'fog_large_object')
+      end
+
+    end
+
+    tests('static large object requests') do
+      pending if Fog.mocking?
+
+      tests('upload test segments').succeeds do
+        Fog::Storage[:openstack].put_object(@segments[:a][:container], @segments[:a][:name], @segments[:a][:data])
+        Fog::Storage[:openstack].put_object(@segments[:b][:container], @segments[:b][:name], @segments[:b][:data])
+      end
+
+      tests('#put_static_obj_manifest with missing container').raises(Fog::Storage::OpenStack::NotFound) do
+        Fog::Storage[:openstack].put_static_obj_manifest('fognoncontainer', 'fog_large_object', [])
+      end
+
+      tests('#put_static_obj_manifest with missing object') do
+        segments = [
+          { :path => "#{ @segments[:c][:container] }/#{ @segments[:c][:name] }",
+            :etag => @segments[:c][:etag],
+            :size_bytes => @segments[:c][:size] }
+        ]
+        expected = { 'Errors' => [[segments[0][:path], '404 Not Found']] }
+
+        error = nil
+        begin
+          Fog::Storage[:openstack].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments)
+        rescue => err
+          error = err
+        end
+
+        raises(Excon::Errors::BadRequest) do
+          raise error if error
+        end
+
+        returns(expected, 'returns error information') do
+          Fog::JSON.decode(error.response.body)
+        end
+      end
+
+      tests('#put_static_obj_manifest with invalid etag') do
+        segments = [
+          { :path => "#{ @segments[:a][:container] }/#{ @segments[:a][:name] }",
+            :etag => @segments[:b][:etag],
+            :size_bytes => @segments[:a][:size] }
+        ]
+        expected = { 'Errors' => [[segments[0][:path], 'Etag Mismatch']] }
+
+        error = nil
+        begin
+          Fog::Storage[:openstack].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments)
+        rescue => err
+          error = err
+        end
+
+        raises(Excon::Errors::BadRequest) do
+          raise error if error
+        end
+
+        returns(expected, 'returns error information') do
+          Fog::JSON.decode(error.response.body)
+        end
+      end
+
+      tests('#put_static_obj_manifest with invalid byte_size') do
+        segments = [
+          { :path => "#{ @segments[:a][:container] }/#{ @segments[:a][:name] }",
+            :etag => @segments[:a][:etag],
+            :size_bytes => @segments[:b][:size] }
+        ]
+        expected = { 'Errors' => [[segments[0][:path], 'Size Mismatch']] }
+
+        error = nil
+        begin
+          Fog::Storage[:openstack].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments)
+        rescue => err
+          error = err
+        end
+
+        raises(Excon::Errors::BadRequest) do
+          raise error if error
+        end
+
+        returns(expected, 'returns error information') do
+          Fog::JSON.decode(error.response.body)
+        end
+      end
+
+      tests('#delete_static_large_object with missing container') do
+        expected = {
+          'Number Not Found'  => 1,
+          'Response Status'   => '200 OK',
+          'Errors'            => [],
+          'Number Deleted'    => 0,
+          'Response Body'     => ''
+        }
+
+        returns(expected, 'reports missing object') do
+          Fog::Storage[:openstack].delete_static_large_object('fognoncontainer', 'fog_large_object').body
+        end
+      end
+
+      tests('#delete_static_large_object with missing manifest') do
+        expected = {
+          'Number Not Found'  => 1,
+          'Response Status'   => '200 OK',
+          'Errors'            => [],
+          'Number Deleted'    => 0,
+          'Response Body'     => ''
+        }
+
+        returns(expected, 'reports missing manifest') do
+          Fog::Storage[:openstack].delete_static_large_object(@directory.identity, 'fog_non_object').body
+        end
+      end
+
+      tests('#delete_static_large_object with missing segment') do
+
+        tests('#put_static_obj_manifest for segments :a and :b').succeeds do
+          segments = [
+            { :path => "#{ @segments[:a][:container] }/#{ @segments[:a][:name] }",
+              :etag => @segments[:a][:etag],
+              :size_bytes => @segments[:a][:size] },
+            { :path => "#{ @segments[:b][:container] }/#{ @segments[:b][:name] }",
+              :etag => @segments[:b][:etag],
+              :size_bytes => @segments[:b][:size] }
+          ]
+          Fog::Storage[:openstack].put_static_obj_manifest(@directory.identity, 'fog_large_object', segments)
+        end
+
+        tests('#delete_object segment :b').succeeds do
+          Fog::Storage[:openstack].delete_object(@segments[:b][:container], @segments[:b][:name])
+        end
+
+        tests('#delete_static_large_object') do
+          expected = {
+            'Number Not Found'  => 1,
+            'Response Status'   => '200 OK',
+            'Errors'            => [],
+            'Number Deleted'    => 2,
+            'Response Body'     => ''
+          }
+          returns(expected, 'deletes manifest and segment :a, and reports missing segment :b') do
+            Fog::Storage[:openstack].delete_static_large_object(@directory.identity, 'fog_large_object').body
+          end
+        end
+
+      end
+    end
+
+  end
+
+  unless Fog.mocking?
+    @directory.destroy
+    @directory2.destroy
+  end
+end

data/tests/openstack/requests/storage/object_tests.rb

@@ -0,0 +1,208 @@
+Shindo.tests('Fog::Storage[:openstack] | object requests', ["openstack"]) do
+
+  unless Fog.mocking?
+    @directory = Fog::Storage[:openstack].directories.create(:key => 'fogobjecttests')
+  end
+
+  module OpenStackStorageHelpers
+    def override_path(path)
+      @path = path
+    end
+  end
+
+  tests('success') do
+
+    tests("#put_object('fogobjecttests', 'fog_object')").succeeds do
+      pending if Fog.mocking?
+      Fog::Storage[:openstack].put_object('fogobjecttests', 'fog_object', lorem_file)
+    end
+
+    tests("#get_object('fogobjectests', 'fog_object')").succeeds do
+      pending if Fog.mocking?
+      Fog::Storage[:openstack].get_object('fogobjecttests', 'fog_object').body == lorem_file.read
+    end
+
+    tests("#get_object('fogobjecttests', 'fog_object', &block)").succeeds do
+      pending if Fog.mocking?
+      data = ''
+      Fog::Storage[:openstack].get_object('fogobjecttests', 'fog_object') do |chunk, remaining_bytes, total_bytes|
+        data << chunk
+      end
+      data == lorem_file.read
+    end
+
+    tests("#public_url('fogobjectests', 'fog_object')").succeeds do
+      pending if Fog.mocking?
+      Fog::Storage[:openstack].directories.first.files.first.public_url
+    end
+
+    tests("#public_url('fogobjectests')").succeeds do
+      pending if Fog.mocking?
+      Fog::Storage[:openstack].directories.first.public_url
+    end
+
+    tests("#head_object('fogobjectests', 'fog_object')").succeeds do
+      pending if Fog.mocking?
+      Fog::Storage[:openstack].head_object('fogobjecttests', 'fog_object')
+    end
+
+    tests("#delete_object('fogobjecttests', 'fog_object')").succeeds do
+      pending if Fog.mocking?
+      Fog::Storage[:openstack].delete_object('fogobjecttests', 'fog_object')
+    end
+
+    def test_temp_url(url_s, time, desired_scheme)
+      object_url = URI.parse(url_s)
+      query_params = URI.decode_www_form(object_url.query)
+
+      tests("the link is #{desired_scheme}").returns(desired_scheme) { object_url.scheme }
+      tests('the container and object are present in the path').returns(true) do
+        (object_url.path =~ /\/#{@directory.identity}\/fog_object/) != nil
+      end
+      tests('a temp_url_sig is present').returns(true) do
+        query_params.any? { |p| p[0] == 'temp_url_sig' }
+      end
+      tests('temp_url_expires matches the expiration').returns(true) do
+        query_params.any? { |p| p == ['temp_url_expires', time.to_i.to_s] }
+      end
+    end
+
+    tests("#get_object_http_url('directory.identity', 'fog_object', expiration timestamp)").succeeds do
+      pending if Fog.mocking?
+      ts = Time.at(1395343213)
+      url_s = Fog::Storage[:openstack].get_object_http_url(@directory.identity, 'fog_object', ts)
+      test_temp_url(url_s, ts, 'http')
+    end
+
+    tests("#get_object_https_url('directory.identity', 'fog_object', expiration timestamp)").succeeds do
+      pending if Fog.mocking?
+      ts = Time.at(1395343215)
+      url_s = Fog::Storage[:openstack].get_object_https_url(@directory.identity, 'fog_object', ts)
+      test_temp_url(url_s, ts, 'https')
+    end
+
+    tests("put_object with block") do
+      pending if Fog.mocking?
+
+      tests("#put_object('fogobjecttests', 'fog_object', &block)").succeeds do
+        begin
+          file = lorem_file
+          buffer_size = file.stat.size / 2 # chop it up into two buffers
+          Fog::Storage[:openstack].put_object('fogobjecttests', 'fog_block_object', nil) do
+            file.read(buffer_size).to_s
+          end
+        ensure
+          file.close
+        end
+      end
+
+      tests('#get_object').succeeds do
+        Fog::Storage[:openstack].get_object('fogobjecttests', 'fog_block_object').body == lorem_file.read
+      end
+
+      tests('#delete_object').succeeds do
+        Fog::Storage[:openstack].delete_object('fogobjecttests', 'fog_block_object')
+      end
+    end
+
+    tests('#delete_multiple_objects') do
+      pending if Fog.mocking?
+
+      Fog::Storage[:openstack].put_object('fogobjecttests', 'fog_object', lorem_file)
+      Fog::Storage[:openstack].put_object('fogobjecttests', 'fog_object2', lorem_file)
+      Fog::Storage[:openstack].directories.create(:key => 'fogobjecttests2')
+      Fog::Storage[:openstack].put_object('fogobjecttests2', 'fog_object', lorem_file)
+
+      expected = {
+        "Number Not Found"  => 0,
+        "Response Status"   => "200 OK",
+        "Errors"            => [],
+        "Number Deleted"    => 2,
+        "Response Body"     => ""
+      }
+
+      returns(expected, 'deletes multiple objects') do
+        Fog::Storage[:openstack].delete_multiple_objects('fogobjecttests', ['fog_object', 'fog_object2']).body
+      end
+
+      returns(expected, 'deletes object and container') do
+        Fog::Storage[:openstack].delete_multiple_objects(nil, ['fogobjecttests2/fog_object', 'fogobjecttests2']).body
+      end
+    end
+
+  end
+
+  tests('failure') do
+
+    tests("#get_object('fogobjecttests', 'fog_non_object')").raises(Fog::Storage::OpenStack::NotFound) do
+      pending if Fog.mocking?
+      Fog::Storage[:openstack].get_object('fogobjecttests', 'fog_non_object')
+    end
+
+    tests("#get_object('fognoncontainer', 'fog_non_object')").raises(Fog::Storage::OpenStack::NotFound) do
+      pending if Fog.mocking?
+      Fog::Storage[:openstack].get_object('fognoncontainer', 'fog_non_object')
+    end
+
+    tests("#head_object('fogobjecttests', 'fog_non_object')").raises(Fog::Storage::OpenStack::NotFound) do
+      pending if Fog.mocking?
+      Fog::Storage[:openstack].head_object('fogobjecttests', 'fog_non_object')
+    end
+
+    tests("#head_object('fognoncontainer', 'fog_non_object')").raises(Fog::Storage::OpenStack::NotFound) do
+      pending if Fog.mocking?
+      Fog::Storage[:openstack].head_object('fognoncontainer', 'fog_non_object')
+    end
+
+    tests("#delete_object('fogobjecttests', 'fog_non_object')").raises(Fog::Storage::OpenStack::NotFound) do
+      pending if Fog.mocking?
+      Fog::Storage[:openstack].delete_object('fogobjecttests', 'fog_non_object')
+    end
+
+    tests("#delete_object('fognoncontainer', 'fog_non_object')").raises(Fog::Storage::OpenStack::NotFound) do
+      pending if Fog.mocking?
+      Fog::Storage[:openstack].delete_object('fognoncontainer', 'fog_non_object')
+    end
+
+    tests('#delete_multiple_objects') do
+      pending if Fog.mocking?
+
+      expected = {
+        "Number Not Found"  => 2,
+        "Response Status"   => "200 OK",
+        "Errors"            => [],
+        "Number Deleted"    => 0,
+        "Response Body"     => ""
+      }
+
+      returns(expected, 'reports missing objects') do
+        Fog::Storage[:openstack].delete_multiple_objects('fogobjecttests', ['fog_non_object', 'fog_non_object2']).body
+      end
+
+      returns(expected, 'reports missing container') do
+        Fog::Storage[:openstack].delete_multiple_objects('fognoncontainer', ['fog_non_object', 'fog_non_object2']).body
+      end
+
+      tests('deleting non-empty container') do
+        Fog::Storage[:openstack].put_object('fogobjecttests', 'fog_object', lorem_file)
+
+        expected = {
+          "Number Not Found"  => 0,
+          "Response Status"   => "400 Bad Request",
+          "Errors"            => [['fogobjecttests', '409 Conflict']],
+          "Number Deleted"    => 1,
+          "Response Body"     => ""
+        }
+
+        returns(expected, 'deletes object but not container') do
+          Fog::Storage[:openstack].delete_multiple_objects(nil, ['fogobjecttests', 'fogobjecttests/fog_object']).body
+        end
+      end
+    end
+  end
+
+  unless Fog.mocking?
+    @directory.destroy
+  end
+
+end