fog-google 1.3.3 → 1.4.0

data/lib/fog/storage/google_json/requests/put_object.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Fog
   module Storage
     class GoogleJSON
@@ -46,18 +48,7 @@ module Fog
                        kms_key_name: nil,
                        predefined_acl: nil,
                        **options)
-          if data.is_a?(String)
-            data = StringIO.new(data)
-            options[:content_type] ||= "text/plain"
-          elsif data.is_a?(::File)
-            options[:content_type] ||= Fog::Storage.parse_data(data)[:headers]["Content-Type"]
-          end
-
-          # Paperclip::AbstractAdapter
-          if data.respond_to?(:content_type) && data.respond_to?(:path)
-            options[:content_type] ||= data.content_type
-            data = data.path
-          end
+          data, options = normalize_data(data, options)
 
           object_config = ::Google::Apis::StorageV1::Object.new(
             options.merge(:name => object_name)
@@ -78,6 +69,25 @@ module Fog
             :upload_source => data
           )
         end
+
+        protected
+
+        def normalize_data(data, options)
+          raise ArgumentError.new("data is required") unless data
+          if data.is_a?(String)
+            data = StringIO.new(data)
+            options[:content_type] ||= "text/plain"
+          elsif data.is_a?(::File)
+            options[:content_type] ||= Fog::Storage.parse_data(data)[:headers]["Content-Type"]
+          end
+
+          # Paperclip::AbstractAdapter
+          if data.respond_to?(:content_type) && data.respond_to?(:path)
+            options[:content_type] ||= data.content_type
+            data = data.path
+          end
+          [data, options]
+        end
       end
 
       class Mock

data/lib/fog/storage/google_xml/models/directory.rb

@@ -7,9 +7,8 @@ module Fog
         attribute :creation_date, :aliases => "CreationDate"
 
         def acl=(new_acl)
-          valid_acls = ["private", "public-read", "public-read-write", "authenticated-read"]
-          unless valid_acls.include?(new_acl)
-            raise ArgumentError.new("acl must be one of [#{valid_acls.join(', ')}]")
+          unless Utils::VALID_ACLS.include?(new_acl)
+            raise ArgumentError.new("acl must be one of [#{Utils::VALID_ACLS.join(', ')}]")
           end
           @acl = new_acl
         end

data/lib/fog/storage/google_xml/models/file.rb

@@ -17,20 +17,9 @@ module Fog
         attribute :owner,               :aliases => "Owner"
         attribute :storage_class,       :aliases => ["x-goog-storage-class", "StorageClass"]
 
-        # https://cloud.google.com/storage/docs/access-control#predefined-acl
-        VALID_ACLS = [
-          "authenticated-read",
-          "bucket-owner-full-control",
-          "bucket-owner-read",
-          "private",
-          "project-private",
-          "public-read",
-          "public-read-write"
-        ].freeze
-
         def acl=(new_acl)
-          unless VALID_ACLS.include?(new_acl)
-            raise ArgumentError.new("acl must be one of [#{VALID_ACLS.join(', ')}]")
+          unless Utils::VALID_ACLS.include?(new_acl)
+            raise ArgumentError.new("acl must be one of [#{Utils::VALID_ACLS.join(', ')}]")
           end
           @acl = new_acl
         end

data/lib/fog/storage/google_xml/requests/put_bucket.rb

@@ -38,7 +38,7 @@ module Fog
       class Mock
         def put_bucket(bucket_name, options = {})
           acl = options["x-goog-acl"] || "private"
-          if !["private", "public-read", "public-read-write", "authenticated-read"].include?(acl)
+          if !Utils::VALID_ACLS.include?(acl)
             raise Excon::Errors::BadRequest.new("invalid x-goog-acl")
           else
             data[:acls][:bucket][bucket_name] = self.class.acls(options[acl])

data/lib/fog/storage/google_xml/requests/put_object.rb

@@ -38,7 +38,7 @@ module Fog
       class Mock
         def put_object(bucket_name, object_name, data, options = {})
           acl = options["x-goog-acl"] || "private"
-          if !["private", "public-read", "public-read-write", "authenticated-read"].include?(acl)
+          if !Utils::VALID_ACLS.include?(acl)
             raise Excon::Errors::BadRequest.new("invalid x-goog-acl")
           else
             self.data[:acls][:object][bucket_name] ||= {}

data/lib/fog/storage/google_xml/requests/put_object_acl.rb

@@ -24,7 +24,11 @@ module Fog
         end
 
         def put_object_acl(bucket_name, object_name, acl)
-          data = <<-DATA
+          headers = {}
+          data = ""
+
+          if acl.is_a?(Hash)
+            data = <<-DATA
 <AccessControlList>
   <Owner>
     #{tag('ID', acl['Owner']['ID'])}
@@ -34,10 +38,15 @@ module Fog
   </Entries>
 </AccessControlList>
 DATA
+          elsif acl.is_a?(String) && Utils::VALID_ACLS.include?(acl)
+            headers["x-goog-acl"] = acl
+          else
+            raise Excon::Errors::BadRequest.new("invalid x-goog-acl")
+          end
 
           request(:body     => data,
                   :expects  => 200,
-                  :headers  => {},
+                  :headers  => headers,
                   :host     => "#{bucket_name}.#{@host}",
                   :method   => "PUT",
                   :query    => { "acl" => nil },

data/lib/fog/storage/google_xml/utils.rb

@@ -2,6 +2,17 @@ module Fog
   module Storage
     class GoogleXML
       module Utils
+        # https://cloud.google.com/storage/docs/access-control#predefined-acl
+        VALID_ACLS = [
+          "authenticated-read",
+          "bucket-owner-full-control",
+          "bucket-owner-read",
+          "private",
+          "project-private",
+          "public-read",
+          "public-read-write"
+        ].freeze
+
         def http_url(params, expires)
           "http://" << host_path_query(params, expires)
         end

data/tasks/test.rake

@@ -1,6 +1,7 @@
 require "rake/testtask"
 
 Rake::TestTask.new do |t|
+  t.description = "Run integration and unit tests"
   t.libs << "test"
   t.pattern = File.join("test", "**", "test_*.rb")
   t.warning = false
@@ -9,6 +10,67 @@ end
 namespace :test do
   mock = ENV["FOG_MOCK"] || "true"
   task :travis do
-    sh("export FOG_MOCK=#{mock} && bundle exec shindont")
+    sh("bundle exec rake test:unit")
+  end
+
+  desc "Run all integration tests in parallel"
+  multitask :parallel => ["test:compute",
+                          "test:monitoring",
+                          "test:pubsub",
+                          "test:sql",
+                          "test:storage"]
+
+  Rake::TestTask.new do |t|
+    t.name = "unit"
+    t.description = "Run Unit tests"
+    t.libs << "test"
+    t.pattern = FileList['test/unit/**/test_*.rb']
+    t.warning = false
+    t.verbose = true
+  end
+
+  Rake::TestTask.new do |t|
+    t.name = "compute"
+    t.description = "Run Compute API tests"
+    t.libs << "test"
+    t.pattern = FileList['test/integration/compute/test_*.rb']
+    t.warning = false
+    t.verbose = true
+  end
+
+  Rake::TestTask.new do |t|
+    t.name = "monitoring"
+    t.description = "Run Monitoring API tests"
+    t.libs << "test"
+    t.pattern = FileList['test/integration/monitoring/test_*.rb']
+    t.warning = false
+    t.verbose = true
+  end
+
+  Rake::TestTask.new do |t|
+    t.name = "pubsub"
+    t.description = "Run PubSub API tests"
+    t.libs << "test"
+    t.pattern = FileList['test/integration/pubsub/test_*.rb']
+    t.warning = false
+    t.verbose = true
+  end
+
+  Rake::TestTask.new do |t|
+    t.name = "sql"
+    t.description = "Run SQL API tests"
+    t.libs << "test"
+    t.pattern = FileList['test/integration/sql/test_*.rb']
+    t.warning = false
+    t.verbose = true
+  end
+
+  Rake::TestTask.new do |t|
+    t.name = "storage"
+    t.description = "Run Storage API tests"
+    t.libs << "test"
+    t.pattern = FileList['test/integration/storage/test_*.rb']
+    t.warning = false
+    t.verbose = true
   end
 end

data/test/integration/compute/addresses/addresses_shared.rb

@@ -4,7 +4,7 @@ require "securerandom"
 
 class TestComputeAddressShared < FogIntegrationTest
   DEFAULT_REGION = "us-central1".freeze
-  ADDRESS_RESOURCE_PREFIX = "fog-test-address".freeze
+  ADDRESS_RESOURCE_PREFIX = "fog-int-test-address".freeze
 
   include ClientHelper
 

data/test/integration/compute/test_compute_addresses_collection.rb

@@ -5,12 +5,13 @@ class TestComputeAddressesCollection < FogIntegrationTest
   DEFAULT_REGION = "us-central1".freeze
   DEFAULT_ZONE = "us-central1-b".freeze
   RESOURCE_PREFIX = "fog-test-addresscol".freeze
+  TEST_ASYNC = false
 
   # Ensure we clean up any created resources
   Minitest.after_run do
     client = Fog::Compute::Google.new
-    client.addresses.each { |a| a.destroy if a.name.start_with?(RESOURCE_PREFIX) }
-    client.servers.each { |s| s.destroy if s.name.start_with?(RESOURCE_PREFIX) }
+    client.addresses.each { |a| a.destroy(TEST_ASYNC) if a.name.start_with?(RESOURCE_PREFIX) }
+    client.servers.each { |s| s.destroy(TEST_ASYNC) if s.name.start_with?(RESOURCE_PREFIX) }
   end
 
   def test_address_workflow
@@ -50,7 +51,7 @@ class TestComputeAddressesCollection < FogIntegrationTest
       ],
       :external_ip => my_address.address
     )
-    my_server.wait_for { provisioning? }
+    my_server.wait_for { staging? }
 
     # And verify that it's correctly assigned
     assert_equal(

data/test/integration/compute/test_compute_networks_collection.rb

@@ -5,12 +5,13 @@ class TestComputeNetworksCollection < FogIntegrationTest
   DEFAULT_REGION = "us-central1".freeze
   DEFAULT_ZONE = "us-central1-b".freeze
   RESOURCE_PREFIX = "fog-test-networkscol".freeze
+  TEST_ASYNC = false
 
   # Ensure we clean up any created resources
   Minitest.after_run do
     client = Fog::Compute::Google.new
-    client.networks.each { |a| a.destroy if a.name.start_with?(RESOURCE_PREFIX) }
-    client.servers.each { |s| s.destroy if s.name.start_with?(RESOURCE_PREFIX) }
+    client.networks.each { |a| a.destroy(TEST_ASYNC) if a.name.start_with?(RESOURCE_PREFIX) }
+    client.servers.each { |s| s.destroy(TEST_ASYNC) if s.name.start_with?(RESOURCE_PREFIX) }
   end
 
   def test_network_workflow
@@ -30,14 +31,15 @@ class TestComputeNetworksCollection < FogIntegrationTest
 
     # Be aware that although the address resource is created, it might not yet
     # have an ip address. You can poll until the address has been assigned.
-    my_network.wait_for { !my_network.ipv4_range.nil? }
+    my_network.wait_for(60) { !my_network.ipv4_range.nil? }
     assert_match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}/,
                  my_network.ipv4_range,
                  "My address's address should have a valid ipv4 address")
 
     # Now that we have an address, we can create a server using the static ip
     server_name = new_resource_name
-    client.servers.create(
+
+    my_server = client.servers.create(
       :name => server_name,
       :machine_type => "f1-micro",
       :zone => client.zones.get(DEFAULT_ZONE).self_link,
@@ -49,9 +51,10 @@ class TestComputeNetworksCollection < FogIntegrationTest
         }
       ],
       :network_interfaces => [my_network.get_as_interface_config]
-    ).wait_for { ready? }
+    )
+
+    my_server.wait_for { ready? }
 
-    my_server = client.servers.get(server_name, DEFAULT_ZONE)
     # We need to verify that the network has been correctly assigned
     assert_equal(
       my_network.self_link,

data/test/integration/compute/test_servers.rb

@@ -4,8 +4,17 @@ require "integration/factories/servers_factory"
 class TestServers < FogIntegrationTest
   include TestCollection
 
+  # Cleanup is handled by TestCollection
   def setup
     @subject = Fog::Compute[:google].servers
     @factory = ServersFactory.new(namespaced_name)
   end
+
+  def test_set_metadata
+    server = @factory.create
+    server.wait_for { ready? }
+    server.set_metadata({ "foo" => "bar", "baz" => "foo" }, false)
+    assert_equal [{ :key => "foo", :value => "bar" },
+                  { :key=>"baz", :value=>"foo" }], server.metadata[:items]
+  end
 end

data/test/integration/compute/test_target_pools.rb

@@ -7,10 +7,32 @@ class TestTargetPools < FogIntegrationTest
   def setup
     @subject = Fog::Compute[:google].target_pools
     @factory = TargetPoolsFactory.new(namespaced_name)
+    @servers = Fog::Compute[:google].servers
   end
 
   # Override to include zone in get request
   def get_resource(identity)
     @subject.get(identity, TEST_ZONE)
   end
+
+  def test_get_health
+    target_pool = @factory.create
+    server = @servers.get(target_pool.instances[0].split("/").last)
+    server.wait_for { ready? }
+
+    # There's no way to track the readiness of the instance resource in a target pool,
+    # so wrapping in a soft retry:
+    begin
+      retries ||= 0
+      target_pool.get_health
+    rescue ::Google::Apis::ClientError
+      sleep 25
+      retry if (retries += 1) < 3
+    end
+
+    assert_equal(target_pool.get_health[server.self_link][0][:instance], server.self_link,
+                 "target_pool should return instance health details")
+    assert_equal(target_pool.get_health(server.name)[server.self_link][0][:instance], server.self_link,
+                 "target_pool should return instance health details when an instance is specified")
+  end
 end

data/test/integration/factories/collection_factory.rb

@@ -7,7 +7,7 @@ class CollectionFactory
     @resource_counter = 0
   end
 
-  def cleanup(async = true)
+  def cleanup(async = false)
     resources = @subject.all.select { |resource| resource.name.start_with? PREFIX }
     resources.each { |r| r.destroy(async) }
     resources.each { |r| Fog.wait_for { !@subject.all.map(&:identity).include? r.identity } }

data/test/integration/monitoring/test_timeseries.rb

@@ -1,6 +1,13 @@
 require "helpers/integration_test_helper"
+require "retriable"
 
 class TestMetricDescriptors < FogIntegrationTest
+  # Retriable is used to wrap each request in this test due to Stackdriver API being slow with
+  # metric propagation (sometimes 80+ seconds) and client returning
+  # Google::Apis::ClientError: badRequest if the metric hasn't yet been created instead of a 404.
+  NOT_READY_REGEX = /The provided filter doesn't refer to any known metric./
+  RETRIABLE_TRIES = 3
+  RETRIABLE_BASE_INTERVAL = 30
   TEST_METRIC_PREFIX = "custom.googleapis.com/fog-google-test/timeseries".freeze
   LABEL_DESCRIPTORS = [
     {
@@ -44,13 +51,37 @@ class TestMetricDescriptors < FogIntegrationTest
     resp = @client.create_timeseries(:timeseries => [expected])
     assert_empty(resp.to_h)
 
-    series = @client.timeseries_collection.all(
-      :filter => "metric.type = \"#{metric_type}\"",
-      :interval => {
-        :start_time => start_time.to_datetime.rfc3339,
-        :end_time => Time.now.to_datetime.rfc3339
-      }
-    )
+    # Wait for metric to be created
+    Retriable.retriable(on: {Google::Apis::ClientError => NOT_READY_REGEX},
+                        tries: RETRIABLE_TRIES,
+                        base_interval: RETRIABLE_BASE_INTERVAL) do
+      @client.list_timeseries(
+          :filter => "metric.type = \"#{metric_type}\"",
+          :interval => {
+            # Subtracting one second because timeSeries.list API
+            # doesn't return points that are exactly the same time
+            # as the interval for some reason.
+            :start_time => (start_time - 1).to_datetime.rfc3339,
+            :end_time => Time.now.to_datetime.rfc3339
+          }
+      ).time_series
+    end
+
+    series = Retriable.retriable(on: { Google::Apis::ClientError => NOT_READY_REGEX },
+                                 tries: RETRIABLE_TRIES,
+                                 base_interval: RETRIABLE_BASE_INTERVAL) do
+      @client.timeseries_collection.all(
+        :filter => "metric.type = \"#{metric_type}\"",
+        :interval => {
+          # Subtracting one second because timeSeries.list API
+          # doesn't return points that are exactly the same time
+          # as the interval for some reason.
+          :start_time => (start_time - 1).to_datetime.rfc3339,
+          :end_time => Time.now.to_datetime.rfc3339
+        }
+      )
+    end
+
     assert_equal(1, series.size)
     actual = series.first
     assert_equal(expected[:metric], actual.metric)
@@ -84,28 +115,42 @@ class TestMetricDescriptors < FogIntegrationTest
       _some_timeseries(start_time, metric_type, labels)
     end
 
-    @client.create_timeseries(:timeseries => timeseries)
+    Retriable.retriable(on: Google::Apis::ServerError,
+                        tries: RETRIABLE_TRIES,
+                        base_interval: RETRIABLE_BASE_INTERVAL) do
+      @client.create_timeseries(:timeseries => timeseries)
+    end
     interval = {
-      :start_time => start_time.to_datetime.rfc3339,
+      # Subtracting one second because timeSeries.list API
+      # doesn't return points that are exactly the same time
+      # as the interval for some reason.
+      :start_time => (start_time - 1).to_datetime.rfc3339,
       :end_time => Time.now.to_datetime.rfc3339
     }
 
-    # Wait for creation
-    Fog.wait_for(30) do
-      # Test all created timeseries are returned.
-      list_result = @client.list_timeseries(
+
+    # Wait for metric to be created
+    # Retriable is used instead of wait_for due to API client returning Google::Apis::ClientError: badRequest if the
+    # metric hasn't yet been created
+    Retriable.retriable(on: { Google::Apis::ClientError => NOT_READY_REGEX },
+                        tries: RETRIABLE_TRIES,
+                        base_interval: RETRIABLE_BASE_INTERVAL) do
+      @client.list_timeseries(
         :filter => "metric.type = \"#{metric_type}\"",
         :interval => interval
       ).time_series
-      !list_result.nil? && list_result.size == timeseries.size
     end
 
     # Test page size
-    resp = @client.list_timeseries(
-      :filter => "metric.type = \"#{metric_type}\"",
-      :interval => interval,
-      :page_size => 1
-    )
+    resp = Retriable.retriable(on: { Google::Apis::ClientError => NOT_READY_REGEX },
+                               tries: RETRIABLE_TRIES,
+                               base_interval: RETRIABLE_BASE_INTERVAL) do
+      @client.list_timeseries(
+        :filter => "metric.type = \"#{metric_type}\"",
+        :interval => interval,
+        :page_size => 1
+      )
+    end
     assert_equal(resp.time_series.size, 1,
                  "expected timeseries count to be equal to page size 1")
 
@@ -123,13 +168,17 @@ class TestMetricDescriptors < FogIntegrationTest
            "expected different timeseries when using page_token")
 
     # Test filter
-    series = @client.timeseries_collection.all(
-      :filter => %[
-        metric.type = "#{metric_type}" AND
-        metric.label.test_string_label = "first"
-      ],
-      :interval => interval
-    )
+    series = Retriable.retriable(on: { Google::Apis::ClientError => NOT_READY_REGEX },
+                                 tries: RETRIABLE_TRIES,
+                                 base_interval: RETRIABLE_BASE_INTERVAL) do
+      @client.timeseries_collection.all(
+        :filter => %[
+          metric.type = "#{metric_type}" AND
+          metric.label.test_string_label = "first"
+        ],
+        :interval => interval
+      )
+    end
     assert_equal(series.size, 1,
                  "expected returned timeseries to be filtered to 1 value")
     assert_equal("true", series.first.metric[:labels][:test_bool_label])
@@ -138,7 +187,8 @@ class TestMetricDescriptors < FogIntegrationTest
 
   def _delete_test_resources
     list_resp = @client.monitoring.list_project_metric_descriptors(
-      :filter => "metric.type = starts_with(\"#{TEST_METRIC_PREFIX}\")"
+      "projects/#{@client.project}",
+      filter: "metric.type = starts_with(\"#{TEST_METRIC_PREFIX}\")"
     )
     unless list_resp.metric_descriptors.nil?
       puts "Found #{list_resp.metric_descriptors.size} test metric descriptors."
@@ -164,7 +214,7 @@ class TestMetricDescriptors < FogIntegrationTest
     )
 
     # Wait for metric descriptor to be created
-    Fog.wait_for(30, 2) do
+    Fog.wait_for(180, 2) do
       begin
         @client.get_metric_descriptor(metric_type)
         true