fog-google 1.19.0 → 1.24.1

data/lib/fog/compute/google/models/regions.rb

@@ -4,9 +4,18 @@ module Fog
       class Regions < Fog::Collection
         model Fog::Compute::Google::Region
 
-        def all
-          data = service.list_regions.to_h
-          load(data[:items] || [])
+        def all(opts = {})
+          items = []
+          next_page_token = nil
+          loop do
+            data = service.list_regions(**opts)
+            next_items = data.to_h[:items] || []
+            items.concat(next_items)
+            next_page_token = data.next_page_token
+            break if next_page_token.nil? || next_page_token.empty?
+            opts[:page_token] = next_page_token
+          end
+          load(items)
         end
 
         def get(identity)

data/lib/fog/compute/google/models/routes.rb

@@ -4,9 +4,18 @@ module Fog
       class Routes < Fog::Collection
         model Fog::Compute::Google::Route
 
-        def all
-          data = service.list_routes.to_h
-          load(data[:items] || [])
+        def all(opts = {})
+          items = []
+          next_page_token = nil
+          loop do
+            data = service.list_routes(**opts)
+            next_items = data.to_h[:items] || []
+            items.concat(next_items)
+            next_page_token = data.next_page_token
+            break if next_page_token.nil? || next_page_token.empty?
+            opts[:page_token] = next_page_token
+          end
+          load(items)
         end
 
         def get(identity)

data/lib/fog/compute/google/models/server.rb

@@ -27,7 +27,7 @@ module Fog
         #   [
         #     {
         #       :initialize_params => {
-        #         :source_image => "projects/debian-cloud/global/images/family/debian-9"
+        #         :source_image => "projects/debian-cloud/global/images/family/debian-11"
         #       }
         #     }
         #   ]
@@ -279,7 +279,7 @@ module Fog
           if disk.is_a? Disk
             disk_obj = disk.get_attached_disk
           elsif disk.is_a? String
-            disk_obj = service.disks.attached_disk_obj(disk, attached_disk_options)
+            disk_obj = service.disks.attached_disk_obj(disk, **attached_disk_options)
           end
 
           data = service.attach_disk(identity, zone_name, disk_obj)
@@ -339,10 +339,10 @@ module Fog
           operation
         end
 
-        def stop(async = true)
+        def stop(async = true, discard_local_ssd=false)
           requires :identity, :zone
 
-          data = service.stop_server(identity, zone_name)
+          data = service.stop_server(identity, zone_name, discard_local_ssd)
           operation = Fog::Compute::Google::Operations
                       .new(:service => service)
                       .get(data.name, data.zone)

data/lib/fog/compute/google/models/servers.rb

@@ -13,17 +13,31 @@ module Fog
             :page_token => page_token
           }
 
-          if zone
-            data = service.list_servers(zone, **opts).to_h[:items] || []
-          else
-            data = []
-            service.list_aggregated_servers(**opts).items.each_value do |scoped_lst|
-              if scoped_lst && scoped_lst.instances
-                data.concat(scoped_lst.instances.map(&:to_h))
+          items = []
+          next_page_token = nil
+
+          loop do
+            if zone
+              data = service.list_servers(zone, **opts)
+              next_items = data.to_h[:items] || []
+              items.concat(next_items)
+              next_page_token = data.next_page_token
+            else
+              data = service.list_aggregated_servers(**opts)
+              data.items.each_value do |scoped_lst|
+                if scoped_lst && scoped_lst.instances
+                  items.concat(scoped_lst.instances.map(&:to_h))
+                end
               end
+              next_page_token = data.next_page_token
             end
+
+            break if next_page_token.nil? || next_page_token.empty?
+
+            opts[:page_token] = next_page_token
           end
-          load(data)
+
+          load(items)
         end
 
         # TODO: This method needs to take self_links as well as names
@@ -50,7 +64,7 @@ module Fog
 
           if disks.nil? || disks.empty?
             # create the persistent boot disk
-            source_img = service.images.get_from_family("debian-9")
+            source_img = service.images.get_from_family("debian-11")
             disk_defaults = {
               :name => name,
               :size_gb => 10,

data/lib/fog/compute/google/models/ssl_certificates.rb

@@ -14,9 +14,18 @@ module Fog
           nil
         end
 
-        def all(_filters = {})
-          data = service.list_ssl_certificates.to_h[:items] || []
-          load(data)
+        def all(opts = {})
+          items = []
+          next_page_token = nil
+          loop do
+            data = service.list_ssl_certificates(**opts)
+            next_items = data.to_h[:items] || []
+            items.concat(next_items)
+            next_page_token = data.next_page_token
+            break if next_page_token.nil? || next_page_token.empty?
+            opts[:page_token] = next_page_token
+          end
+          load(items)
         end
       end
     end

data/lib/fog/compute/google/models/subnetworks.rb

@@ -11,16 +11,24 @@ module Fog
             :order_by => order_by,
             :page_token => page_token
           }
-
-          if region
-            data = service.list_subnetworks(region, **filters).to_h[:items] || []
-          else
-            data = []
-            service.list_aggregated_subnetworks(**filters).to_h[:items].each_value do |region_obj|
-              data.concat(region_obj[:subnetworks]) if region_obj[:subnetworks]
+          items = []
+          next_page_token = nil
+          loop do
+            if region
+              data = service.list_subnetworks(region, **filters)
+              next_items = data.items || []
+              items.concat(next_items)
+              next_page_token = data.next_page_token
+            else
+              data = service.list_aggregated_subnetworks(**filters)
+              data.items.each_value do |region_obj|
+                items.concat(region_obj.subnetworks) if region_obj && region_obj.subnetworks
+              end
             end
+            break if next_page_token.nil? || next_page_token.empty?
+            filters[:page_token] = next_page_token
           end
-          load(data)
+          load(items.map(&:to_h))
         end
 
         def get(identity, region = nil)

data/lib/fog/compute/google/models/target_http_proxies.rb

@@ -4,9 +4,18 @@ module Fog
       class TargetHttpProxies < Fog::Collection
         model Fog::Compute::Google::TargetHttpProxy
 
-        def all(_filters = {})
-          data = service.list_target_http_proxies.to_h[:items] || []
-          load(data)
+        def all(opts = {})
+          items = []
+          next_page_token = nil
+          loop do
+            data = service.list_target_http_proxies(*opts)
+            next_items = data.to_h[:items] || []
+            items.concat(next_items)
+            next_page_token = data.next_page_token
+            break if next_page_token.nil? || next_page_token.empty?
+            opts[:page_token] = next_page_token
+          end
+          load(items)
         end
 
         def get(identity)

data/lib/fog/compute/google/models/target_https_proxies.rb

@@ -4,9 +4,18 @@ module Fog
       class TargetHttpsProxies < Fog::Collection
         model Fog::Compute::Google::TargetHttpsProxy
 
-        def all(_filters = {})
-          data = service.list_target_https_proxies.to_h[:items] || []
-          load(data)
+        def all(opts = {})
+          items = []
+          next_page_token = nil
+          loop do
+            data = service.list_target_https_proxies(**opts)
+            next_items = data.to_h[:items] || []
+            items.concat(next_items)
+            next_page_token = data.next_page_token
+            break if next_page_token.nil? || next_page_token.empty?
+            opts[:page_token] = next_page_token
+          end
+          load(items)
         end
 
         def get(identity)

data/lib/fog/compute/google/models/target_instances.rb

@@ -13,17 +13,25 @@ module Fog
             :page_token => page_token
           }
 
-          if zone
-            data = service.list_target_instances(zone, **opts).to_h[:items] || []
-          else
-            data = []
-            service.list_aggregated_target_instances(**opts).items.each_value do |scoped_list|
-              unless scoped_list.nil? || scoped_list.target_instances.nil?
-                data += scoped_list.target_instances.map(&:to_h)
+          items = []
+          next_page_token = nil
+          loop do
+            if zone
+              data = service.list_target_instances(zone, **opts)
+              next_items = data.to_h[:items] || []
+              items.concat(next_items)
+              next_page_token = data.next_page_token
+            else
+              data = service.list_aggregated_target_instances(**opts)
+              data.items.each_value do |scoped_list|
+                items.concat(scoped_list.target_instances.map(&:to_h)) if scoped_list && scoped_list.target_instances
               end
+              next_page_token = data.next_page_token
             end
+            break if next_page_token.nil? || next_page_token.empty?
+            opts[:page_token] = next_page_token
           end
-          load(data)
+          load(items)
         end
 
         def get(identity, zone = nil)

data/lib/fog/compute/google/models/target_pools.rb

@@ -11,17 +11,25 @@ module Fog
             :order_by => order_by,
             :page_token => page_token
           }
-          if region.nil?
-            data = []
-            service.list_aggregated_target_pools(**opts).items.each_value do |lst|
-              unless lst.nil? || lst.target_pools.nil?
-                data += lst.to_h[:target_pools]
+          items = []
+          next_page_token = nil
+          loop do
+            if region.nil?
+              data = service.list_aggregated_target_pools(**opts)
+              data.items.each_value do |lst|
+                items.concat(lst.to_h[:target_pools]) if lst && lst.target_pools
               end
+              next_page_token = data.next_page_token
+            else
+              data = service.list_target_pools(region, **opts)
+              next_items = data.to_h[:items] || []
+              items.concat(next_items)
+              next_page_token = data.next_page_token
             end
-          else
-            data = service.list_target_pools(region, **opts).to_h[:items]
+            break if next_page_token.nil? || next_page_token.empty?
+            opts[:page_token] = next_page_token
           end
-          load(data)
+          load(items)
         end
 
         def get(identity, region = nil)

data/lib/fog/compute/google/models/url_maps.rb

@@ -4,9 +4,18 @@ module Fog
       class UrlMaps < Fog::Collection
         model Fog::Compute::Google::UrlMap
 
-        def all
-          data = service.list_url_maps.to_h[:items] || []
-          load(data)
+        def all(opts = {})
+          items = []
+          next_page_token = nil
+          loop do
+            data = service.list_url_maps(**opts)
+            next_items = data.to_h[:items] || []
+            items.concat(next_items)
+            next_page_token = data.next_page_token
+            break if next_page_token.nil? || next_page_token.empty?
+            opts[:page_token] = next_page_token
+          end
+          load(items)
         end
 
         def get(identity)

data/lib/fog/compute/google/models/zones.rb

@@ -4,9 +4,18 @@ module Fog
       class Zones < Fog::Collection
         model Fog::Compute::Google::Zone
 
-        def all
-          data = service.list_zones.to_h[:items] || []
-          load(data)
+        def all(opts = {})
+          items = []
+          next_page_token = nil
+          loop do
+            data = service.list_zones
+            next_items = data.to_h[:items] || []
+            items.concat(next_items)
+            next_page_token = data.next_page_token
+            break if next_page_token.nil? || next_page_token.empty?
+            opts[:page_token] = next_page_token
+          end
+          load(items)
         end
 
         def get(identity)

data/lib/fog/compute/google/requests/insert_server.rb

@@ -52,7 +52,7 @@ module Fog
         #       :disks => [
         #         {
         #           :initialize_params => {
-        #             :source_image => "projects/debian-cloud/global/images/family/debian-9"
+        #             :source_image => "projects/debian-cloud/global/images/family/debian-11"
         #           }
         #         }
         #       ]

data/lib/fog/compute/google/requests/stop_server.rb

@@ -10,8 +10,8 @@ module Fog
       end
 
       class Real
-        def stop_server(identity, zone)
-          @compute.stop_instance(@project, zone.split("/")[-1], identity)
+        def stop_server(identity, zone, discard_local_ssd=false)
+          @compute.stop_instance(@project, zone.split("/")[-1], identity, discard_local_ssd: discard_local_ssd)
         end
       end
     end

data/lib/fog/google/shared.rb

@@ -67,20 +67,21 @@ module Fog
           ::Google::Apis.logger.level = ::Logger::DEBUG
         end
 
-        auth = nil
+        initialize_auth(options).tap do |auth|
+          ::Google::Apis::RequestOptions.default.authorization = auth
+        end
+      end
 
+      def initialize_auth(options)
         if options[:google_json_key_location] || options[:google_json_key_string]
-          auth = process_key_auth(options)
+          process_key_auth(options)
         elsif options[:google_auth]
-          auth = options[:google_auth]
+          options[:google_auth]
         elsif options[:google_application_default]
-          auth = process_application_default_auth(options)
+          process_application_default_auth(options)
         else
-          auth = process_fallback_auth(options)
+          process_fallback_auth(options)
         end
-
-        ::Google::Apis::RequestOptions.default.authorization = auth
-        auth
       end
 
       ##

data/lib/fog/google/version.rb

@@ -1,5 +1,5 @@
 module Fog
   module Google
-    VERSION = "1.19.0".freeze
+    VERSION = "1.24.1".freeze
   end
 end

data/lib/fog/storage/google_json/models/file.rb

@@ -5,6 +5,7 @@ module Fog
         identity :key, :aliases => ["Key", :name]
 
         attribute :acl
+        attribute :uniform
         attribute :predefined_acl,      :aliases => ["predefinedAcl", :predefined_acl]
         attribute :cache_control,       :aliases => ["cacheControl", :cache_control]
         attribute :content_disposition, :aliases => ["contentDisposition", :content_disposition]
@@ -37,6 +38,10 @@ module Fog
           "publicReadWrite"
         ].freeze
 
+        def uniform=(enable)
+          @uniform=enable
+        end
+
         def predefined_acl=(new_predefined_acl)
           unless VALID_PREDEFINED_ACLS.include?(new_predefined_acl)
             raise ArgumentError.new("acl must be one of [#{VALID_PREDEFINED_ACLS.join(', ')}]")
@@ -111,8 +116,8 @@ module Fog
             FILE_INSERTABLE_FIELDS.map { |k| [k, attributes[k]] }
                                   .reject { |pair| pair[1].nil? }
           ]
-          
-          options[:predefined_acl] ||= @predefined_acl
+
+          options[:predefined_acl] ||= @predefined_acl unless @uniform
 
           service.put_object(directory.key, key, body, **options)
           self.content_length = Fog::Storage.get_body_size(body)

data/lib/fog/storage/google_json/real.rb

@@ -10,16 +10,12 @@ module Fog
 
         def initialize(options = {})
           shared_initialize(options[:google_project], GOOGLE_STORAGE_JSON_API_VERSION, GOOGLE_STORAGE_JSON_BASE_URL)
+          @options = options.dup
           options[:google_api_scope_url] = GOOGLE_STORAGE_JSON_API_SCOPE_URLS.join(" ")
           @host = options[:host] || "storage.googleapis.com"
 
           # TODO(temikus): Do we even need this client?
           @client = initialize_google_client(options)
-          # IAM client used for SignBlob API
-          @iam_service = ::Google::Apis::IamcredentialsV1::IAMCredentialsService.new
-          apply_client_options(@iam_service, {
-                                 google_api_scope_url: GOOGLE_STORAGE_JSON_IAM_API_SCOPE_URLS.join(" ")
-                               })
 
           @storage_json = ::Google::Apis::StorageV1::StorageService.new
           apply_client_options(@storage_json, options)
@@ -135,11 +131,24 @@ DATA
         #   See https://cloud.google.com/storage/docs/access-control/signed-urls-v2
         # @return [String] Signature binary blob
         def default_signer(string_to_sign)
-          key = OpenSSL::PKey::RSA.new(@storage_json.authorization.signing_key)
+          key = @storage_json.authorization.signing_key
+          key = OpenSSL::PKey::RSA.new(@storage_json.authorization.signing_key) unless key.respond_to?(:sign)
           digest = OpenSSL::Digest::SHA256.new
           return key.sign(digest, string_to_sign)
         end
 
+        # IAM client used for SignBlob API.
+        # Lazily initialize this since it requires another authorization request.
+        def iam_service
+          return @iam_service if defined?(@iam_service)
+
+          @iam_service = ::Google::Apis::IamcredentialsV1::IAMCredentialsService.new
+          apply_client_options(@iam_service, @options)
+          iam_options = @options.merge(google_api_scope_url: GOOGLE_STORAGE_JSON_IAM_API_SCOPE_URLS.join(" "))
+          @iam_service.authorization = initialize_auth(iam_options)
+          @iam_service
+        end
+
         ##
         # Fallback URL signer using the IAM SignServiceAccountBlob API, see
         #   Google::Apis::IamcredentialsV1::IAMCredentialsService#sign_service_account_blob
@@ -161,7 +170,7 @@ DATA
           )
 
           resource = "projects/-/serviceAccounts/#{google_access_id}"
-          response = @iam_service.sign_service_account_blob(resource, request)
+          response = iam_service.sign_service_account_blob(resource, request)
 
           return response.signed_blob
         end

data/lib/fog/storage/google_json/utils.rb

@@ -1,3 +1,5 @@
+require 'addressable'
+
 module Fog
   module Storage
     class GoogleJSON
@@ -19,10 +21,7 @@ module Fog
 
         def host_path_query(params, expires)
           params[:headers]["Date"] = expires.to_i
-          # implementation from CGI.escape, but without ' ' to  '+' conversion
-          params[:path] = params[:path].b.gsub(/([^a-zA-Z0-9_.\-~]+)/) { |m|
-            '%' + m.unpack('H2' * m.bytesize).join('%').upcase
-          }.gsub("%2F", "/")
+          params[:path] = ::Addressable::URI.encode_component(params[:path], ::Addressable::URI::CharacterClasses::PATH)
 
           query = []
 

data/lib/fog/storage/google_json.rb

@@ -29,7 +29,7 @@ module Fog
 
       # Version of IAM API used for blob signing, see Fog::Storage::GoogleJSON::Real#iam_signer
       GOOGLE_STORAGE_JSON_IAM_API_VERSION = "v1".freeze
-      GOOGLE_STORAGE_JSON_IAM_API_SCOPE_URLS = %w(https://www.googleapis.com/auth/devstorage.full_control).freeze
+      GOOGLE_STORAGE_JSON_IAM_API_SCOPE_URLS = %w(https://www.googleapis.com/auth/iam).freeze
 
       # TODO: Come up with a way to only request a subset of permissions.
       # https://cloud.google.com/storage/docs/json_api/v1/how-tos/authorizing

data/test/helpers/integration_test_helper.rb

@@ -17,9 +17,9 @@ TEST_ZONE = "us-central1-f".freeze
 TEST_REGION = "us-central1".freeze
 TEST_SIZE_GB = 10
 TEST_MACHINE_TYPE = "n1-standard-1".freeze
-TEST_IMAGE = "debian-9-stretch-v20180611".freeze
+TEST_IMAGE = "debian-11-bullseye-v20220920".freeze
 TEST_IMAGE_PROJECT = "debian-cloud".freeze
-TEST_IMAGE_FAMILY = "debian-9".freeze
+TEST_IMAGE_FAMILY = "debian-11".freeze
 
 # XXX This depends on a public image in gs://fog-test-bucket; there may be a better way to do this
 # The image was created like so: https://cloud.google.com/compute/docs/images/export-image
@@ -93,7 +93,7 @@ KEY
 RETRIABLE_TRIES = 3
 RETRIABLE_BASE_INTERVAL = 50
 
-class FogIntegrationTest < MiniTest::Test
+class FogIntegrationTest < Minitest::Test
   def namespaced_name
     "#{self.class}_#{name}"
   end

data/test/integration/compute/core_compute/test_servers.rb

@@ -132,6 +132,92 @@ class TestServers < FogIntegrationTest
     assert server.ready?
   end
 
+  def test_start_stop_discard_local_ssd
+    server = @factory.create
+
+    async = true
+    discard_local_ssd = true
+
+    server.stop(async, discard_local_ssd)
+    server.wait_for { stopped? }
+
+    assert server.stopped?
+  end
+
+  def test_attach_disk
+    # Creating server
+    server = @factory.create
+    server.wait_for { ready? }
+
+    disk_name = "fog-test-1-testservers-test-attach-disk-attachable"  # suffix forces disk name to differ from the existing disk
+    # Creating disk #{disk_name}
+    disk = @disks.create(
+      :name => disk_name,
+      :source_image => TEST_IMAGE,
+      :size_gb => 64
+    )
+    device_name = "#{disk.name}-device"
+
+    # Attaching disk #{disk.name} as device #{device_name}
+    self_link = "https://www.googleapis.com/compute/v1/projects/#{TEST_PROJECT}/zones/#{TEST_ZONE}/disks/#{disk.name}"
+    server.attach_disk(self_link, true, device_name: device_name)
+
+    # Waiting for attachment
+    disk.wait_for { ! users.nil? && users != []}
+
+    assert_equal "https://www.googleapis.com/compute/v1/projects/#{TEST_PROJECT}/zones/#{TEST_ZONE}/instances/#{server.name}", disk.users[0]
+
+    server.reload
+    server_attached_disk = server.disks.select{|d| d[:boot] == false}[0]
+    assert_equal device_name, server_attached_disk[:device_name]
+  end
+
+  def test_detach_disk
+    # Creating server
+    server = @factory.create
+    server.wait_for { ready? }
+
+    disk_name = "fog-test-1-testservers-test-detach-attachable"  # suffix forces disk name to differ from the existing disk
+    # Creating disk #{disk_name}
+    disk = @disks.create(
+      :name => disk_name,
+      :source_image => TEST_IMAGE,
+      :size_gb => 64
+    )
+    device_name = "#{disk.name}-device"
+
+    # Attaching disk #{disk.name} as device #{device_name}
+    self_link = "https://www.googleapis.com/compute/v1/projects/#{TEST_PROJECT}/zones/#{TEST_ZONE}/disks/#{disk.name}"
+    server.attach_disk(self_link, true, device_name: device_name)
+    disk.wait_for { ! users.nil? && users != []}
+
+    server.reload
+    server_attached_disk = server.disks.select{|d| d[:boot] == false}[0]
+    assert_equal device_name, server_attached_disk[:device_name]
+
+    # Detaching (synchronous) disk #{disk.name}
+    server.detach_disk(device_name, false)
+
+    disk.reload
+    assert disk.users.nil? || disk.users == []
+
+    # Re-attaching disk #{disk.name} as device #{device_name}
+    server.attach_disk(self_link, true, device_name: device_name)
+    disk.wait_for { ! users.nil? && users != []}
+
+    server.reload
+    server_attached_disk = server.disks.select{|d| d[:boot] == false}[0]
+    assert_equal device_name, server_attached_disk[:device_name]
+
+    # Detaching (async) disk #{disk.name}
+    server.detach_disk(device_name, true)
+
+    # Waiting for detachment
+    disk.wait_for { users.nil? || users == []}
+
+    assert disk.users.nil? || disk.users == []
+  end
+
   def test_reset_windows_password
     win_disk = @disks.create(
       :name => "fog-test-1-testservers-test-reset-windows-password-2",