gitlab-fog-google 1.13.0

data/lib/fog/storage/google_json/mock.rb

@@ -0,0 +1,27 @@
+module Fog
+  module Storage
+    class GoogleJSON
+      class Mock
+        include Utils
+        include Fog::Google::Shared
+
+        MockClient = Struct.new(:issuer)
+
+        def initialize(options = {})
+          shared_initialize(options[:google_project], GOOGLE_STORAGE_JSON_API_VERSION, GOOGLE_STORAGE_JSON_BASE_URL)
+          @client = MockClient.new('test')
+          @storage_json = MockClient.new('test')
+          @iam_service = MockClient.new('test')
+        end
+
+        def signature(_params)
+          "foo"
+        end
+
+        def google_access_id
+          "my-account@project.iam.gserviceaccount"
+        end
+      end
+    end
+  end
+end

data/lib/fog/storage/google_json/models/directories.rb

@@ -0,0 +1,41 @@
+module Fog
+  module Storage
+    class GoogleJSON
+      class Directories < Fog::Collection
+        model Fog::Storage::GoogleJSON::Directory
+
+        def all(opts = {})
+          data = service.list_buckets(opts).to_h[:items] || []
+          load(data)
+        end
+
+        def get(bucket_name, options = {})
+          if_metageneration_match = options[:if_metageneration_match]
+          if_metageneration_not_match = options[:if_metageneration_not_match]
+          projection = options[:projection]
+
+          data = service.get_bucket(
+            bucket_name,
+            :if_metageneration_match => if_metageneration_match,
+            :if_metageneration_not_match => if_metageneration_not_match,
+            :projection => projection
+          ).to_h
+
+          directory = new(data)
+          # Because fog-aws accepts these arguments on files at the
+          # directories.get level, we need to preload the directory files
+          # with these attributes here.
+          files_attr_names = %i(delimiter page_token max_results prefix)
+
+          file_opts = options.select { |k, _| files_attr_names.include? k }
+          directory.files(file_opts)
+          directory
+        rescue ::Google::Apis::ClientError => e
+          # metageneration check failures returns HTTP 412 Precondition Failed
+          raise e unless e.status_code == 404 || e.status_code == 412
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/fog/storage/google_json/models/directory.rb

@@ -0,0 +1,61 @@
+module Fog
+  module Storage
+    class GoogleJSON
+      ##
+      # Represents a Google Storage bucket
+      class Directory < Fog::Model
+        identity :key, :aliases => ["Name", "name", :name]
+
+        attribute :acl
+        attribute :billing
+        attribute :cors
+        attribute :default_object_acl, aliases => "defaultObjectAcl"
+        attribute :etag
+        attribute :id
+        attribute :kind
+        attribute :labels
+        attribute :lifecycle
+        attribute :location
+        attribute :logging
+        attribute :metageneration
+        attribute :name
+        attribute :owner
+        attribute :project_number, aliases => "projectNumber"
+        attribute :self_link, aliases => "selfLink"
+        attribute :storage_class, aliases => "storageClass"
+        attribute :time_created, aliases => "timeCreated"
+        attribute :updated
+        attribute :versioning
+        attribute :website
+
+        def destroy
+          requires :key
+          service.delete_bucket(key)
+          true
+        rescue ::Google::Apis::ClientError => e
+          raise e unless e.status_code == 404
+          false
+        end
+
+        def files(attr = {})
+          @files ||= begin
+            Fog::Storage::GoogleJSON::Files.new(
+              attr.merge(:directory => self, :service => service)
+            )
+          end
+        end
+
+        def public_url
+          requires :key
+          "#{GOOGLE_STORAGE_BUCKET_BASE_URL}#{key}"
+        end
+
+        def save
+          requires :key
+          service.put_bucket(key, attributes)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/fog/storage/google_json/models/file.rb

@@ -0,0 +1,126 @@
+module Fog
+  module Storage
+    class GoogleJSON
+      class File < Fog::Model
+        identity :key, :aliases => ["Key", :name]
+
+        attribute :acl
+        attribute :predefined_acl,      :aliases => ["predefinedAcl", :predefined_acl]
+        attribute :cache_control,       :aliases => ["cacheControl", :cache_control]
+        attribute :content_disposition, :aliases => ["contentDisposition", :content_disposition]
+        attribute :content_encoding,    :aliases => ["contentEncoding", :content_encoding]
+        attribute :content_length,      :aliases => ["size", :size], :type => :integer
+        attribute :content_md5,         :aliases => ["md5Hash", :md5_hash]
+        attribute :content_type,        :aliases => ["contentType", :content_type]
+        attribute :crc32c
+        attribute :etag,                :aliases => ["etag", :etag]
+        attribute :time_created,        :aliases => ["timeCreated", :time_created]
+        attribute :last_modified,       :aliases => ["updated", :updated]
+        attribute :generation
+        attribute :metageneration
+        attribute :metadata,            :aliases => ["metadata", :metadata]
+        attribute :self_link,           :aliases => ["selfLink", :self_link]
+        attribute :media_link,          :aliases => ["mediaLink", :media_link]
+        attribute :owner
+        attribute :storage_class, :aliases => "storageClass"
+
+        # attribute :body
+
+        # https://cloud.google.com/storage/docs/access-control/lists#predefined-acls
+        VALID_PREDEFINED_ACLS = [
+          "authenticatedRead",
+          "bucketOwnerFullControl",
+          "bucketOwnerRead",
+          "private",
+          "projectPrivate",
+          "publicRead",
+          "publicReadWrite"
+        ].freeze
+
+        def predefined_acl=(new_predefined_acl)
+          unless VALID_PREDEFINED_ACLS.include?(new_predefined_acl)
+            raise ArgumentError.new("acl must be one of [#{VALID_PREDEFINED_ACLS.join(', ')}]")
+          end
+          @predefined_acl = new_predefined_acl
+        end
+
+        def body
+          last_modified && (file = collection.get(identity)) ? attributes[:body] ||= file.body : attributes[:body] ||= ""
+        end
+
+        def body=(new_body)
+          attributes[:body] = new_body
+        end
+
+        attr_reader :directory
+
+        def copy(target_directory_key, target_file_key, options = {})
+          requires :directory, :key
+          service.copy_object(directory.key, key, target_directory_key, target_file_key, options)
+          target_directory = service.directories.new(:key => target_directory_key)
+          target_directory.files.get(target_file_key)
+        end
+
+        def destroy
+          requires :directory, :key
+          service.delete_object(directory.key, key)
+          true
+        rescue ::Google::Apis::ClientError => e
+          raise e unless e.status_code == 404
+          false
+        end
+
+        def public=(new_public)
+          unless new_public.nil?
+            if new_public
+              @predefined_acl = "publicRead"
+            else
+              @predefined_acl = "projectPrivate"
+            end
+          end
+          new_public
+        end
+
+        def public_url
+          requires :directory, :key
+          "https://storage.googleapis.com/#{directory.key}/#{key}"
+        end
+
+        FILE_INSERTABLE_FIELDS = %i(
+          content_type
+          predefined_acl
+          cache_control
+          content_disposition
+          content_encoding
+          metadata
+        ).freeze
+
+        def save
+          requires :body, :directory, :key
+
+          options = Hash[
+            FILE_INSERTABLE_FIELDS.map { |k| [k, attributes[k]] }
+                                  .reject { |pair| pair[1].nil? }
+          ]
+          
+          options[:predefined_acl] ||= @predefined_acl
+
+          service.put_object(directory.key, key, body, options)
+          self.content_length = Fog::Storage.get_body_size(body)
+          self.content_type ||= Fog::Storage.get_content_type(body)
+          true
+        end
+
+        # params[:expires] : Eg: Time.now to integer value.
+        def url(expires, options = {})
+          requires :key
+          collection.get_https_url(key, expires, options)
+        end
+
+        private
+
+        attr_writer :directory
+      end
+    end
+  end
+end

data/lib/fog/storage/google_json/models/files.rb

@@ -0,0 +1,64 @@
+module Fog
+  module Storage
+    class GoogleJSON
+      class Files < Fog::Collection
+        model Fog::Storage::GoogleJSON::File
+
+        extend Fog::Deprecation
+        deprecate :get_url, :get_https_url
+
+        attribute :common_prefixes, :aliases => "CommonPrefixes"
+        attribute :delimiter,       :aliases => "Delimiter"
+        attribute :directory
+        attribute :page_token,      :aliases => %w(pageToken page_token)
+        attribute :max_results,     :aliases => ["MaxKeys", "max-keys"]
+        attribute :prefix,          :aliases => "Prefix"
+
+        def all(options = {})
+          requires :directory
+          data = service.list_objects(directory.key, attributes.merge(options))
+                        .to_h[:items] || []
+          load(data)
+        end
+
+        alias_method :each_file_this_page, :each
+        def each
+          if block_given?
+            subset = dup.all
+
+            subset.each_file_this_page { |f| yield f }
+          end
+          self
+        end
+
+        def get(key, options = {}, &block)
+          requires :directory
+          data = service.get_object(directory.key, key, options, &block).to_h
+          new(data)
+        rescue ::Google::Apis::ClientError => e
+          raise e unless e.status_code == 404
+          nil
+        end
+
+        def get_https_url(key, expires, options = {})
+          requires :directory
+          service.get_object_https_url(directory.key, key, expires, options)
+        end
+
+        def metadata(key, options = {})
+          requires :directory
+          data = service.get_object_metadata(directory.key, key, options).to_h
+          new(data)
+        rescue ::Google::Apis::ClientError
+          nil
+        end
+        alias_method :head, :metadata
+
+        def new(opts = {})
+          requires :directory
+          super({ :directory => directory }.merge(opts))
+        end
+      end
+    end
+  end
+end

data/lib/fog/storage/google_json/real.rb

@@ -0,0 +1,171 @@
+module Fog
+  module Storage
+    class GoogleJSON
+      class Real
+        include Utils
+        include Fog::Google::Shared
+
+        attr_accessor :client
+        attr_reader :storage_json
+
+        def initialize(options = {})
+          shared_initialize(options[:google_project], GOOGLE_STORAGE_JSON_API_VERSION, GOOGLE_STORAGE_JSON_BASE_URL)
+          options[:google_api_scope_url] = GOOGLE_STORAGE_JSON_API_SCOPE_URLS.join(" ")
+          @host = options[:host] || "storage.googleapis.com"
+
+          # TODO(temikus): Do we even need this client?
+          @client = initialize_google_client(options)
+          # IAM client used for SignBlob API
+          @iam_service = ::Google::Apis::IamcredentialsV1::IAMCredentialsService.new
+          apply_client_options(@iam_service, {
+                                 google_api_scope_url: GOOGLE_STORAGE_JSON_IAM_API_SCOPE_URLS.join(" ")
+                               })
+
+          @storage_json = ::Google::Apis::StorageV1::StorageService.new
+          apply_client_options(@storage_json, options)
+
+          @storage_json.client_options.open_timeout_sec = options[:open_timeout_sec] if options[:open_timeout_sec]
+          @storage_json.client_options.read_timeout_sec = options[:read_timeout_sec] if options[:read_timeout_sec]
+          @storage_json.client_options.send_timeout_sec = options[:send_timeout_sec] if options[:send_timeout_sec]
+        end
+
+        def signature(params)
+          string_to_sign = <<-DATA
+#{params[:method]}
+#{params[:headers]['Content-MD5']}
+#{params[:headers]['Content-Type']}
+#{params[:headers]['Date']}
+DATA
+
+          google_headers = {}
+          canonical_google_headers = ""
+          params[:headers].each do |key, value|
+            google_headers[key] = value if key[0..6] == "x-goog-"
+          end
+
+          google_headers = google_headers.sort_by { |a| a[0] }
+          google_headers.each do |key, value|
+            canonical_google_headers << "#{key}:#{value}\n"
+          end
+          string_to_sign << canonical_google_headers.to_s
+
+          canonical_resource = "/"
+          if subdomain = params.delete(:subdomain)
+            canonical_resource << "#{CGI.escape(subdomain).downcase}/"
+          end
+          canonical_resource << params[:path].to_s
+          canonical_resource << "?"
+          (params[:query] || {}).each_key do |key|
+            if %w(acl cors location logging requestPayment versions versioning).include?(key)
+              canonical_resource << "#{key}&"
+            end
+          end
+          canonical_resource.chop!
+          string_to_sign << canonical_resource.to_s
+
+          # TODO(temikus): make signer configurable or add ability to supply your own via lambda
+          if !@storage_json.authorization.signing_key.nil?
+            signed_string = default_signer(string_to_sign)
+          else
+            # If client doesn't contain signing key attempt to auth via IAM SignBlob API
+            signed_string = iam_signer(string_to_sign)
+          end
+
+          Base64.encode64(signed_string).chomp!
+        end
+
+        private
+
+        def google_access_id
+          @google_access_id ||= get_google_access_id
+        end
+
+        ##
+        # Fetches the google service account name
+        #
+        # @return [String] Service account name, typically needed for GoogleAccessId, e.g.
+        #   my-account@project.iam.gserviceaccount
+        # @raises [Fog::Errors::Error] If authorisation is incorrect or inapplicable for current action
+        def get_google_access_id
+          if @storage_json.authorization.is_a?(::Google::Auth::UserRefreshCredentials)
+            raise Fog::Errors::Error.new("User / Application Default Credentials are not supported for storage"\
+                                         "url signing, please use a service account or metadata authentication.")
+          end
+
+          if !@storage_json.authorization.issuer.nil?
+            return @storage_json.authorization.issuer
+          else
+            get_access_id_from_metadata
+          end
+        end
+
+        ##
+        # Attempts to fetch the google service account name from metadata using Google::Cloud::Env
+        #
+        # @return [String] Service account name, typically needed for GoogleAccessId, e.g.
+        #   my-account@project.iam.gserviceaccount
+        # @raises [Fog::Errors::Error] If Metadata service is not available or returns an invalid response
+        def get_access_id_from_metadata
+          if @google_cloud_env.metadata?
+            access_id = @google_cloud_env.lookup_metadata("instance", "service-accounts/default/email")
+          else
+            raise Fog::Errors::Error.new("Metadata service not available, unable to retrieve service account info.")
+          end
+
+          if access_id.nil?
+            raise Fog::Errors::Error.new("Metadata service found but didn't return data." \
+               "Please file a bug: https://github.com/fog/fog-google")
+          end
+
+          return access_id
+        end
+
+        ##
+        # Default url signer using service account keys
+        #
+        # @param [String] string_to_sign Special collection of headers and options for V2 storage signing, e.g.:
+        #
+        #   StringToSign = HTTP_Verb + "\n" +
+        #                  Content_MD5 + "\n" +
+        #                  Content_Type + "\n" +
+        #                  Expires + "\n" +
+        #                  Canonicalized_Extension_Headers +
+        #                  Canonicalized_Resource
+        #
+        #   See https://cloud.google.com/storage/docs/access-control/signed-urls-v2
+        # @return [String] Signature binary blob
+        def default_signer(string_to_sign)
+          key = OpenSSL::PKey::RSA.new(@storage_json.authorization.signing_key)
+          digest = OpenSSL::Digest::SHA256.new
+          return key.sign(digest, string_to_sign)
+        end
+
+        ##
+        # Fallback URL signer using the IAM SignServiceAccountBlob API, see
+        #   Google::Apis::IamcredentialsV1::IAMCredentialsService#sign_service_account_blob
+        #
+        # @param [String] string_to_sign Special collection of headers and options for V2 storage signing, e.g.:
+        #
+        #   StringToSign = HTTP_Verb + "\n" +
+        #                  Content_MD5 + "\n" +
+        #                  Content_Type + "\n" +
+        #                  Expires + "\n" +
+        #                  Canonicalized_Extension_Headers +
+        #                  Canonicalized_Resource
+        #
+        #   See https://cloud.google.com/storage/docs/access-control/signed-urls-v2
+        # @return [String] Signature binary blob
+        def iam_signer(string_to_sign)
+          request = {
+            "payload": string_to_sign
+          }
+
+          resource = "projects/-/serviceAccounts/#{google_access_id}"
+          response = @iam_service.sign_service_account_blob resource, request, {}
+
+          return response.signed_blob
+        end
+      end
+    end
+  end
+end