gitlab-fog-google 1.13.0

data/lib/fog/storage/google_xml/requests/put_object_url.rb

@@ -0,0 +1,42 @@
+module Fog
+  module Storage
+    class GoogleXML
+      class Real
+        # Get an expiring object url from Google Storage for putting an object
+        #
+        # ==== Parameters
+        # * bucket_name<~String> - Name of bucket containing object
+        # * object_name<~String> - Name of object to get expiring url for
+        # * expires<~Time> - An expiry time for this url
+        #
+        # ==== Returns
+        # * response<~Excon::Response>:
+        #   * body<~String> - url for object
+        #
+        def put_object_url(bucket_name, object_name, expires, headers = {})
+          raise ArgumentError.new("bucket_name is required") unless bucket_name
+          raise ArgumentError.new("object_name is required") unless object_name
+          https_url({
+                      :headers  => headers,
+                      :host     => @host,
+                      :method   => "PUT",
+                      :path     => "#{bucket_name}/#{object_name}"
+                    }, expires)
+        end
+      end
+
+      class Mock
+        def put_object_url(bucket_name, object_name, expires, headers = {})
+          raise ArgumentError.new("bucket_name is required") unless bucket_name
+          raise ArgumentError.new("object_name is required") unless object_name
+          https_url({
+                      :headers  => headers,
+                      :host     => @host,
+                      :method   => "PUT",
+                      :path     => "#{bucket_name}/#{object_name}"
+                    }, expires)
+        end
+      end
+    end
+  end
+end

data/lib/fog/storage/google_xml/utils.rb

@@ -0,0 +1,83 @@
+module Fog
+  module Storage
+    class GoogleXML
+      module Utils
+        # https://cloud.google.com/storage/docs/access-control#predefined-acl
+        VALID_ACLS = [
+          "authenticated-read",
+          "bucket-owner-full-control",
+          "bucket-owner-read",
+          "private",
+          "project-private",
+          "public-read",
+          "public-read-write"
+        ].freeze
+
+        def http_url(params, expires)
+          "http://" << host_path_query(params, expires)
+        end
+
+        def https_url(params, expires)
+          "https://" << host_path_query(params, expires)
+        end
+
+        def url(params, expires)
+          Fog::Logger.deprecation("Fog::Storage::Google => #url is deprecated, use #https_url instead [light_black](#{caller.first})[/]")
+          https_url(params, expires)
+        end
+
+        private
+
+        def host_path_query(params, expires)
+          params[:headers]["Date"] = expires.to_i
+          params[:path] = Fog::Google.escape(params[:path]).gsub("%2F", "/")
+          query = []
+
+          if params[:query]
+            filtered = params[:query].reject { |k, v| k.nil? || v.nil? }
+            query = filtered.map { |k, v| [k.to_s, Fog::Google.escape(v)].join("=") }
+          end
+
+          query << "GoogleAccessId=#{@google_storage_access_key_id}"
+          query << "Signature=#{CGI.escape(signature(params))}"
+          query << "Expires=#{params[:headers]['Date']}"
+          "#{params[:host]}/#{params[:path]}?#{query.join('&')}"
+        end
+
+        def request_params(params)
+          subdomain = params[:host].split(".#{@host}").first
+          if @path_style || subdomain !~ /^(?!goog)(?:[a-z]|\d(?!\d{0,2}(?:\.\d{1,3}){3}$))(?:[a-z0-9]|\.(?![\.\-])|\-(?![\.])){1,61}[a-z0-9]$/
+            if subdomain =~ /_/
+              # https://github.com/fog/fog/pull/1258#issuecomment-10248620.
+              Fog::Logger.warning("fog: the specified google storage bucket name (#{subdomain}) is not DNS compliant (only characters a through z, digits 0 through 9, and the hyphen).")
+            else
+              # - Bucket names must contain only lowercase letters, numbers, dashes (-), underscores (_), and dots (.). Names containing dots require verification.
+              # - Bucket names must start and end with a number or letter.
+              # - Bucket names must contain 3 to 63 characters. Names containing dots can contain up to 222 characters, but each dot-separated component can be no longer than 63 characters.
+              # - Bucket names cannot be represented as an IP address in dotted-decimal notation (for example, 192.168.5.4).
+              # - Bucket names cannot begin with the "goog" prefix.
+              # - Also, for DNS compliance, you should not have a period adjacent to another period or dash. For example, ".." or "-." or ".-" are not acceptable.
+              Fog::Logger.warning("fog: the specified google storage bucket name (#{subdomain}) is not a valid dns name.  See: https://developers.google.com/storage/docs/bucketnaming") unless @path_style
+            end
+
+            params[:host] = params[:host].split("#{subdomain}.")[-1]
+            params[:path] =
+              if params[:path]
+                "#{subdomain}/#{params[:path]}"
+              else
+                subdomain.to_s
+              end
+
+            subdomain = nil
+          end
+
+          params[:subdomain] = subdomain if subdomain && subdomain != @host
+
+          params[:scheme] ||= @scheme
+          params[:port] ||= @port
+          params
+        end
+      end
+    end
+  end
+end

data/tasks/bundler.rake

@@ -0,0 +1,3 @@
+# This installs the tasks that help with gem creation and
+# publishing.
+Bundler::GemHelper.install_tasks

data/tasks/changelog.rake

@@ -0,0 +1,39 @@
+require "fileutils"
+
+# Helper method to insert text after a line that matches the regex
+def insert_after_line(file, insert, regex = /^## Next/)
+  tempfile = File.open("#{file}.tmp", "w")
+  f = File.new(file)
+  f.each do |line|
+    tempfile << line
+    next unless line =~ regex
+
+    tempfile << "\n"
+    tempfile << insert
+    tempfile << "\n"
+  end
+  f.close
+  tempfile.close
+
+  FileUtils.mv("#{file}.tmp", file)
+end
+
+# Extracts all changes that have been made after the latest pushed tag
+def changes_since_last_tag
+  `git --no-pager log $(git describe --tags --abbrev=0)..HEAD --grep="Merge" --pretty=format:"%t - %s%n%b%n"`
+end
+
+# Extracts all github users contributed since last tag
+def users_since_last_tag
+  `git --no-pager log $(git describe --tags --abbrev=0)..HEAD --grep="Merge" --pretty=format:"%s" | cut -d' ' -f 6 | cut -d/ -f1 | sort | uniq`
+end
+
+namespace :changelog do
+  task :generate do
+    insert_after_line("CHANGELOG.md", changes_since_last_tag, /^## Next/)
+    contributors = users_since_last_tag.split("\n").map { |name| "@" + name }
+    printf("Huge thanks to all our contributors 🎆\n")
+    printf("Special thanks to: " + contributors.join(" ") + "\n")
+    printf("\nI'll merge this and release the gem once all tests pass.\n")
+  end
+end

data/tasks/console.rake

@@ -0,0 +1,17 @@
+# From http://erniemiller.org/2014/02/05/7-lines-every-gems-rakefile-should-have/
+# with some modification.
+
+desc "Project IRB console"
+task :console do
+  require "bundler"
+  Bundler.require(:default, :development)
+
+  # Reload helper to avoid resetting the environment when debugging
+  def reload!
+    files = $LOADED_FEATURES.select { |feat| feat =~ /\/fog-google\// }
+    files.each { |file| load file }
+  end
+
+  ARGV.clear
+  Pry.start
+end

data/tasks/lint.rake

@@ -0,0 +1,9 @@
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new(:lint)
+
+namespace :lint do
+  task :changed do
+    sh("git status --porcelain | cut -c4- | grep '.rb' | xargs rubocop")
+  end
+end

data/tasks/test.rake

@@ -0,0 +1,115 @@
+require "rake/testtask"
+
+Rake::TestTask.new do |t|
+  t.description = "Run integration and unit tests"
+  t.libs << "test"
+  t.pattern = File.join("test", "**", "test_*.rb")
+  t.warning = false
+end
+
+namespace :test do
+  mock = ENV["FOG_MOCK"] || "true"
+  task :travis do
+    sh("bundle exec rake test:unit")
+  end
+
+  desc "Run all integration tests in parallel"
+  multitask :parallel => ["test:compute",
+                          "test:monitoring",
+                          "test:pubsub",
+                          "test:sql",
+                          "test:storage"]
+
+  Rake::TestTask.new do |t|
+    t.name = "unit"
+    t.description = "Run Unit tests"
+    t.libs << "test"
+    t.pattern = FileList["test/unit/**/test_*.rb"]
+    t.warning = false
+    t.verbose = true
+  end
+
+  # This autogenerates rake tasks based on test folder structures
+  # This is done to simplify running many test suites in parallel
+  COMPUTE_TEST_TASKS = []
+  Dir.glob("test/integration/compute/**").each do |task|
+    suite_collection = task.gsub(/test\/integration\/compute\//, "")
+    component_name = task.gsub(/test\/integration\//, "").split("/").first
+    Rake::TestTask.new(:"#{component_name}-#{suite_collection}") do |t|
+      t.libs << "test"
+      t.description = "Autotask - run #{component_name} integration tests - #{suite_collection}"
+      t.pattern = FileList["test/integration/#{component_name}/#{suite_collection}/test_*.rb"]
+      t.warning = false
+      t.verbose = true
+    end
+    COMPUTE_TEST_TASKS << "#{component_name}-#{suite_collection}"
+  end
+
+  desc "Run Compute API tests"
+  task :compute => COMPUTE_TEST_TASKS
+
+  desc "Run Compute API tests in parallel"
+  multitask :compute_parallel => COMPUTE_TEST_TASKS
+
+  Rake::TestTask.new do |t|
+    t.name = "monitoring"
+    t.description = "Run Monitoring API tests"
+    t.libs << "test"
+    t.pattern = FileList["test/integration/monitoring/test_*.rb"]
+    t.warning = false
+    t.verbose = true
+  end
+
+  Rake::TestTask.new do |t|
+    t.name = "pubsub"
+    t.description = "Run PubSub API tests"
+    t.libs << "test"
+    t.pattern = FileList["test/integration/pubsub/test_*.rb"]
+    t.warning = false
+    t.verbose = true
+  end
+
+  # This autogenerates rake tasks based on test folder structures
+  # This is done to simplify running many test suites in parallel
+  SQL_TEST_TASKS = []
+  Dir.glob("test/integration/sql/**").each do |task|
+    suite_collection = task.gsub(/test\/integration\/sql\//, "")
+    component_name = task.gsub(/test\/integration\//, "").split("/").first
+    Rake::TestTask.new(:"#{component_name}-#{suite_collection}") do |t|
+      t.libs << "test"
+      t.description = "Autotask - run #{component_name} integration tests - #{suite_collection}"
+      t.pattern = FileList["test/integration/#{component_name}/#{suite_collection}/test_*.rb"]
+      t.warning = false
+      t.verbose = true
+    end
+    SQL_TEST_TASKS << "#{component_name}-#{suite_collection}"
+  end
+
+  desc "Run SQL API tests"
+  task :sql => SQL_TEST_TASKS
+
+  desc "Run SQL API tests in parallel"
+  multitask :sql_parallel => SQL_TEST_TASKS
+
+  # TODO(temikus): Remove after v1 is renamed in pipeline
+  desc "Run SQL API tests - v1 compat alias"
+  task :"sql-sqlv2" => :sql
+
+  Rake::TestTask.new do |t|
+    t.name = "sql"
+    t.description = "Run SQL API tests"
+    t.libs << "test"
+    t.pattern = FileList["test/integration/sql/test_*.rb"]
+    t.warning = false
+    t.verbose = true
+  end
+
+  Rake::TestTask.new do |t|
+    t.name = "storage"
+    t.description = "Run Storage API tests"
+    t.libs << "test"
+    t.pattern = FileList["test/integration/storage/test_*.rb"]
+    t.warning = false
+    t.verbose = true
+  end
+end

data/test/helpers/client_helper.rb

@@ -0,0 +1,63 @@
+##
+# Helper mixin for tests (especially compute-based ones). Provide access to the
+# client in use via the #client method to use.
+
+module ClientHelper
+  # Check to see if an compute operation is finished.
+  #
+  # @param op [Google::Apis::ComputeV1::Operation] the operation object returned from an api call
+  # @return [Bolean] true if the operation is no longer executing, false
+  #   otherwise
+  def operation_finished?(op)
+    region = op.region
+    zone = op.zone
+    name = op.name
+
+    if zone.nil?
+      result = client.get_region_operation(region, name)
+    else
+      result = client.get_zone_operation(zone, name)
+    end
+    !%w(PENDING RUNNING).include?(result.status)
+  end
+
+  # Check to see if an SQL operation is finished.
+  #
+  # @param op [Google::Apis::SqladminV1beta4::Operation] the operation object
+  #   returned from an SQL Admin api call
+  # @return [Boolean] true if the operation is no longer executing, false
+  #   otherwise
+  def sql_operation_finished?(op)
+    result = client.get_operation(op.name)
+    !%w(PENDING RUNNING).include?(result.status)
+  end
+
+  # Pause execution until an operation (compute or sql) returned
+  # from a passed block is finished.
+  #
+  # @example Pause until server is provisioned
+  #   @result = wait_until_complete { client.insert_server(name, zone, opts) }
+  # @yieldreturn The resulting operation object from a block.
+  # @return the result of the operation
+
+  def wait_until_complete
+    result = yield
+
+    case result.kind
+    when "compute#operation"
+      region = result.region
+      zone = result.zone
+      Fog.wait_for { operation_finished?(result) }
+      if zone.nil?
+        client.get_region_operation(region, result.name)
+      else
+        client.get_zone_operation(zone, result.name)
+      end
+    when "sql#operation"
+      Fog.wait_for { sql_operation_finished?(result) }
+      client.get_operation(result.name)
+    else
+      result
+    end
+  end
+end

data/test/helpers/integration_test_helper.rb

@@ -0,0 +1,108 @@
+require "helpers/test_helper"
+require "helpers/test_collection"
+require "retriable"
+
+# Use :test credentials in ~/.fog for live integration testing
+# XXX not sure if this will work on Travis CI or not
+Fog.credential = :test
+
+# Enable test debugging, providing additional information in some methods
+DEBUG = ENV['FOG_TEST_DEBUG'] || false
+
+# Helpers
+
+TEST_PROJECT = Fog.credentials[:google_project]
+
+TEST_ZONE = "us-central1-f".freeze
+TEST_REGION = "us-central1".freeze
+TEST_SIZE_GB = 10
+TEST_MACHINE_TYPE = "n1-standard-1".freeze
+TEST_IMAGE = "debian-9-stretch-v20180611".freeze
+TEST_IMAGE_PROJECT = "debian-cloud".freeze
+TEST_IMAGE_FAMILY = "debian-9".freeze
+
+# XXX This depends on a public image in gs://fog-test-bucket; there may be a better way to do this
+# The image was created like so: https://cloud.google.com/compute/docs/images#export_an_image_to_google_cloud_storage
+TEST_RAW_DISK_SOURCE = "https://storage.googleapis.com/fog-testing-bucket/fog-test-raw-disk-source.image.tar.gz".freeze
+
+TEST_SQL_TIER = "db-n1-standard-1".freeze
+TEST_SQL_REGION = TEST_REGION
+
+# Test certificate/key for SSL Proxy tests, generated via:
+# $ openssl genrsa -out example.key 2048
+# $ openssl req -new -key example.key -out example.csr
+# ...
+# Common Name (eg, fully qualified host name) []:example.com
+# Email Address []:admin@example.com
+# ...
+# $ openssl x509 -req -in example.csr -signkey example.key -out example.crt
+
+TEST_PEM_CERTIFICATE = <<CERT
+-----BEGIN CERTIFICATE-----
+MIIC7DCCAdQCCQCDSZQ3PYJW6zANBgkqhkiG9w0BAQUFADA4MRQwEgYDVQQDDAtl
+eGFtcGxlLmNvbTEgMB4GCSqGSIb3DQEJARYRYWRtaW5AZXhhbXBsZS5jb20wHhcN
+MTgwNjI4MDEzMzA5WhcNMTgwNzI4MDEzMzA5WjA4MRQwEgYDVQQDDAtleGFtcGxl
+LmNvbTEgMB4GCSqGSIb3DQEJARYRYWRtaW5AZXhhbXBsZS5jb20wggEiMA0GCSqG
+SIb3DQEBAQUAA4IBDwAwggEKAoIBAQC2DQkr6HBIoVsi7I+uHtmmge1KvbWP4nwg
+tmKe8FtprjyaBcY729Sc0LTgkOh0krVem9nsxd/Lea47jHTV/vM03xsOuEKRKy3S
+Horlr0TBRvB7tSmZCH0jn4x+V/k2qdLKhiDAtbLZ1UC3SWu3OjbE9ozf3fNPmssT
+bK2sNesa2XlydzL8wyZMajGLs8CEHVqpZwLSPTeGB3LAhdRjfzJLijqe124otVIb
+3+tbxYZVY5GLVQIIR/DlwRzZLxptPALvHq3t3eOfC3eZImXqUzGmoug6nDrdKuEs
+C8yaoCNC3sSecoaV5Kyy70Hae+wFV7ocZGejm9AtuHXF1pgM4outAgMBAAEwDQYJ
+KoZIhvcNAQEFBQADggEBAFRYuOe7nwR8Sn7TDyzPcqau3yF2qYATtrDIzo7xmh3o
+T18FYCeX9gu/M9ytziM2lb/+WRqvcu55FrQiBz434E4wYH/IGUZK849V03dCro10
+Po5Hhayt5+tv5Ti0JqmJoC/HN0hNF+WQA8kikiteTTohIDj9rVX2uULA61rnHXSZ
+h6ZZRce3PdRKIUjOcQ4pIoI8QXSIToBsdY+Cieg9PHCt8mZhevU7COE1lfPFtdlY
+HJhg5xhHVUt40qsahpvUjnPBU8rJPPE06ougf5qjr0D8ZoYeuUQT0VJYwwacfpeb
+hs6AFOH5IAj6gjdIRhOWzYSxBYeyzJGfiPSbK4rowFM=
+-----END CERTIFICATE-----
+CERT
+
+TEST_PEM_CERTIFICATE_PRIVATE_KEY = <<KEY
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAtg0JK+hwSKFbIuyPrh7ZpoHtSr21j+J8ILZinvBbaa48mgXG
+O9vUnNC04JDodJK1XpvZ7MXfy3muO4x01f7zNN8bDrhCkSst0h6K5a9EwUbwe7Up
+mQh9I5+Mflf5NqnSyoYgwLWy2dVAt0lrtzo2xPaM393zT5rLE2ytrDXrGtl5cncy
+/MMmTGoxi7PAhB1aqWcC0j03hgdywIXUY38yS4o6ntduKLVSG9/rW8WGVWORi1UC
+CEfw5cEc2S8abTwC7x6t7d3jnwt3mSJl6lMxpqLoOpw63SrhLAvMmqAjQt7EnnKG
+leSssu9B2nvsBVe6HGRno5vQLbh1xdaYDOKLrQIDAQABAoIBAD5GA0cjwZT2rQgr
+R5LWNrmAZD1W246WeMNv4BhiO8LQuSYup3q+XeIuelD/AKUvsh7kzQzzOvSNcQ4p
+o6W4ClWho83LNeoWjRv9GqIq7Cf5LjYC6HHSt4vB/fsR+Mu8F8DzVKzW+pENI5AO
+62vH3AhQFixV7e7jEmhYmqf34a5S9U3+0OkzRRbFVA5rBxKK4xLqrX/app4/TKs6
+msn+qCJUAVxggcKIACF0slzbGuKfCSOyfPM4dmKV1n857wDu2YB9QwaaI+aHtR9S
+rassm1TCcqirUIOsTcAaZ5szjkhAhH+pg/xVjl/2ppz1W/m9cfNQ4zwle1YuUmpS
+OxyqUcECgYEA8jQ0zZG4J7MOX29o2F+D4VI5I1kp6uLsfhb8KztEgpgdMy+nGNsK
+wlOd2fJU3hK51dg82HKSSEzHHPud7O39pWEo3iSmgdZ+pXhjXe95x+bVPfP01eSp
+CSyMP8t6c9RxVtpVU1IbrjMXROuwEIxaIzMhT4YYDGOoSj2xLn43N9kCgYEAwGuv
+MJeCDMI2Zr+OA1N3FUk6Py8q1zxJN3Wt3CwSjp57K3g2LaqN6gwgIbN3d1fEV9cM
+ATtjjsFC+toFrP8BZ1wj3DY8Q4OsOmXng1V4bw4Y5xsT6SxXh7E0gU2L8MCjUdri
+WUUu++yRRcUpWUGRRMYx6cN529HPNbAPm5Y8QfUCgYAGpCHShS1cgU9ilIZ2cGAI
+XJ3Od1Jr6176shyl/tEJF6ytS6A0UUVBQNOyNy/WiwLndy9r6/BQ5TIMfGW/KmNr
+FnftZ2ndY4lDdkKbP8bCEXVFZpwPBV8RLlSGJ0krReb5r7DpQPYbV7FKpX/FZGPQ
+VUWTjaS5Kj5iEsD5+mH/OQKBgQCKhftc8/V/0eDwHz7RTikQfeMc0Yv530CmWGWN
+d3z0h0sMhEIcpgf8UjZfjJ+YnuqOghX2XRbTEnZxuLsVS49rJX37bl/8CrLWZ74/
+YiyNZoyu82NmHbH10bCG1ZjE/SmWKAmDUrb8TdZXcBTQWM+Hv1b3fu4fPe/6KoFR
+9Nn0cQKBgQC8BViTyfPTSuTEb8z7uzLEQk13MH+LTHFngFevWoq3OStEsmsIYK1K
+GcF1N9ZzPH024uwdcLjjwpul/avUHqzfp5ZILuI83Z/OqYYLiCQ0BbAQno7Cp5Be
+wLjafhPTSAIS0jijglJ7uzaSbFUW11fw1/EIqIFNe0R0Xf9lsyPxFA==
+-----END RSA PRIVATE KEY-----
+KEY
+
+# Retry module config - standard retriable gem settings
+# see: https://github.com/kamui/retriable
+RETRIABLE_TRIES = 3
+RETRIABLE_BASE_INTERVAL = 50
+
+class FogIntegrationTest < MiniTest::Test
+  def namespaced_name
+    "#{self.class}_#{name}"
+  end
+
+  def retry_on(klass)
+    Retriable.retriable(on: klass,
+                        tries: RETRIABLE_TRIES,
+                        base_interval: RETRIABLE_BASE_INTERVAL) do
+      yield
+    end
+  end
+end