fog-google 1.10.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +3 -3
- data/CHANGELOG.md +20 -0
- data/README.md +37 -0
- data/fog-google.gemspec +2 -2
- data/lib/fog/compute/google.rb +0 -4
- data/lib/fog/compute/google/models/server.rb +3 -3
- data/lib/fog/compute/google/requests/set_server_metadata.rb +1 -1
- data/lib/fog/google/requests/pubsub/pull_subscription.rb +5 -1
- data/lib/fog/google/shared.rb +7 -2
- data/lib/fog/google/version.rb +1 -1
- data/lib/fog/storage/google_json.rb +4 -0
- data/lib/fog/storage/google_json/mock.rb +6 -0
- data/lib/fog/storage/google_json/real.rb +106 -3
- data/lib/fog/storage/google_json/utils.rb +1 -1
- data/lib/fog/storage/google_xml/requests/get_bucket.rb +0 -1
- data/lib/fog/storage/google_xml/requests/head_object.rb +7 -6
- data/tasks/changelog.rake +8 -1
- data/test/helpers/integration_test_helper.rb +15 -1
- data/test/integration/monitoring/test_timeseries.rb +11 -25
- data/test/integration/pubsub/test_pubsub_models.rb +3 -3
- data/test/integration/pubsub/test_pubsub_requests.rb +2 -2
- data/test/integration/sql/test_certs.rb +0 -1
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e41bc807cc19b9984780ffcc9c253bac3ec3d13d
|
4
|
+
data.tar.gz: 69df7076d8eda3dfb5e22b1afcdb2ef97ee16e43
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: af8cff6ac06cab566a31db97eb6b2065c53a74be8a77e133b4816f132a9b8079d7ff01e0031a393120bc3c98e8d680371ebfb07c72c5ddabded8f96d6f4d8b54
|
7
|
+
data.tar.gz: f81555df397158c969d85dd2b3215842e46f2e2217371e54fae1125fef349b76ea8d02c8218ebe3dc9ba04c6965e9c93ba96db2f501d070dfaf060a772badc07
|
data/.rubocop.yml
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
# Fog custom enforced styles
|
2
|
-
|
2
|
+
Layout/LineLength:
|
3
3
|
Enabled: false
|
4
4
|
|
5
5
|
Style/ConditionalAssignment:
|
@@ -94,7 +94,7 @@ Naming/PredicateName:
|
|
94
94
|
- is_
|
95
95
|
- has_
|
96
96
|
- have_
|
97
|
-
|
97
|
+
ForbiddenPrefixes:
|
98
98
|
- is_
|
99
99
|
Exclude:
|
100
100
|
- spec/**/*
|
@@ -246,7 +246,7 @@ Style/WhenThen:
|
|
246
246
|
Lint/EachWithObjectArgument:
|
247
247
|
Description: Check for immutable argument given to each_with_object.
|
248
248
|
Enabled: true
|
249
|
-
Lint/
|
249
|
+
Lint/SuppressedException:
|
250
250
|
Description: Don't suppress exception.
|
251
251
|
StyleGuide: https://github.com/bbatsov/ruby-style-guide#dont-hide-exceptions
|
252
252
|
Enabled: false
|
data/CHANGELOG.md
CHANGED
@@ -6,6 +6,26 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html
|
|
6
6
|
|
7
7
|
## Next
|
8
8
|
|
9
|
+
## 1.11.0
|
10
|
+
|
11
|
+
### User-facing
|
12
|
+
|
13
|
+
#### Added
|
14
|
+
|
15
|
+
- \#503 - Add fallback URL signing mechanism via IAM SignBlob API [temikus]
|
16
|
+
|
17
|
+
#### Fixed
|
18
|
+
|
19
|
+
- \#498 Add `:idempotent` flag to Fog::Storage::GoogleXML::Real#head_object, fixing
|
20
|
+
`Excon::Error::Socket: end of file reached (EOFError)` in certain scenarios, see \#416 [temikus]
|
21
|
+
- \#500 Set default options automatically if missing in `Pubsub#pull_subscription`
|
22
|
+
|
23
|
+
### Development changes
|
24
|
+
|
25
|
+
#### Fixed
|
26
|
+
|
27
|
+
- \#501 DRY'ed up the retry methods in monitoring tests [temikus]
|
28
|
+
- \#500 Cleanup unneeded constants in Server model, fix flaky PubSub tests [temikus]
|
9
29
|
|
10
30
|
## 1.10.0
|
11
31
|
|
data/README.md
CHANGED
@@ -108,6 +108,43 @@ defined:
|
|
108
108
|
=> [ <Fog::Compute::Google::Server ... ]
|
109
109
|
```
|
110
110
|
|
111
|
+
#### CarrierWave integration
|
112
|
+
|
113
|
+
It is common to integrate Fog with Carrierwave. Here's a minimal config that's commonly put in `config/initializers/carrierwave.rb`:
|
114
|
+
|
115
|
+
```
|
116
|
+
CarrierWave.configure do |config|
|
117
|
+
config.fog_provider = 'fog/google'
|
118
|
+
config.fog_credentials = {
|
119
|
+
provider: 'Google',
|
120
|
+
google_project: Rails.application.secrets.google_cloud_storage_project_name,
|
121
|
+
google_json_key_string: Rails.application.secrets.google_cloud_storage_credential_content
|
122
|
+
# can optionally use google_json_key_location if using an actual file;
|
123
|
+
}
|
124
|
+
config.fog_directory = Rails.application.secrets.google_cloud_storage_bucket_name
|
125
|
+
end
|
126
|
+
```
|
127
|
+
|
128
|
+
This needs a corresponding secret in `config/secrets.yml`, e.g.:
|
129
|
+
|
130
|
+
```
|
131
|
+
development:
|
132
|
+
google_cloud_storage_project_name: your-project-name
|
133
|
+
google_cloud_storage_credential_content: '{
|
134
|
+
"type": "service_account",
|
135
|
+
"project_id": "your-project-name",
|
136
|
+
"private_key_id": "REDACTED",
|
137
|
+
"private_key": "-----BEGIN PRIVATE KEY-----REDACTED-----END PRIVATE KEY-----\n",
|
138
|
+
"client_email": "REDACTED@your-project-name.iam.gserviceaccount.com",
|
139
|
+
"client_id": "REDACTED",
|
140
|
+
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
|
141
|
+
"token_uri": "https://accounts.google.com/o/oauth2/token",
|
142
|
+
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
|
143
|
+
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/REDACTED%40your-project-name.iam.gserviceaccount.com"
|
144
|
+
}'
|
145
|
+
google_cloud_storage_bucket_name: your-bucket-name
|
146
|
+
```
|
147
|
+
|
111
148
|
#### SSH-ing into instances
|
112
149
|
|
113
150
|
If you want to be able to bootstrap SSH-able instances, (using `servers.bootstrap`,) be sure you have a key in `~/.ssh/id_rsa` and `~/.ssh/id_rsa.pub`
|
data/fog-google.gemspec
CHANGED
@@ -25,9 +25,9 @@ Gem::Specification.new do |spec|
|
|
25
25
|
spec.add_dependency "fog-json", "~> 1.2"
|
26
26
|
spec.add_dependency "fog-xml", "~> 0.1.0"
|
27
27
|
|
28
|
-
# Hard Requirement as of 1.0
|
29
28
|
spec.add_dependency "google-api-client", ">= 0.32", "< 0.34"
|
30
|
-
|
29
|
+
spec.add_dependency "google-cloud-env", "~> 1.2"
|
30
|
+
|
31
31
|
# Debugger
|
32
32
|
spec.add_development_dependency "pry"
|
33
33
|
|
data/lib/fog/compute/google.rb
CHANGED
@@ -27,10 +27,6 @@ module Fog
|
|
27
27
|
https://www.googleapis.com/auth/ndev.cloudman
|
28
28
|
https://www.googleapis.com/auth/cloud-platform).freeze
|
29
29
|
GOOGLE_COMPUTE_DEFAULT_NETWORK = "default".freeze
|
30
|
-
# TODO: Do we need those constants?
|
31
|
-
RUNNING = "RUNNING".freeze
|
32
|
-
PROVISIONING = "PROVISIONING".freeze
|
33
|
-
STAGING = "STAGING".freeze
|
34
30
|
|
35
31
|
request_path "fog/compute/google/requests"
|
36
32
|
request :add_backend_service_backends
|
@@ -447,7 +447,7 @@ module Fog
|
|
447
447
|
#
|
448
448
|
# @return [TrueClass or FalseClass]
|
449
449
|
def provisioning?
|
450
|
-
status == PROVISIONING
|
450
|
+
status == "PROVISIONING"
|
451
451
|
end
|
452
452
|
|
453
453
|
# Check if instance is staging. On staging vs. provisioning difference:
|
@@ -455,7 +455,7 @@ module Fog
|
|
455
455
|
#
|
456
456
|
# @return [TrueClass or FalseClass]
|
457
457
|
def staging?
|
458
|
-
status == STAGING
|
458
|
+
status == "STAGING"
|
459
459
|
end
|
460
460
|
|
461
461
|
# Check if instance is stopped.
|
@@ -469,7 +469,7 @@ module Fog
|
|
469
469
|
#
|
470
470
|
# @return [TrueClass or FalseClass]
|
471
471
|
def ready?
|
472
|
-
status == RUNNING
|
472
|
+
status == "RUNNING"
|
473
473
|
end
|
474
474
|
|
475
475
|
def zone_name
|
@@ -21,7 +21,7 @@ module Fog
|
|
21
21
|
# fingerprint = instance.metadata['fingerprint']
|
22
22
|
# @param [Hash] metadata A new metadata object
|
23
23
|
# Should have the following structure:
|
24
|
-
# {
|
24
|
+
# {'foo' => 'bar', 'baz'=>'foo'}
|
25
25
|
#
|
26
26
|
# @returns [::Google::Apis::ComputeV1::Operation] set operation
|
27
27
|
def set_server_metadata(instance, zone, fingerprint, metadata_items = [])
|
@@ -16,7 +16,11 @@ module Fog
|
|
16
16
|
# @option options [Number] :max_messages maximum number of messages to
|
17
17
|
# retrieve (defaults to 10)
|
18
18
|
# @see https://cloud.google.com/pubsub/reference/rest/v1/projects.subscriptions/pull
|
19
|
-
def pull_subscription(subscription, options = {
|
19
|
+
def pull_subscription(subscription, options = {})
|
20
|
+
defaults = { :return_immediately => true,
|
21
|
+
:max_messages => 10 }
|
22
|
+
options = defaults.merge(options)
|
23
|
+
|
20
24
|
pull_request = ::Google::Apis::PubsubV1::PullRequest.new(
|
21
25
|
:return_immediately => options[:return_immediately],
|
22
26
|
:max_messages => options[:max_messages]
|
data/lib/fog/google/shared.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
require "google-cloud-env"
|
2
|
+
|
1
3
|
module Fog
|
2
4
|
module Google
|
3
5
|
module Shared
|
@@ -14,6 +16,9 @@ module Fog
|
|
14
16
|
@project = project
|
15
17
|
@api_version = api_version
|
16
18
|
@api_url = base_url + api_version + "/projects/"
|
19
|
+
# google-cloud-env allows us to figure out which GCP runtime we're running in and query metadata
|
20
|
+
# e.g. whether we're running in GCE/GKE/AppEngine or what region the instance is running in
|
21
|
+
@google_cloud_env = ::Google::Cloud::Env.get
|
17
22
|
end
|
18
23
|
|
19
24
|
##
|
@@ -33,14 +38,14 @@ module Fog
|
|
33
38
|
def initialize_google_client(options)
|
34
39
|
# NOTE: loaded here to avoid requiring this as a core Fog dependency
|
35
40
|
begin
|
36
|
-
#
|
37
|
-
# of funky things, like this nonsense.
|
41
|
+
# TODO: google-api-client is in gemspec now, re-assess if this initialization logic is still needed
|
38
42
|
require "google/apis/monitoring_#{Fog::Google::Monitoring::GOOGLE_MONITORING_API_VERSION}"
|
39
43
|
require "google/apis/compute_#{Fog::Compute::Google::GOOGLE_COMPUTE_API_VERSION}"
|
40
44
|
require "google/apis/dns_#{Fog::DNS::Google::GOOGLE_DNS_API_VERSION}"
|
41
45
|
require "google/apis/pubsub_#{Fog::Google::Pubsub::GOOGLE_PUBSUB_API_VERSION}"
|
42
46
|
require "google/apis/sqladmin_#{Fog::Google::SQL::GOOGLE_SQL_API_VERSION}"
|
43
47
|
require "google/apis/storage_#{Fog::Storage::GoogleJSON::GOOGLE_STORAGE_JSON_API_VERSION}"
|
48
|
+
require "google/apis/iamcredentials_#{Fog::Storage::GoogleJSON::GOOGLE_STORAGE_JSON_IAM_API_VERSION}"
|
44
49
|
require "googleauth"
|
45
50
|
rescue LoadError => error
|
46
51
|
Fog::Errors::Error.new("Please install the google-api-client (>= 0.9) gem before using this provider")
|
data/lib/fog/google/version.rb
CHANGED
@@ -27,6 +27,10 @@ module Fog
|
|
27
27
|
GOOGLE_STORAGE_JSON_BASE_URL = "https://www.googleapis.com/storage/".freeze
|
28
28
|
GOOGLE_STORAGE_BUCKET_BASE_URL = "https://storage.googleapis.com/".freeze
|
29
29
|
|
30
|
+
# Version of IAM API used for blob signing, see Fog::Storage::GoogleJSON::Real#iam_signer
|
31
|
+
GOOGLE_STORAGE_JSON_IAM_API_VERSION = "v1".freeze
|
32
|
+
GOOGLE_STORAGE_JSON_IAM_API_SCOPE_URLS = %w(https://www.googleapis.com/auth/devstorage.full_control).freeze
|
33
|
+
|
30
34
|
# TODO: Come up with a way to only request a subset of permissions.
|
31
35
|
# https://cloud.google.com/storage/docs/json_api/v1/how-tos/authorizing
|
32
36
|
GOOGLE_STORAGE_JSON_API_SCOPE_URLS = %w(https://www.googleapis.com/auth/devstorage.full_control).freeze
|
@@ -10,11 +10,17 @@ module Fog
|
|
10
10
|
def initialize(options = {})
|
11
11
|
shared_initialize(options[:google_project], GOOGLE_STORAGE_JSON_API_VERSION, GOOGLE_STORAGE_JSON_BASE_URL)
|
12
12
|
@client = MockClient.new('test')
|
13
|
+
@storage_json = MockClient.new('test')
|
14
|
+
@iam_service = MockClient.new('test')
|
13
15
|
end
|
14
16
|
|
15
17
|
def signature(_params)
|
16
18
|
"foo"
|
17
19
|
end
|
20
|
+
|
21
|
+
def google_access_id
|
22
|
+
"my-account@project.iam.gserviceaccount"
|
23
|
+
end
|
18
24
|
end
|
19
25
|
end
|
20
26
|
end
|
@@ -13,7 +13,14 @@ module Fog
|
|
13
13
|
options[:google_api_scope_url] = GOOGLE_STORAGE_JSON_API_SCOPE_URLS.join(" ")
|
14
14
|
@host = options[:host] || "storage.googleapis.com"
|
15
15
|
|
16
|
+
# TODO(temikus): Do we even need this client?
|
16
17
|
@client = initialize_google_client(options)
|
18
|
+
# IAM client used for SignBlob API
|
19
|
+
@iam_service = ::Google::Apis::IamcredentialsV1::IAMCredentialsService.new
|
20
|
+
apply_client_options(@iam_service, {
|
21
|
+
google_api_scope_url: GOOGLE_STORAGE_JSON_IAM_API_SCOPE_URLS.join(" ")
|
22
|
+
})
|
23
|
+
|
17
24
|
@storage_json = ::Google::Apis::StorageV1::StorageService.new
|
18
25
|
apply_client_options(@storage_json, options)
|
19
26
|
|
@@ -56,12 +63,108 @@ DATA
|
|
56
63
|
canonical_resource.chop!
|
57
64
|
string_to_sign << canonical_resource.to_s
|
58
65
|
|
59
|
-
|
60
|
-
|
61
|
-
|
66
|
+
# TODO(temikus): make signer configurable or add ability to supply your own via lambda
|
67
|
+
if !@storage_json.authorization.signing_key.nil?
|
68
|
+
signed_string = default_signer(string_to_sign)
|
69
|
+
else
|
70
|
+
# If client doesn't contain signing key attempt to auth via IAM SignBlob API
|
71
|
+
signed_string = iam_signer(string_to_sign)
|
72
|
+
end
|
62
73
|
|
63
74
|
Base64.encode64(signed_string).chomp!
|
64
75
|
end
|
76
|
+
|
77
|
+
private
|
78
|
+
|
79
|
+
def google_access_id
|
80
|
+
@google_access_id ||= get_google_access_id
|
81
|
+
end
|
82
|
+
|
83
|
+
##
|
84
|
+
# Fetches the google service account name
|
85
|
+
#
|
86
|
+
# @return [String] Service account name, typically needed for GoogleAccessId, e.g.
|
87
|
+
# my-account@project.iam.gserviceaccount
|
88
|
+
# @raises [Fog::Errors::Error] If authorisation is incorrect or inapplicable for current action
|
89
|
+
def get_google_access_id
|
90
|
+
if @storage_json.authorization.is_a?(::Google::Auth::UserRefreshCredentials)
|
91
|
+
raise Fog::Errors::Error.new("User / Application Default Credentials are not supported for storage"\
|
92
|
+
"url signing, please use a service account or metadata authentication.")
|
93
|
+
end
|
94
|
+
|
95
|
+
if !@storage_json.authorization.issuer.nil?
|
96
|
+
return @storage_json.authorization.issuer
|
97
|
+
else
|
98
|
+
get_access_id_from_metadata
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
##
|
103
|
+
# Attempts to fetch the google service account name from metadata using Google::Cloud::Env
|
104
|
+
#
|
105
|
+
# @return [String] Service account name, typically needed for GoogleAccessId, e.g.
|
106
|
+
# my-account@project.iam.gserviceaccount
|
107
|
+
# @raises [Fog::Errors::Error] If Metadata service is not available or returns an invalid response
|
108
|
+
def get_access_id_from_metadata
|
109
|
+
if @google_cloud_env.metadata?
|
110
|
+
access_id = @google_cloud_env.lookup_metadata("instance", "service-accounts/default/email")
|
111
|
+
else
|
112
|
+
raise Fog::Errors::Error.new("Metadata service not available, unable to retrieve service account info.")
|
113
|
+
end
|
114
|
+
|
115
|
+
if access_id.nil?
|
116
|
+
raise Fog::Errors::Error.new("Metadata service found but didn't return data." \
|
117
|
+
"Please file a bug: https://github.com/fog/fog-google")
|
118
|
+
end
|
119
|
+
|
120
|
+
return access_id
|
121
|
+
end
|
122
|
+
|
123
|
+
##
|
124
|
+
# Default url signer using service account keys
|
125
|
+
#
|
126
|
+
# @param [String] string_to_sign Special collection of headers and options for V2 storage signing, e.g.:
|
127
|
+
#
|
128
|
+
# StringToSign = HTTP_Verb + "\n" +
|
129
|
+
# Content_MD5 + "\n" +
|
130
|
+
# Content_Type + "\n" +
|
131
|
+
# Expires + "\n" +
|
132
|
+
# Canonicalized_Extension_Headers +
|
133
|
+
# Canonicalized_Resource
|
134
|
+
#
|
135
|
+
# See https://cloud.google.com/storage/docs/access-control/signed-urls-v2
|
136
|
+
# @return [String] Signature binary blob
|
137
|
+
def default_signer(string_to_sign)
|
138
|
+
key = OpenSSL::PKey::RSA.new(@storage_json.authorization.signing_key)
|
139
|
+
digest = OpenSSL::Digest::SHA256.new
|
140
|
+
return key.sign(digest, string_to_sign)
|
141
|
+
end
|
142
|
+
|
143
|
+
##
|
144
|
+
# Fallback URL signer using the IAM SignServiceAccountBlob API, see
|
145
|
+
# Google::Apis::IamcredentialsV1::IAMCredentialsService#sign_service_account_blob
|
146
|
+
#
|
147
|
+
# @param [String] string_to_sign Special collection of headers and options for V2 storage signing, e.g.:
|
148
|
+
#
|
149
|
+
# StringToSign = HTTP_Verb + "\n" +
|
150
|
+
# Content_MD5 + "\n" +
|
151
|
+
# Content_Type + "\n" +
|
152
|
+
# Expires + "\n" +
|
153
|
+
# Canonicalized_Extension_Headers +
|
154
|
+
# Canonicalized_Resource
|
155
|
+
#
|
156
|
+
# See https://cloud.google.com/storage/docs/access-control/signed-urls-v2
|
157
|
+
# @return [String] Signature binary blob
|
158
|
+
def iam_signer(string_to_sign)
|
159
|
+
request = {
|
160
|
+
"payload": string_to_sign
|
161
|
+
}
|
162
|
+
|
163
|
+
resource = "projects/-/serviceAccounts/#{google_access_id}"
|
164
|
+
response = @iam_service.sign_service_account_blob resource, request, {}
|
165
|
+
|
166
|
+
return response.signed_blob
|
167
|
+
end
|
65
168
|
end
|
66
169
|
end
|
67
170
|
end
|
@@ -27,7 +27,7 @@ module Fog
|
|
27
27
|
query = filtered.map { |k, v| [k.to_s, Fog::Google.escape(v)].join("=") }
|
28
28
|
end
|
29
29
|
|
30
|
-
query << "GoogleAccessId=#{
|
30
|
+
query << "GoogleAccessId=#{google_access_id}"
|
31
31
|
query << "Signature=#{CGI.escape(signature(params))}"
|
32
32
|
query << "Expires=#{params[:headers]['Date']}"
|
33
33
|
"#{params[:host]}/#{params[:path]}?#{query.join('&')}"
|
@@ -33,12 +33,13 @@ module Fog
|
|
33
33
|
headers["If-Modified-Since"] = Fog::Time.at(options["If-Modified-Since"].to_i).to_date_header if options["If-Modified-Since"]
|
34
34
|
headers["If-Unmodified-Since"] = Fog::Time.at(options["If-Unmodified-Since"].to_i).to_date_header if options["If-Modified-Since"]
|
35
35
|
headers.merge!(options)
|
36
|
-
request(:expects
|
37
|
-
:headers
|
38
|
-
:host
|
39
|
-
:
|
40
|
-
:
|
41
|
-
:
|
36
|
+
request(:expects => [200, 206],
|
37
|
+
:headers => headers,
|
38
|
+
:host => "#{bucket_name}.#{@host}",
|
39
|
+
:idempotent => true,
|
40
|
+
:method => "HEAD",
|
41
|
+
:path => Fog::Google.escape(object_name),
|
42
|
+
:query => query)
|
42
43
|
end
|
43
44
|
end
|
44
45
|
|
data/tasks/changelog.rake
CHANGED
@@ -20,11 +20,18 @@ end
|
|
20
20
|
|
21
21
|
# Extracts all changes that have been made after the latest pushed tag
|
22
22
|
def changes_since_last_tag
|
23
|
-
`git --no-pager log $(git describe --tags --abbrev=0)..HEAD --
|
23
|
+
`git --no-pager log $(git describe --tags --abbrev=0)..HEAD --grep="Merge" --pretty=format:"%t - %s%n%b%n"`
|
24
|
+
end
|
25
|
+
|
26
|
+
# Extracts all github users contributed since last tag
|
27
|
+
def users_since_last_tag
|
28
|
+
`git --no-pager log $(git describe --tags --abbrev=0)..HEAD --grep="Merge" --pretty=format:"%s" | cut -d' ' -f 6 | cut -d/ -f1 | uniq`
|
24
29
|
end
|
25
30
|
|
26
31
|
namespace :changelog do
|
27
32
|
task :generate do
|
28
33
|
insert_after_line("CHANGELOG.md", changes_since_last_tag, /^## Next/)
|
34
|
+
printf("Users contributed since last release:\n")
|
35
|
+
printf(users_since_last_tag)
|
29
36
|
end
|
30
37
|
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require "helpers/test_helper"
|
2
2
|
require "helpers/test_collection"
|
3
|
+
require "retriable"
|
3
4
|
|
4
5
|
# Use :test credentials in ~/.fog for live integration testing
|
5
6
|
# XXX not sure if this will work on Travis CI or not
|
@@ -22,7 +23,7 @@ TEST_IMAGE_FAMILY = "debian-9".freeze
|
|
22
23
|
|
23
24
|
# XXX This depends on a public image in gs://fog-test-bucket; there may be a better way to do this
|
24
25
|
# The image was created like so: https://cloud.google.com/compute/docs/images#export_an_image_to_google_cloud_storage
|
25
|
-
TEST_RAW_DISK_SOURCE = "
|
26
|
+
TEST_RAW_DISK_SOURCE = "https://storage.googleapis.com/fog-testing-bucket/fog-test-raw-disk-source.image.tar.gz".freeze
|
26
27
|
|
27
28
|
TEST_SQL_TIER = "db-n1-standard-1".freeze
|
28
29
|
TEST_SQL_REGION = TEST_REGION
|
@@ -87,8 +88,21 @@ wLjafhPTSAIS0jijglJ7uzaSbFUW11fw1/EIqIFNe0R0Xf9lsyPxFA==
|
|
87
88
|
-----END RSA PRIVATE KEY-----
|
88
89
|
KEY
|
89
90
|
|
91
|
+
# Retry module config - standard retriable gem settings
|
92
|
+
# see: https://github.com/kamui/retriable
|
93
|
+
RETRIABLE_TRIES = 3
|
94
|
+
RETRIABLE_BASE_INTERVAL = 50
|
95
|
+
|
90
96
|
class FogIntegrationTest < MiniTest::Test
|
91
97
|
def namespaced_name
|
92
98
|
"#{self.class}_#{name}"
|
93
99
|
end
|
100
|
+
|
101
|
+
def retry_on(klass)
|
102
|
+
Retriable.retriable(on: klass,
|
103
|
+
tries: RETRIABLE_TRIES,
|
104
|
+
base_interval: RETRIABLE_BASE_INTERVAL) do
|
105
|
+
yield
|
106
|
+
end
|
107
|
+
end
|
94
108
|
end
|
@@ -1,12 +1,11 @@
|
|
1
1
|
require "helpers/integration_test_helper"
|
2
|
-
require "retriable"
|
3
2
|
|
4
3
|
class TestMetricDescriptors < FogIntegrationTest
|
5
|
-
#
|
6
|
-
# metric propagation (sometimes 80+ seconds) and client
|
7
|
-
# Google::Apis::ClientError: badRequest if the metric
|
8
|
-
|
9
|
-
|
4
|
+
# This module has a decent amount of retry wrappers as the Stackdriver API
|
5
|
+
# can be quite slow with metric propagation (sometimes 80+ seconds) and client
|
6
|
+
# returning errors, e.g. Google::Apis::ClientError: badRequest if the metric
|
7
|
+
# hasn't yet been created instead of a 404.
|
8
|
+
|
10
9
|
TEST_METRIC_PREFIX = "custom.googleapis.com/fog-google-test/timeseries".freeze
|
11
10
|
LABEL_DESCRIPTORS = [
|
12
11
|
{
|
@@ -51,10 +50,7 @@ class TestMetricDescriptors < FogIntegrationTest
|
|
51
50
|
assert_empty(resp.to_h)
|
52
51
|
|
53
52
|
# Wait for metric to be created
|
54
|
-
|
55
|
-
Retriable.retriable(on: Google::Apis::ClientError,
|
56
|
-
tries: RETRIABLE_TRIES,
|
57
|
-
base_interval: RETRIABLE_BASE_INTERVAL) do
|
53
|
+
retry_on(Google::Apis::ClientError) do
|
58
54
|
@client.list_timeseries(
|
59
55
|
:filter => "metric.type = \"#{metric_type}\"",
|
60
56
|
:interval => {
|
@@ -67,9 +63,7 @@ class TestMetricDescriptors < FogIntegrationTest
|
|
67
63
|
).time_series
|
68
64
|
end
|
69
65
|
|
70
|
-
series =
|
71
|
-
tries: RETRIABLE_TRIES,
|
72
|
-
base_interval: RETRIABLE_BASE_INTERVAL) do
|
66
|
+
series = retry_on(Google::Apis::ClientError) do
|
73
67
|
@client.timeseries_collection.all(
|
74
68
|
:filter => "metric.type = \"#{metric_type}\"",
|
75
69
|
:interval => {
|
@@ -115,9 +109,7 @@ class TestMetricDescriptors < FogIntegrationTest
|
|
115
109
|
_some_timeseries(start_time, metric_type, labels)
|
116
110
|
end
|
117
111
|
|
118
|
-
|
119
|
-
tries: RETRIABLE_TRIES,
|
120
|
-
base_interval: RETRIABLE_BASE_INTERVAL) do
|
112
|
+
retry_on(Google::Apis::ServerError) do
|
121
113
|
@client.create_timeseries(:timeseries => timeseries)
|
122
114
|
end
|
123
115
|
interval = {
|
@@ -132,9 +124,7 @@ class TestMetricDescriptors < FogIntegrationTest
|
|
132
124
|
# Wait for metric to be created
|
133
125
|
# Retriable is used instead of wait_for due to API client returning Google::Apis::ClientError: badRequest if the
|
134
126
|
# metric hasn't yet been created
|
135
|
-
|
136
|
-
tries: RETRIABLE_TRIES,
|
137
|
-
base_interval: RETRIABLE_BASE_INTERVAL) do
|
127
|
+
retry_on(Google::Apis::ClientError) do
|
138
128
|
@client.list_timeseries(
|
139
129
|
:filter => "metric.type = \"#{metric_type}\"",
|
140
130
|
:interval => interval
|
@@ -142,9 +132,7 @@ class TestMetricDescriptors < FogIntegrationTest
|
|
142
132
|
end
|
143
133
|
|
144
134
|
# Test page size
|
145
|
-
resp =
|
146
|
-
tries: RETRIABLE_TRIES,
|
147
|
-
base_interval: RETRIABLE_BASE_INTERVAL) do
|
135
|
+
resp = retry_on(Google::Apis::ClientError) do
|
148
136
|
@client.list_timeseries(
|
149
137
|
:filter => "metric.type = \"#{metric_type}\"",
|
150
138
|
:interval => interval,
|
@@ -170,9 +158,7 @@ class TestMetricDescriptors < FogIntegrationTest
|
|
170
158
|
"expected different timeseries when using page_token")
|
171
159
|
|
172
160
|
# Test filter
|
173
|
-
series =
|
174
|
-
tries: RETRIABLE_TRIES,
|
175
|
-
base_interval: RETRIABLE_BASE_INTERVAL) do
|
161
|
+
series = retry_on(Google::Apis::ClientError) do
|
176
162
|
@client.timeseries_collection.all(
|
177
163
|
:filter => %[
|
178
164
|
metric.type = "#{metric_type}" AND
|
@@ -102,7 +102,7 @@ class TestPubsubModels < PubSubShared
|
|
102
102
|
:topic => some_topic_name)
|
103
103
|
@client.topics.get(some_topic_name).publish(["data" => message_bytes])
|
104
104
|
|
105
|
-
result = subscription.pull
|
105
|
+
result = subscription.pull(:return_immediately => false)
|
106
106
|
assert_operator(result.length, :>, 0)
|
107
107
|
|
108
108
|
contained = result.any? { |received| received.message[:data] == message_bytes }
|
@@ -115,7 +115,7 @@ class TestPubsubModels < PubSubShared
|
|
115
115
|
:topic => some_topic_name)
|
116
116
|
@client.topics.get(some_topic_name).publish(["data" => Base64.strict_encode64("some message")])
|
117
117
|
|
118
|
-
result = subscription.pull
|
118
|
+
result = subscription.pull(:return_immediately => false)
|
119
119
|
assert_operator(result.length, :>, 0)
|
120
120
|
|
121
121
|
subscription.acknowledge([result[0].ack_id])
|
@@ -127,7 +127,7 @@ class TestPubsubModels < PubSubShared
|
|
127
127
|
:topic => some_topic_name)
|
128
128
|
@client.topics.get(some_topic_name).publish(["data" => Base64.strict_encode64("some message")])
|
129
129
|
|
130
|
-
result = subscription.pull
|
130
|
+
result = subscription.pull(:return_immediately => false)
|
131
131
|
assert_operator(result.length, :>, 0)
|
132
132
|
|
133
133
|
result[0].acknowledge
|
@@ -86,7 +86,7 @@ class TestPubsubRequests < PubSubShared
|
|
86
86
|
@client.create_subscription(subscription_name, some_topic_name)
|
87
87
|
@client.publish_topic(some_topic_name, [:data => message_bytes])
|
88
88
|
|
89
|
-
result = @client.pull_subscription(subscription_name)
|
89
|
+
result = @client.pull_subscription(subscription_name, {:return_immediately => false})
|
90
90
|
|
91
91
|
contained = result.received_messages.any? { |received| received.message.data == message_bytes }
|
92
92
|
assert_equal(true, contained, "sent messsage not contained within pulled responses")
|
@@ -96,7 +96,7 @@ class TestPubsubRequests < PubSubShared
|
|
96
96
|
subscription_name = new_subscription_name
|
97
97
|
@client.create_subscription(subscription_name, some_topic_name)
|
98
98
|
@client.publish_topic(some_topic_name, [:data => Base64.strict_encode64("some message")])
|
99
|
-
pull_result = @client.pull_subscription(subscription_name)
|
99
|
+
pull_result = @client.pull_subscription(subscription_name, {:return_immediately => false})
|
100
100
|
assert_operator(pull_result.received_messages.length, :>, 0)
|
101
101
|
|
102
102
|
@client.acknowledge_subscription(subscription_name,
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fog-google
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.11.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nat Welch
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2020-
|
12
|
+
date: 2020-08-13 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fog-core
|
@@ -73,6 +73,20 @@ dependencies:
|
|
73
73
|
- - "<"
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0.34'
|
76
|
+
- !ruby/object:Gem::Dependency
|
77
|
+
name: google-cloud-env
|
78
|
+
requirement: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '1.2'
|
83
|
+
type: :runtime
|
84
|
+
prerelease: false
|
85
|
+
version_requirements: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '1.2'
|
76
90
|
- !ruby/object:Gem::Dependency
|
77
91
|
name: pry
|
78
92
|
requirement: !ruby/object:Gem::Requirement
|