fluent-plugin-google-cloud 0.4.10 → 0.4.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +17 -15
- data/fluent-plugin-google-cloud.gemspec +8 -8
- data/lib/fluent/plugin/out_google_cloud.rb +46 -4
- data/test/plugin/test_out_google_cloud.rb +20 -5
- metadata +28 -22
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d5af03ce7a0e466c5bf132e69b7720f11edc41c7
|
4
|
+
data.tar.gz: c6726cb7a73c3cdae27e24ec78b444d82123790f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cbf1f33d49a88fcba407d75e564120c32bed24353fb88c4c383314e3f17d0437995c5abf05bc55feff9f18fff4ffa6bcfd97018a88c45b552a251b0a0c1720bf
|
7
|
+
data.tar.gz: c9b2d17c4dba28ee407a7beef3d3a1081379da96a4e108e495bc5db0390bee4bde7bbed45321c8e614841c5664cf2cbc4f7d52f5733c9a0ab545a5527bc30824
|
data/Gemfile.lock
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-google-cloud (0.4.
|
5
|
-
fluentd (
|
6
|
-
google-api-client (
|
4
|
+
fluent-plugin-google-cloud (0.4.11)
|
5
|
+
fluentd (~> 0.10)
|
6
|
+
google-api-client (>= 0.8.6, <= 0.9)
|
7
7
|
googleauth (~> 0.4)
|
8
|
-
json (~> 1.8
|
8
|
+
json (~> 1.8)
|
9
9
|
|
10
10
|
GEM
|
11
11
|
remote: https://rubygems.org/
|
@@ -58,10 +58,11 @@ GEM
|
|
58
58
|
memoist (~> 0.12)
|
59
59
|
multi_json (~> 1.11)
|
60
60
|
signet (~> 0.6)
|
61
|
+
hashdiff (0.2.2)
|
61
62
|
http_parser.rb (0.6.0)
|
62
63
|
i18n (0.7.0)
|
63
64
|
json (1.8.3)
|
64
|
-
jwt (1.5.
|
65
|
+
jwt (1.5.2)
|
65
66
|
launchy (2.4.3)
|
66
67
|
addressable (~> 2.3)
|
67
68
|
little-plugger (1.1.4)
|
@@ -70,20 +71,20 @@ GEM
|
|
70
71
|
multi_json (~> 1.10)
|
71
72
|
memoist (0.12.0)
|
72
73
|
metaclass (0.0.4)
|
73
|
-
minitest (5.8.
|
74
|
+
minitest (5.8.2)
|
74
75
|
mocha (1.1.0)
|
75
76
|
metaclass (~> 0.0.1)
|
76
77
|
msgpack (0.5.12)
|
77
78
|
multi_json (1.11.2)
|
78
79
|
multipart-post (2.0.0)
|
79
|
-
parser (2.2.
|
80
|
+
parser (2.2.3.0)
|
80
81
|
ast (>= 1.1, < 3.0)
|
81
|
-
power_assert (0.2.
|
82
|
+
power_assert (0.2.5)
|
82
83
|
powerpack (0.1.1)
|
83
84
|
rainbow (2.0.0)
|
84
85
|
rake (10.4.2)
|
85
86
|
retriable (1.4.1)
|
86
|
-
rubocop (0.
|
87
|
+
rubocop (0.34.2)
|
87
88
|
astrolabe (~> 1.3)
|
88
89
|
parser (>= 2.2.2.5, < 3.0)
|
89
90
|
powerpack (~> 0.1)
|
@@ -99,16 +100,17 @@ GEM
|
|
99
100
|
jwt (~> 1.5)
|
100
101
|
multi_json (~> 1.10)
|
101
102
|
string-scrub (0.0.5)
|
102
|
-
test-unit (3.
|
103
|
+
test-unit (3.1.5)
|
103
104
|
power_assert
|
104
105
|
thread_safe (0.3.5)
|
105
106
|
tzinfo (1.2.2)
|
106
107
|
thread_safe (~> 0.1)
|
107
108
|
tzinfo-data (1.2015.7)
|
108
109
|
tzinfo (>= 1.0.0)
|
109
|
-
webmock (1.
|
110
|
+
webmock (1.22.3)
|
110
111
|
addressable (>= 2.3.6)
|
111
112
|
crack (>= 0.3.2)
|
113
|
+
hashdiff
|
112
114
|
yajl-ruby (1.2.1)
|
113
115
|
|
114
116
|
PLATFORMS
|
@@ -117,10 +119,10 @@ PLATFORMS
|
|
117
119
|
DEPENDENCIES
|
118
120
|
fluent-plugin-google-cloud!
|
119
121
|
mocha (~> 1.1)
|
120
|
-
rake (
|
121
|
-
rubocop (
|
122
|
-
test-unit (~> 3.0
|
123
|
-
webmock (
|
122
|
+
rake (~> 10.3)
|
123
|
+
rubocop (= 0.34.2)
|
124
|
+
test-unit (~> 3.0)
|
125
|
+
webmock (~> 1.17)
|
124
126
|
|
125
127
|
BUNDLED WITH
|
126
128
|
1.10.6
|
@@ -10,7 +10,7 @@ eos
|
|
10
10
|
gem.homepage = \
|
11
11
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
12
12
|
gem.license = 'Apache 2.0'
|
13
|
-
gem.version = '0.4.
|
13
|
+
gem.version = '0.4.11'
|
14
14
|
gem.authors = ['Todd Derr', 'Alex Robinson']
|
15
15
|
gem.email = ['salty@google.com']
|
16
16
|
|
@@ -18,14 +18,14 @@ eos
|
|
18
18
|
gem.test_files = gem.files.grep(/^(test)/)
|
19
19
|
gem.require_paths = ['lib']
|
20
20
|
|
21
|
-
gem.add_runtime_dependency 'fluentd', '
|
22
|
-
gem.add_runtime_dependency 'google-api-client', '
|
21
|
+
gem.add_runtime_dependency 'fluentd', '~> 0.10'
|
22
|
+
gem.add_runtime_dependency 'google-api-client', '>= 0.8.6', '<= 0.9'
|
23
23
|
gem.add_runtime_dependency 'googleauth', '~> 0.4'
|
24
|
-
gem.add_runtime_dependency 'json', '~> 1.8
|
24
|
+
gem.add_runtime_dependency 'json', '~> 1.8'
|
25
25
|
|
26
26
|
gem.add_development_dependency 'mocha', '~> 1.1'
|
27
|
-
gem.add_development_dependency 'rake', '
|
28
|
-
gem.add_development_dependency 'rubocop', '
|
29
|
-
gem.add_development_dependency 'webmock', '
|
30
|
-
gem.add_development_dependency 'test-unit', '~> 3.0
|
27
|
+
gem.add_development_dependency 'rake', '~> 10.3'
|
28
|
+
gem.add_development_dependency 'rubocop', '= 0.34.2'
|
29
|
+
gem.add_development_dependency 'webmock', '~> 1.17'
|
30
|
+
gem.add_development_dependency 'test-unit', '~> 3.0'
|
31
31
|
end
|
@@ -195,6 +195,14 @@ module Fluent
|
|
195
195
|
fail Fluent::ConfigError, 'Unknown platform ' + @platform
|
196
196
|
end
|
197
197
|
|
198
|
+
# If we still don't have a project ID, try to obtain it from the
|
199
|
+
# credentials.
|
200
|
+
if @project_id.nil?
|
201
|
+
@project_id = CredentialsInfo.project_id
|
202
|
+
@log.info 'Set Project ID from credentials: ', @project_id unless
|
203
|
+
@project_id.nil?
|
204
|
+
end
|
205
|
+
|
198
206
|
# all metadata parameters must now be set
|
199
207
|
unless @project_id && @zone && @vm_id
|
200
208
|
missing = []
|
@@ -264,6 +272,11 @@ module Fluent
|
|
264
272
|
common_labels["#{COMPUTE_SERVICE}/resource_id"] = @vm_id
|
265
273
|
common_labels["#{COMPUTE_SERVICE}/resource_name"] = @vm_name
|
266
274
|
end
|
275
|
+
|
276
|
+
# Log an informational message containing the Logs viewer URL
|
277
|
+
@log.info 'Logs viewer address: ',
|
278
|
+
'https://console.developers.google.com/project/', @project_id,
|
279
|
+
'/logs?service=', @service_name, '&key1=instance&key2=', @vm_id
|
267
280
|
end
|
268
281
|
|
269
282
|
def start
|
@@ -550,9 +563,7 @@ module Fluent
|
|
550
563
|
@platform == Platform::GCE
|
551
564
|
# See https://cloud.google.com/compute/docs/metadata
|
552
565
|
open('http://' + METADATA_SERVICE_ADDR + '/computeMetadata/v1/' +
|
553
|
-
metadata_path, 'Metadata-Flavor' => 'Google')
|
554
|
-
f.read
|
555
|
-
end
|
566
|
+
metadata_path, 'Metadata-Flavor' => 'Google', &:read)
|
556
567
|
end
|
557
568
|
|
558
569
|
def fetch_ec2_metadata
|
@@ -566,6 +577,37 @@ module Fluent
|
|
566
577
|
end
|
567
578
|
end
|
568
579
|
|
580
|
+
# TODO: This functionality should eventually be available in another
|
581
|
+
# library, but implement it ourselves for now.
|
582
|
+
module CredentialsInfo
|
583
|
+
# Determine the project ID from the credentials, if possible.
|
584
|
+
# Returns the project ID (as a string) on success, or nil on failure.
|
585
|
+
def self.project_id
|
586
|
+
return nil if @auth_method == 'private_key'
|
587
|
+
creds = Google::Auth.get_application_default(LOGGING_SCOPE)
|
588
|
+
if creds.issuer
|
589
|
+
id = extract_project_id(creds.issuer)
|
590
|
+
return id unless id.nil?
|
591
|
+
end
|
592
|
+
if creds.client_id
|
593
|
+
id = extract_project_id(creds.client_id)
|
594
|
+
return id unless id.nil?
|
595
|
+
end
|
596
|
+
nil
|
597
|
+
end
|
598
|
+
|
599
|
+
# Extracts the project id from str. Assumes the project ID is at the
|
600
|
+
# front of str, and consists of a string of digits terminated by a
|
601
|
+
# dash (-) which is not part of the project ID. Example:
|
602
|
+
# 270694816269-1l1r2hb813leuppurdeik0apglbs80sv.apps.googleusercontent.com
|
603
|
+
# Returns the project ID (as a string) on success, or nil on failure.
|
604
|
+
def self.extract_project_id(str)
|
605
|
+
@project_regexp = /^(?<project_id>\d+)-/
|
606
|
+
match_data = @project_regexp.match(str)
|
607
|
+
match_data ? match_data['project_id'] : nil
|
608
|
+
end
|
609
|
+
end
|
610
|
+
|
569
611
|
def detect_cloudfunctions(attributes)
|
570
612
|
return unless attributes.include?('gcf_region')
|
571
613
|
# Cloud Functions detected
|
@@ -729,7 +771,7 @@ module Fluent
|
|
729
771
|
def init_api_client
|
730
772
|
@client = Google::APIClient.new(
|
731
773
|
application_name: 'Fluentd Google Cloud Logging plugin',
|
732
|
-
application_version: '0.4.
|
774
|
+
application_version: '0.4.11',
|
733
775
|
retries: 1)
|
734
776
|
|
735
777
|
if @auth_method == 'private_key'
|
@@ -82,6 +82,14 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
82
82
|
AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
|
83
83
|
FAKE_AUTH_TOKEN = 'abc123'
|
84
84
|
|
85
|
+
# Paths to test credentials files
|
86
|
+
CREDENTIALS_FILE = 'test/plugin/data/credentials.json'
|
87
|
+
INVALID_CREDENTIALS_FILE = 'test/plugin/data/invalid_credentials.json'
|
88
|
+
|
89
|
+
# Project ID for the test credentials.
|
90
|
+
# This must match the prefix of 'client_id' in CREDENTIALS_FILE.
|
91
|
+
CREDENTIALS_PROJECT_ID = '847859579879'
|
92
|
+
|
85
93
|
# Configuration files for various test scenarios
|
86
94
|
APPLICATION_DEFAULT_CONFIG = %(
|
87
95
|
)
|
@@ -491,6 +499,14 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
491
499
|
assert_equal 1, exception_count
|
492
500
|
end
|
493
501
|
|
502
|
+
def test_ec2_metadata_project_id_from_credentials
|
503
|
+
setup_ec2_metadata_stubs
|
504
|
+
ENV['GOOGLE_APPLICATION_CREDENTIALS'] = CREDENTIALS_FILE
|
505
|
+
d = create_driver
|
506
|
+
d.run
|
507
|
+
assert_equal CREDENTIALS_PROJECT_ID, d.instance.project_id
|
508
|
+
end
|
509
|
+
|
494
510
|
def test_one_log
|
495
511
|
setup_gce_metadata_stubs
|
496
512
|
setup_logging_stubs
|
@@ -503,7 +519,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
503
519
|
def test_one_log_with_json_credentials
|
504
520
|
setup_gce_metadata_stubs
|
505
521
|
setup_logging_stubs
|
506
|
-
ENV['GOOGLE_APPLICATION_CREDENTIALS'] =
|
522
|
+
ENV['GOOGLE_APPLICATION_CREDENTIALS'] = CREDENTIALS_FILE
|
507
523
|
d = create_driver
|
508
524
|
d.emit('message' => log_entry(0))
|
509
525
|
d.run
|
@@ -513,8 +529,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
513
529
|
def test_one_log_with_invalid_json_credentials
|
514
530
|
setup_gce_metadata_stubs
|
515
531
|
setup_logging_stubs
|
516
|
-
ENV['GOOGLE_APPLICATION_CREDENTIALS'] =
|
517
|
-
'test/plugin/data/invalid_credentials.json'
|
532
|
+
ENV['GOOGLE_APPLICATION_CREDENTIALS'] = INVALID_CREDENTIALS_FILE
|
518
533
|
d = create_driver
|
519
534
|
d.emit('message' => log_entry(0))
|
520
535
|
exception_count = 0
|
@@ -540,7 +555,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
540
555
|
# don't set up any metadata stubs, so the test will fail if we try to
|
541
556
|
# fetch metadata (and explicitly check this as well).
|
542
557
|
Fluent::GoogleCloudOutput.any_instance.expects(:fetch_metadata).never
|
543
|
-
ENV['GOOGLE_APPLICATION_CREDENTIALS'] =
|
558
|
+
ENV['GOOGLE_APPLICATION_CREDENTIALS'] = CREDENTIALS_FILE
|
544
559
|
setup_logging_stubs
|
545
560
|
d = create_driver(NO_METADATA_SERVICE_CONFIG + CUSTOM_METADATA_CONFIG)
|
546
561
|
d.emit('message' => log_entry(0))
|
@@ -549,7 +564,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
549
564
|
end
|
550
565
|
|
551
566
|
def test_one_log_ec2
|
552
|
-
ENV['GOOGLE_APPLICATION_CREDENTIALS'] =
|
567
|
+
ENV['GOOGLE_APPLICATION_CREDENTIALS'] = CREDENTIALS_FILE
|
553
568
|
setup_ec2_metadata_stubs
|
554
569
|
setup_logging_stubs
|
555
570
|
d = create_driver(CONFIG_EC2_PROJECT_ID)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-google-cloud
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Todd Derr
|
@@ -9,36 +9,42 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2015-
|
12
|
+
date: 2015-11-09 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
16
16
|
requirement: !ruby/object:Gem::Requirement
|
17
17
|
requirements:
|
18
|
-
- - "
|
18
|
+
- - "~>"
|
19
19
|
- !ruby/object:Gem::Version
|
20
20
|
version: '0.10'
|
21
21
|
type: :runtime
|
22
22
|
prerelease: false
|
23
23
|
version_requirements: !ruby/object:Gem::Requirement
|
24
24
|
requirements:
|
25
|
-
- - "
|
25
|
+
- - "~>"
|
26
26
|
- !ruby/object:Gem::Version
|
27
27
|
version: '0.10'
|
28
28
|
- !ruby/object:Gem::Dependency
|
29
29
|
name: google-api-client
|
30
30
|
requirement: !ruby/object:Gem::Requirement
|
31
31
|
requirements:
|
32
|
-
- - "
|
32
|
+
- - ">="
|
33
33
|
- !ruby/object:Gem::Version
|
34
34
|
version: 0.8.6
|
35
|
+
- - "<="
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0.9'
|
35
38
|
type: :runtime
|
36
39
|
prerelease: false
|
37
40
|
version_requirements: !ruby/object:Gem::Requirement
|
38
41
|
requirements:
|
39
|
-
- - "
|
42
|
+
- - ">="
|
40
43
|
- !ruby/object:Gem::Version
|
41
44
|
version: 0.8.6
|
45
|
+
- - "<="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0.9'
|
42
48
|
- !ruby/object:Gem::Dependency
|
43
49
|
name: googleauth
|
44
50
|
requirement: !ruby/object:Gem::Requirement
|
@@ -59,14 +65,14 @@ dependencies:
|
|
59
65
|
requirements:
|
60
66
|
- - "~>"
|
61
67
|
- !ruby/object:Gem::Version
|
62
|
-
version: 1.8
|
68
|
+
version: '1.8'
|
63
69
|
type: :runtime
|
64
70
|
prerelease: false
|
65
71
|
version_requirements: !ruby/object:Gem::Requirement
|
66
72
|
requirements:
|
67
73
|
- - "~>"
|
68
74
|
- !ruby/object:Gem::Version
|
69
|
-
version: 1.8
|
75
|
+
version: '1.8'
|
70
76
|
- !ruby/object:Gem::Dependency
|
71
77
|
name: mocha
|
72
78
|
requirement: !ruby/object:Gem::Requirement
|
@@ -85,58 +91,58 @@ dependencies:
|
|
85
91
|
name: rake
|
86
92
|
requirement: !ruby/object:Gem::Requirement
|
87
93
|
requirements:
|
88
|
-
- - "
|
94
|
+
- - "~>"
|
89
95
|
- !ruby/object:Gem::Version
|
90
|
-
version: 10.3
|
96
|
+
version: '10.3'
|
91
97
|
type: :development
|
92
98
|
prerelease: false
|
93
99
|
version_requirements: !ruby/object:Gem::Requirement
|
94
100
|
requirements:
|
95
|
-
- - "
|
101
|
+
- - "~>"
|
96
102
|
- !ruby/object:Gem::Version
|
97
|
-
version: 10.3
|
103
|
+
version: '10.3'
|
98
104
|
- !ruby/object:Gem::Dependency
|
99
105
|
name: rubocop
|
100
106
|
requirement: !ruby/object:Gem::Requirement
|
101
107
|
requirements:
|
102
|
-
- -
|
108
|
+
- - '='
|
103
109
|
- !ruby/object:Gem::Version
|
104
|
-
version: 0.
|
110
|
+
version: 0.34.2
|
105
111
|
type: :development
|
106
112
|
prerelease: false
|
107
113
|
version_requirements: !ruby/object:Gem::Requirement
|
108
114
|
requirements:
|
109
|
-
- -
|
115
|
+
- - '='
|
110
116
|
- !ruby/object:Gem::Version
|
111
|
-
version: 0.
|
117
|
+
version: 0.34.2
|
112
118
|
- !ruby/object:Gem::Dependency
|
113
119
|
name: webmock
|
114
120
|
requirement: !ruby/object:Gem::Requirement
|
115
121
|
requirements:
|
116
|
-
- - "
|
122
|
+
- - "~>"
|
117
123
|
- !ruby/object:Gem::Version
|
118
|
-
version: 1.17
|
124
|
+
version: '1.17'
|
119
125
|
type: :development
|
120
126
|
prerelease: false
|
121
127
|
version_requirements: !ruby/object:Gem::Requirement
|
122
128
|
requirements:
|
123
|
-
- - "
|
129
|
+
- - "~>"
|
124
130
|
- !ruby/object:Gem::Version
|
125
|
-
version: 1.17
|
131
|
+
version: '1.17'
|
126
132
|
- !ruby/object:Gem::Dependency
|
127
133
|
name: test-unit
|
128
134
|
requirement: !ruby/object:Gem::Requirement
|
129
135
|
requirements:
|
130
136
|
- - "~>"
|
131
137
|
- !ruby/object:Gem::Version
|
132
|
-
version: 3.0
|
138
|
+
version: '3.0'
|
133
139
|
type: :development
|
134
140
|
prerelease: false
|
135
141
|
version_requirements: !ruby/object:Gem::Requirement
|
136
142
|
requirements:
|
137
143
|
- - "~>"
|
138
144
|
- !ruby/object:Gem::Version
|
139
|
-
version: 3.0
|
145
|
+
version: '3.0'
|
140
146
|
description: |2
|
141
147
|
Fluentd output plugin for the Google Cloud Logging API, which will make
|
142
148
|
logs viewable in the Developer Console's log viewer and can optionally
|