fluent-plugin-google-cloud 0.4.5 → 0.4.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 66d74a0b5f7bb8a69f0b229fab8db399efe33a5e
4
- data.tar.gz: fa1f657fbbe617ab9ef5159c1577caba6f09b44f
3
+ metadata.gz: ef4989cf52201417aaa16cd77abc9680b55ddab7
4
+ data.tar.gz: 4c313c6fbe55b6c17c2eaec8b5c756c947135cab
5
5
  SHA512:
6
- metadata.gz: 0091443b2d9a27ad9cde62f200d0c11d2ded76633ac126649680a5eb1b54bc914b19bc8148ec6fb3af083e4889c652e0f7f56289dff09cec4eb77ba56d22eaa2
7
- data.tar.gz: c35ee5aa129a3420e83e17d753573c1c6a64fce215d6a783a692efe26f33c91516f8ad2cff2a6104827bf89e642df306bff7980671b513f6b80ebc9952fb8b60
6
+ metadata.gz: f50bd833b6b65548f439a5493f802fe99b266479d077a68abc4fec52e1f04b6223967344eabed170eb1a846525a5ab16e0dbc3b60df6638eaafa1f172c1c76e1
7
+ data.tar.gz: 4744b04e0a82268535afc934cf2c3a8af71b409fe9ca45f2b4f88f6777e1e80d72e65860ac7a48288c58809120dab1715cbd46287609a4832eea4f68d0287cd8
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- fluent-plugin-google-cloud (0.4.5)
4
+ fluent-plugin-google-cloud (0.4.6)
5
5
  fluentd (>= 0.10)
6
6
  google-api-client (~> 0.8.6)
7
7
  googleauth (~> 0.4)
@@ -121,6 +121,3 @@ DEPENDENCIES
121
121
  rubocop (~> 0.33.0)
122
122
  test-unit (~> 3.0.2)
123
123
  webmock (>= 1.17.0)
124
-
125
- BUNDLED WITH
126
- 1.10.6
@@ -10,7 +10,7 @@ eos
10
10
  gem.homepage = \
11
11
  'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
12
12
  gem.license = 'Apache 2.0'
13
- gem.version = '0.4.5'
13
+ gem.version = '0.4.6'
14
14
  gem.authors = ['Todd Derr', 'Alex Robinson']
15
15
  gem.email = ['salty@google.com']
16
16
 
@@ -20,6 +20,7 @@ module Fluent
20
20
  # Constants for service names.
21
21
  APPENGINE_SERVICE = 'appengine.googleapis.com'
22
22
  COMPUTE_SERVICE = 'compute.googleapis.com'
23
+ CONTAINER_SERVICE = 'container.googleapis.com'
23
24
  DATAFLOW_SERVICE = 'dataflow.googleapis.com'
24
25
  EC2_SERVICE = 'ec2.amazonaws.com'
25
26
 
@@ -42,7 +43,9 @@ module Fluent
42
43
  # DEPRECATED: Parameters necessary to use the private_key auth_method.
43
44
  config_param :private_key_email, :string, :default => nil
44
45
  config_param :private_key_path, :string, :default => nil
45
- config_param :private_key_passphrase, :string, :default => 'notasecret'
46
+ config_param :private_key_passphrase, :string,
47
+ :default => 'notasecret',
48
+ :secret => true
46
49
 
47
50
  # Specify project/instance metadata.
48
51
  #
@@ -103,6 +106,7 @@ module Fluent
103
106
  require 'json'
104
107
  require 'open-uri'
105
108
  require 'socket'
109
+ require 'yaml'
106
110
 
107
111
  # use the global logger
108
112
  @log = $log # rubocop:disable Style/GlobalVars
@@ -128,8 +132,10 @@ module Fluent
128
132
  end
129
133
  end
130
134
 
131
- # TODO: Send instance tags and/or hostname as labels as well?
135
+ # TODO: Send instance tags as labels as well?
132
136
  @common_labels = {}
137
+ @tag_to_kubernetes_labels_regexp =
138
+ /\.(?<pod_name>[^\._]+)_(?<namespace_name>[^_]+)_(?<container_name>.+)$/
133
139
 
134
140
  # set attributes from metadata (unless overriden by static config)
135
141
  @vm_name = Socket.gethostname if @vm_name.nil?
@@ -200,14 +206,18 @@ module Fluent
200
206
  @service_name = DATAFLOW_SERVICE
201
207
  @dataflow_job_id = fetch_gce_metadata('instance/attributes/job_id')
202
208
  common_labels["#{DATAFLOW_SERVICE}/job_id"] = @dataflow_job_id
209
+ elsif attributes.include?('kube-env')
210
+ # Kubernetes/Container Engine
211
+ @service_name = CONTAINER_SERVICE
212
+ common_labels["#{CONTAINER_SERVICE}/instance_id"] = @vm_id
213
+ @raw_kube_env = fetch_gce_metadata('instance/attributes/kube-env')
214
+ @kube_env = YAML.load(@raw_kube_env)
215
+ common_labels["#{CONTAINER_SERVICE}/cluster_name"] =
216
+ cluster_name_from_kube_env(@kube_env)
203
217
  end
204
- # include GCE labels unless we're running on dataflow, which
205
- # uses their own labels exclusively.
206
- if (@service_name != DATAFLOW_SERVICE)
207
- common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
208
- common_labels["#{COMPUTE_SERVICE}/resource_id"] = @vm_id
209
- common_labels["#{COMPUTE_SERVICE}/resource_name"] = @vm_name
210
- end
218
+ common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
219
+ common_labels["#{COMPUTE_SERVICE}/resource_id"] = @vm_id
220
+ common_labels["#{COMPUTE_SERVICE}/resource_name"] = @vm_name
211
221
  when Platform::EC2
212
222
  @service_name = EC2_SERVICE
213
223
  common_labels["#{EC2_SERVICE}/resource_type"] = 'instance'
@@ -252,6 +262,18 @@ module Fluent
252
262
  'commonLabels' => @common_labels,
253
263
  'entries' => []
254
264
  }
265
+ if @service_name == CONTAINER_SERVICE
266
+ # Container logs in Kubernetes are tagged based on where they came
267
+ # from, so we can extract useful metadata from the tag.
268
+ # Do this here to avoid having to repeat it for each record.
269
+ match_data = @tag_to_kubernetes_labels_regexp.match(tag)
270
+ if match_data
271
+ labels = write_log_entries_request['commonLabels']
272
+ %w(namespace_name pod_name container_name).each do |field|
273
+ labels["#{CONTAINER_SERVICE}/#{field}"] = match_data[field]
274
+ end
275
+ end
276
+ end
255
277
  arr.each do |time, record|
256
278
  next unless record.is_a?(Hash)
257
279
  if record.key?('timestamp') &&
@@ -292,7 +314,8 @@ module Fluent
292
314
  'timestamp' => {
293
315
  'seconds' => ts_secs,
294
316
  'nanos' => ts_nanos
295
- }
317
+ },
318
+ 'labels' => {}
296
319
  }
297
320
  }
298
321
  if record.key?('severity')
@@ -302,18 +325,20 @@ module Fluent
302
325
  entry['metadata']['severity'] = 'DEFAULT'
303
326
  end
304
327
 
328
+ # If the record has been annotated by the kubernetes_metadata_filter
329
+ # plugin, then use that metadata. Otherwise, rely on commonLabels
330
+ # populated at the grouped_entries level from the group's tag.
331
+ if @service_name == CONTAINER_SERVICE && record['kubernetes']
332
+ handle_container_metadata(record, entry)
333
+ end
334
+
305
335
  # If a field is present in the label_map, send its value as a label
306
336
  # (mapping the field name to label name as specified in the config)
307
337
  # and do not send that field as part of the payload.
308
- unless label_map.nil?
309
- labels = {}
338
+ if @label_map
310
339
  @label_map.each do |field, label|
311
- next unless record.key?(field)
312
- # label values are required to be strings.
313
- labels[label] = record[field].to_s
314
- record.delete(field)
340
+ field_to_label(record, field, entry['metadata']['labels'], label)
315
341
  end
316
- entry['metadata']['labels'] = labels unless labels.empty?
317
342
  end
318
343
 
319
344
  # use textPayload if the only remainaing key is 'message',
@@ -323,6 +348,12 @@ module Fluent
323
348
  else
324
349
  entry['structPayload'] = record
325
350
  end
351
+
352
+ # Remove the labels metadata if we didn't populate it with anything.
353
+ if entry['metadata']['labels'].empty?
354
+ entry['metadata'].delete('labels')
355
+ end
356
+
326
357
  write_log_entries_request['entries'].push(entry)
327
358
  end
328
359
  # Don't send an empty request if we rejected all the entries.
@@ -441,6 +472,15 @@ module Fluent
441
472
  end
442
473
  end
443
474
 
475
+ def cluster_name_from_kube_env(kube_env)
476
+ return kube_env['CLUSTER_NAME'] if kube_env.key?('CLUSTER_NAME')
477
+ instance_prefix = kube_env['INSTANCE_PREFIX']
478
+ gke_name_match = /^gke-(.+)-[0-9a-f]{8}$/.match(instance_prefix)
479
+ return gke_name_match.captures[0] if gke_name_match &&
480
+ !gke_name_match.captures.empty?
481
+ instance_prefix
482
+ end
483
+
444
484
  # Values permitted by the API for 'severity' (which is an enum).
445
485
  VALID_SEVERITIES = Set.new(
446
486
  %w(DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY))
@@ -500,10 +540,36 @@ module Fluent
500
540
  'DEFAULT'
501
541
  end
502
542
 
543
+ # Requires that record has a 'kubernetes' field.
544
+ def handle_container_metadata(record, entry)
545
+ fields = %w(namespace_id namespace_name pod_id pod_name container_name)
546
+ fields.each do |field|
547
+ field_to_label(record['kubernetes'], field, entry['metadata']['labels'],
548
+ "#{CONTAINER_SERVICE}/#{field}")
549
+ end
550
+ # Prepend label/ to all user-defined labels' keys.
551
+ if record.key?('labels')
552
+ record['kubernetes']['labels'].each do |key, value|
553
+ entry['metadata']['labels']["label/#{key}"] = value
554
+ end
555
+ end
556
+ # We've explicitly consumed all the fields we care about -- don't litter
557
+ # the log entries with the remaining fields that the kubernetes metadata
558
+ # filter plugin includes (or an empty 'kubernetes' field).
559
+ record.delete('kubernetes')
560
+ record.delete('docker')
561
+ end
562
+
563
+ def field_to_label(record, field, labels, label)
564
+ return unless record.key?(field)
565
+ labels[label] = record[field].to_s
566
+ record.delete(field)
567
+ end
568
+
503
569
  def init_api_client
504
570
  @client = Google::APIClient.new(
505
571
  application_name: 'Fluentd Google Cloud Logging plugin',
506
- application_version: '0.4.5',
572
+ application_version: '0.4.6',
507
573
  retries: 1)
508
574
 
509
575
  if @auth_method == 'private_key'
@@ -60,6 +60,14 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
60
60
  MANAGED_VM_BACKEND_NAME = 'default'
61
61
  MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
62
62
 
63
+ # Container Engine / Kubernetes specific labels
64
+ CONTAINER_CLUSTER_NAME = 'cluster-1'
65
+ CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'
66
+ CONTAINER_NAMESPACE_NAME = 'kube-system'
67
+ CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'
68
+ CONTAINER_POD_NAME = 'redis-master-c0l82'
69
+ CONTAINER_CONTAINER_NAME = 'redis'
70
+
63
71
  # Parameters used for authentication
64
72
  AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
65
73
  FAKE_AUTH_TOKEN = 'abc123'
@@ -122,6 +130,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
122
130
  # Service configurations for various services
123
131
  COMPUTE_SERVICE_NAME = 'compute.googleapis.com'
124
132
  APPENGINE_SERVICE_NAME = 'appengine.googleapis.com'
133
+ CONTAINER_SERVICE_NAME = 'container.googleapis.com'
125
134
  EC2_SERVICE_NAME = 'ec2.amazonaws.com'
126
135
 
127
136
  COMPUTE_PARAMS = {
@@ -150,6 +159,46 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
150
159
  }
151
160
  }
152
161
 
162
+ CONTAINER_LOG_NAME = "kubernetes.#{CONTAINER_POD_NAME}_" \
163
+ "#{CONTAINER_NAMESPACE_NAME}_#{CONTAINER_CONTAINER_NAME}"
164
+
165
+ CONTAINER_FROM_METADATA_PARAMS = {
166
+ 'service_name' => CONTAINER_SERVICE_NAME,
167
+ 'log_name' => CONTAINER_LOG_NAME,
168
+ 'project_id' => PROJECT_ID,
169
+ 'zone' => ZONE,
170
+ 'labels' => {
171
+ "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
172
+ "#{CONTAINER_SERVICE_NAME}/cluster_name" => CONTAINER_CLUSTER_NAME,
173
+ "#{CONTAINER_SERVICE_NAME}/namespace_name" => CONTAINER_NAMESPACE_NAME,
174
+ "#{CONTAINER_SERVICE_NAME}/namespace_id" => CONTAINER_NAMESPACE_ID,
175
+ "#{CONTAINER_SERVICE_NAME}/pod_name" => CONTAINER_POD_NAME,
176
+ "#{CONTAINER_SERVICE_NAME}/pod_id" => CONTAINER_POD_ID,
177
+ "#{CONTAINER_SERVICE_NAME}/container_name" => CONTAINER_CONTAINER_NAME,
178
+ "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
179
+ "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
180
+ "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
181
+ }
182
+ }
183
+
184
+ # Almost the same as from metadata, but missing namespace_id and pod_id.
185
+ CONTAINER_FROM_TAG_PARAMS = {
186
+ 'service_name' => CONTAINER_SERVICE_NAME,
187
+ 'log_name' => CONTAINER_LOG_NAME,
188
+ 'project_id' => PROJECT_ID,
189
+ 'zone' => ZONE,
190
+ 'labels' => {
191
+ "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
192
+ "#{CONTAINER_SERVICE_NAME}/cluster_name" => CONTAINER_CLUSTER_NAME,
193
+ "#{CONTAINER_SERVICE_NAME}/namespace_name" => CONTAINER_NAMESPACE_NAME,
194
+ "#{CONTAINER_SERVICE_NAME}/pod_name" => CONTAINER_POD_NAME,
195
+ "#{CONTAINER_SERVICE_NAME}/container_name" => CONTAINER_CONTAINER_NAME,
196
+ "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
197
+ "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
198
+ "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
199
+ }
200
+ }
201
+
153
202
  CUSTOM_PARAMS = {
154
203
  'service_name' => COMPUTE_SERVICE_NAME,
155
204
  'log_name' => 'test',
@@ -175,9 +224,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
175
224
  }
176
225
  }
177
226
 
178
- def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
227
+ def create_driver(conf = APPLICATION_DEFAULT_CONFIG, tag = 'test')
179
228
  Fluent::Test::BufferedOutputTestDriver.new(
180
- Fluent::GoogleCloudOutput).configure(conf, use_v1_config: true)
229
+ Fluent::GoogleCloudOutput, tag).configure(conf, use_v1_config: true)
181
230
  end
182
231
 
183
232
  def test_configure_service_account_application_default
@@ -724,6 +773,58 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
724
773
  end
725
774
  end
726
775
 
776
+ def test_one_container_log_metadata_from_plugin
777
+ setup_gce_metadata_stubs
778
+ setup_container_metadata_stubs
779
+ setup_logging_stubs
780
+ d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
781
+ d.emit(container_log_entry_with_metadata(0))
782
+ d.run
783
+ verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS)
784
+ end
785
+
786
+ def test_multiple_container_logs_metadata_from_plugin
787
+ setup_gce_metadata_stubs
788
+ setup_container_metadata_stubs
789
+ setup_logging_stubs
790
+ d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
791
+ [2, 3, 5, 11, 50].each do |n|
792
+ # The test driver doesn't clear its buffer of entries after running, so
793
+ # do it manually here.
794
+ d.instance_variable_get('@entries').clear
795
+ @logs_sent = []
796
+ n.times { |i| d.emit(container_log_entry_with_metadata(i)) }
797
+ d.run
798
+ verify_log_entries(n, CONTAINER_FROM_METADATA_PARAMS)
799
+ end
800
+ end
801
+
802
+ def test_one_container_log_metadata_from_tag
803
+ setup_gce_metadata_stubs
804
+ setup_container_metadata_stubs
805
+ setup_logging_stubs
806
+ d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
807
+ d.emit('message' => log_entry(0))
808
+ d.run
809
+ verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS)
810
+ end
811
+
812
+ def test_multiple_container_logs_metadata_from_tag
813
+ setup_gce_metadata_stubs
814
+ setup_container_metadata_stubs
815
+ setup_logging_stubs
816
+ d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
817
+ [2, 3, 5, 11, 50].each do |n|
818
+ # The test driver doesn't clear its buffer of entries after running, so
819
+ # do it manually here.
820
+ d.instance_variable_get('@entries').clear
821
+ @logs_sent = []
822
+ n.times { |i| d.emit('message' => log_entry(i)) }
823
+ d.run
824
+ verify_log_entries(n, CONTAINER_FROM_TAG_PARAMS)
825
+ end
826
+ end
827
+
727
828
  # Make parse_severity public so we can test it.
728
829
  class Fluent::GoogleCloudOutput # rubocop:disable Style/ClassAndModuleChildren
729
830
  public :parse_severity
@@ -856,14 +957,13 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
856
957
  end
857
958
 
858
959
  def setup_logging_stubs
859
- [COMPUTE_PARAMS, VMENGINE_PARAMS, CUSTOM_PARAMS, EC2_PARAMS]
860
- .each do |params|
861
- stub_request(:post, uri_for_log(params))
862
- .to_return do |request|
863
- @logs_sent << JSON.parse(request.body)
864
- { body: '' }
865
- end
960
+ [COMPUTE_PARAMS, VMENGINE_PARAMS, CONTAINER_FROM_TAG_PARAMS,
961
+ CONTAINER_FROM_METADATA_PARAMS, CUSTOM_PARAMS, EC2_PARAMS].each do |params|
962
+ stub_request(:post, uri_for_log(params)).to_return do |request|
963
+ @logs_sent << JSON.parse(request.body)
964
+ { body: '' }
866
965
  end
966
+ end
867
967
  end
868
968
 
869
969
  def setup_auth_stubs
@@ -900,6 +1000,29 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
900
1000
  'guestbook2.0')
901
1001
  end
902
1002
 
1003
+ def setup_container_metadata_stubs
1004
+ stub_metadata_request(
1005
+ 'instance/attributes/',
1006
+ "attribute1\nkube-env\nlast_attribute")
1007
+ stub_metadata_request('instance/attributes/kube-env',
1008
+ "ENABLE_NODE_LOGGING: \"true\"\n"\
1009
+ "INSTANCE_PREFIX: gke-cluster-1-740fdafa\n"\
1010
+ 'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
1011
+ end
1012
+
1013
+ def container_log_entry_with_metadata(i)
1014
+ {
1015
+ message: log_entry(i),
1016
+ kubernetes: {
1017
+ namespace_id: CONTAINER_NAMESPACE_ID,
1018
+ namespace_name: CONTAINER_NAMESPACE_NAME,
1019
+ pod_id: CONTAINER_POD_ID,
1020
+ pod_name: CONTAINER_POD_NAME,
1021
+ container_name: CONTAINER_CONTAINER_NAME
1022
+ }
1023
+ }
1024
+ end
1025
+
903
1026
  def log_entry(i)
904
1027
  'test log entry ' + i.to_s
905
1028
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-google-cloud
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.5
4
+ version: 0.4.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Todd Derr
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2015-08-17 00:00:00.000000000 Z
12
+ date: 2015-09-11 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
@@ -181,7 +181,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
181
181
  version: '0'
182
182
  requirements: []
183
183
  rubyforge_project:
184
- rubygems_version: 2.4.6
184
+ rubygems_version: 2.4.3
185
185
  signing_key:
186
186
  specification_version: 4
187
187
  summary: fluentd output plugin for the Google Cloud Logging API