fluent-plugin-google-cloud 0.4.7 → 0.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 494526c8a4880080e9176ccd912a4692d773fb67
4
- data.tar.gz: 008470c8bc3aeab7f902bfd159e030b2dbfafee1
3
+ metadata.gz: e6d35bf9167d8e4847b5e54cc16e0f78dc41310d
4
+ data.tar.gz: 3481166160940b98b9433cbe80811dfb71f98434
5
5
  SHA512:
6
- metadata.gz: aaabfa2e76c33959ab2d68c21380d73ce1e315bef2556fdd99b9772fc9cdb33c0d7ea8632a3742f17a9641e44c5e45914eab1ba49ebf035cd02f324ab0984a81
7
- data.tar.gz: c86c292744824fd8163e08daaefbd034baaaf91ca1c422b68caed7ab3835578070a694c59dbc1cc5adb85c55cf87eadfefbca28cae1b801f8543bdf70426c70a
6
+ metadata.gz: 0db9e3e069795d1965bbe72c49e2c7d9c8efce96087b099045dfb2308d21c55697bed09c6e6758aaab37d7ae267d1300970203df78f78db5a753f756b40717b9
7
+ data.tar.gz: 6006bd9033b7d82aa7a720eb8c330f852487e03de9f2816e18795704df69cedfe362fe210741059fe83ca26bd5e61bf83a5499cb761bc154233e11f633cc1516
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- fluent-plugin-google-cloud (0.4.6)
4
+ fluent-plugin-google-cloud (0.4.7)
5
5
  fluentd (>= 0.10)
6
6
  google-api-client (~> 0.8.6)
7
7
  googleauth (~> 0.4)
@@ -10,7 +10,7 @@ eos
10
10
  gem.homepage = \
11
11
  'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
12
12
  gem.license = 'Apache 2.0'
13
- gem.version = '0.4.7'
13
+ gem.version = '0.4.8'
14
14
  gem.authors = ['Todd Derr', 'Alex Robinson']
15
15
  gem.email = ['salty@google.com']
16
16
 
@@ -19,6 +19,7 @@ module Fluent
19
19
 
20
20
  # Constants for service names.
21
21
  APPENGINE_SERVICE = 'appengine.googleapis.com'
22
+ CLOUDFUNCTIONS_SERVICE = 'cloudfunctions.googleapis.com'
22
23
  COMPUTE_SERVICE = 'compute.googleapis.com'
23
24
  CONTAINER_SERVICE = 'container.googleapis.com'
24
25
  DATAFLOW_SERVICE = 'dataflow.googleapis.com'
@@ -70,6 +71,12 @@ module Fluent
70
71
  # component (Docker, Kubelet, etc.) logs from container logs.
71
72
  config_param :detect_subservice, :bool, :default => true
72
73
 
74
+ # The regular expression to use on Kubernetes logs to extract some basic
75
+ # information about the log source. The regex must contain capture groups
76
+ # for pod_name, namespace_name, and container_name.
77
+ config_param :kubernetes_tag_regexp, :string, :default =>
78
+ '\.(?<pod_name>[^_]+)_(?<namespace_name>[^_]+)_(?<container_name>.+)$'
79
+
73
80
  # label_map (specified as a JSON object) is an unordered set of fluent
74
81
  # field names whose values are sent as labels rather than as part of the
75
82
  # struct payload.
@@ -142,8 +149,19 @@ module Fluent
142
149
 
143
150
  # TODO: Send instance tags as labels as well?
144
151
  @common_labels = {}
145
- @tag_to_kubernetes_labels_regexp =
146
- /\.(?<pod_name>[^\._]+)_(?<namespace_name>[^_]+)_(?<container_name>.+)$/
152
+
153
+ @compiled_kubernetes_tag_regexp = nil
154
+ if @kubernetes_tag_regexp
155
+ @compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp)
156
+ end
157
+
158
+ @cloudfunctions_tag_regexp =
159
+ /\.(?<function_name>.+)-[^-]+_default_worker$/
160
+ @cloudfunctions_log_regexp = /^
161
+ (?:\[(?<severity>.)\])?
162
+ \[(?<timestamp>.{24})\]
163
+ (?:\[(?<execution_id>[^\]]+)\])?
164
+ [ ](?<text>.*)$/x
147
165
 
148
166
  # set attributes from metadata (unless overriden by static config)
149
167
  @vm_name = Socket.gethostname if @vm_name.nil?
@@ -190,12 +208,16 @@ module Fluent
190
208
  # Default this to false; it is only overwritten if we detect Managed VM.
191
209
  @running_on_managed_vm = false
192
210
 
211
+ # Default this to false; it is only overwritten if we detect Cloud
212
+ # Functions.
213
+ @running_cloudfunctions = false
214
+
193
215
  # Set labels, etc. based on the config
194
216
  case @platform
195
217
  when Platform::GCE
196
218
  @service_name = COMPUTE_SERVICE
197
219
  if @detect_subservice
198
- # Check for specialized GCE environments (Managed VM or Dataflow).
220
+ # Check for specialized GCE environments.
199
221
  # TODO: Add config options for these to allow for running outside GCE?
200
222
  attributes = fetch_gce_metadata('instance/attributes/').split
201
223
  # Do nothing, just don't populate other service's labels.
@@ -224,6 +246,7 @@ module Fluent
224
246
  @kube_env = YAML.load(@raw_kube_env)
225
247
  common_labels["#{CONTAINER_SERVICE}/cluster_name"] =
226
248
  cluster_name_from_kube_env(@kube_env)
249
+ detect_cloudfunctions(attributes)
227
250
  end
228
251
  end
229
252
  common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
@@ -273,11 +296,11 @@ module Fluent
273
296
  'commonLabels' => @common_labels,
274
297
  'entries' => []
275
298
  }
276
- if @service_name == CONTAINER_SERVICE
299
+ if @service_name == CONTAINER_SERVICE && @compiled_kubernetes_tag_regexp
277
300
  # Container logs in Kubernetes are tagged based on where they came
278
301
  # from, so we can extract useful metadata from the tag.
279
302
  # Do this here to avoid having to repeat it for each record.
280
- match_data = @tag_to_kubernetes_labels_regexp.match(tag)
303
+ match_data = @compiled_kubernetes_tag_regexp.match(tag)
281
304
  if match_data
282
305
  labels = write_log_entries_request['commonLabels']
283
306
  %w(namespace_name pod_name container_name).each do |field|
@@ -285,8 +308,30 @@ module Fluent
285
308
  end
286
309
  end
287
310
  end
311
+ if @running_cloudfunctions
312
+ # If the current group of entries is coming from a Cloud Functions
313
+ # function, the function name can be extracted from the tag.
314
+ match_data = @cloudfunctions_tag_regexp.match(tag)
315
+ if match_data
316
+ # Service name is set to Cloud Functions only for logs actually
317
+ # coming from a function.
318
+ @service_name = CLOUDFUNCTIONS_SERVICE
319
+ labels = write_log_entries_request['commonLabels']
320
+ labels["#{CLOUDFUNCTIONS_SERVICE}/region"] = @gcf_region
321
+ labels["#{CLOUDFUNCTIONS_SERVICE}/function_name"] =
322
+ match_data['function_name']
323
+ else
324
+ # Other logs are considered as coming from the Container Engine
325
+ # service.
326
+ @service_name = CONTAINER_SERVICE
327
+ end
328
+ end
288
329
  arr.each do |time, record|
289
330
  next unless record.is_a?(Hash)
331
+ if @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
332
+ @cloudfunctions_log_match =
333
+ @cloudfunctions_log_regexp.match(record['log'])
334
+ end
290
335
  if record.key?('timestamp') &&
291
336
  record['timestamp'].is_a?(Hash) &&
292
337
  record['timestamp'].key?('seconds') &&
@@ -312,6 +357,11 @@ module Fluent
312
357
  @log.warn 'timeNanos is deprecated - please use ' \
313
358
  'timestampSeconds and timestampNanos instead.'
314
359
  end
360
+ elsif @service_name == CLOUDFUNCTIONS_SERVICE &&
361
+ @cloudfunctions_log_match
362
+ timestamp = DateTime.parse(@cloudfunctions_log_match['timestamp'])
363
+ ts_secs = timestamp.strftime('%s')
364
+ ts_nanos = timestamp.strftime('%N')
315
365
  else
316
366
  timestamp = Time.at(time)
317
367
  ts_secs = timestamp.tv_sec
@@ -329,12 +379,8 @@ module Fluent
329
379
  'labels' => {}
330
380
  }
331
381
  }
332
- if record.key?('severity')
333
- entry['metadata']['severity'] = parse_severity(record['severity'])
334
- record.delete('severity')
335
- else
336
- entry['metadata']['severity'] = 'DEFAULT'
337
- end
382
+
383
+ set_severity(record, entry)
338
384
 
339
385
  # If the record has been annotated by the kubernetes_metadata_filter
340
386
  # plugin, then use that metadata. Otherwise, rely on commonLabels
@@ -352,14 +398,15 @@ module Fluent
352
398
  end
353
399
  end
354
400
 
355
- # use textPayload if the only remainaing key is 'message',
356
- # otherwise use a struct.
357
- if record.size == 1 && record.key?('message')
358
- entry['textPayload'] = record['message']
359
- else
360
- entry['structPayload'] = record
401
+ if @service_name == CLOUDFUNCTIONS_SERVICE &&
402
+ @cloudfunctions_log_match &&
403
+ @cloudfunctions_log_match['execution_id']
404
+ entry['metadata']['labels']['execution_id'] =
405
+ @cloudfunctions_log_match['execution_id']
361
406
  end
362
407
 
408
+ set_payload(record, entry)
409
+
363
410
  # Remove the labels metadata if we didn't populate it with anything.
364
411
  if entry['metadata']['labels'].empty?
365
412
  entry['metadata'].delete('labels')
@@ -370,10 +417,7 @@ module Fluent
370
417
  # Don't send an empty request if we rejected all the entries.
371
418
  next if write_log_entries_request['entries'].empty?
372
419
 
373
- # Add a prefix to VMEngines logs to prevent namespace collisions,
374
- # and also escape the log name.
375
- log_name = CGI.escape(
376
- @running_on_managed_vm ? "#{APPENGINE_SERVICE}/#{tag}" : tag)
420
+ log_name = CGI.escape(log_name(tag))
377
421
  url = 'https://logging.googleapis.com/v1beta3/projects/' \
378
422
  "#{@project_id}/logs/#{log_name}/entries:write"
379
423
  begin
@@ -483,6 +527,13 @@ module Fluent
483
527
  end
484
528
  end
485
529
 
530
+ def detect_cloudfunctions(attributes)
531
+ return unless attributes.include?('gcf_region')
532
+ # Cloud Functions detected
533
+ @running_cloudfunctions = true
534
+ @gcf_region = fetch_gce_metadata('instance/attributes/gcf_region')
535
+ end
536
+
486
537
  def cluster_name_from_kube_env(kube_env)
487
538
  return kube_env['CLUSTER_NAME'] if kube_env.key?('CLUSTER_NAME')
488
539
  instance_prefix = kube_env['INSTANCE_PREFIX']
@@ -492,6 +543,28 @@ module Fluent
492
543
  instance_prefix
493
544
  end
494
545
 
546
+ def set_severity(record, entry)
547
+ if @service_name == CLOUDFUNCTIONS_SERVICE
548
+ if @cloudfunctions_log_match && @cloudfunctions_log_match['severity']
549
+ entry['metadata']['severity'] =
550
+ parse_severity(@cloudfunctions_log_match['severity'])
551
+ elsif record.key?('stream') && record['stream'] == 'stdout'
552
+ entry['metadata']['severity'] = 'INFO'
553
+ record.delete('stream')
554
+ elsif record.key?('stream') && record['stream'] == 'stderr'
555
+ entry['metadata']['severity'] = 'ERROR'
556
+ record.delete('stream')
557
+ else
558
+ entry['metadata']['severity'] = 'DEFAULT'
559
+ end
560
+ elsif record.key?('severity')
561
+ entry['metadata']['severity'] = parse_severity(record['severity'])
562
+ record.delete('severity')
563
+ else
564
+ entry['metadata']['severity'] = 'DEFAULT'
565
+ end
566
+ end
567
+
495
568
  # Values permitted by the API for 'severity' (which is an enum).
496
569
  VALID_SEVERITIES = Set.new(
497
570
  %w(DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY))
@@ -577,10 +650,33 @@ module Fluent
577
650
  record.delete(field)
578
651
  end
579
652
 
653
+ def set_payload(record, entry)
654
+ # Use textPayload if this is the Cloud Functions service and 'log' key is
655
+ # available, or if the only remainaing key is 'message'.
656
+ if @service_name == CLOUDFUNCTIONS_SERVICE && @cloudfunctions_log_match
657
+ entry['textPayload'] = @cloudfunctions_log_match['text']
658
+ elsif @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
659
+ entry['textPayload'] = record['log']
660
+ elsif record.size == 1 && record.key?('message')
661
+ entry['textPayload'] = record['message']
662
+ else
663
+ entry['structPayload'] = record
664
+ end
665
+ end
666
+
667
+ def log_name(tag)
668
+ if @service_name == CLOUDFUNCTIONS_SERVICE
669
+ return 'cloud-functions'
670
+ else
671
+ # Add a prefix to VMEngines logs to prevent namespace collisions.
672
+ return @running_on_managed_vm ? "#{APPENGINE_SERVICE}/#{tag}" : tag
673
+ end
674
+ end
675
+
580
676
  def init_api_client
581
677
  @client = Google::APIClient.new(
582
678
  application_name: 'Fluentd Google Cloud Logging plugin',
583
- application_version: '0.4.7',
679
+ application_version: '0.4.8',
584
680
  retries: 1)
585
681
 
586
682
  if @auth_method == 'private_key'
@@ -65,9 +65,18 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
65
65
  CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'
66
66
  CONTAINER_NAMESPACE_NAME = 'kube-system'
67
67
  CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'
68
- CONTAINER_POD_NAME = 'redis-master-c0l82'
68
+ CONTAINER_POD_NAME = 'redis-master-c0l82.foo.bar'
69
69
  CONTAINER_CONTAINER_NAME = 'redis'
70
70
 
71
+ # Cloud Functions specific labels
72
+ CLOUDFUNCTIONS_FUNCTION_NAME = 'function-1'
73
+ CLOUDFUNCTIONS_REGION = 'us-central1'
74
+ CLOUDFUNCTIONS_EXECUTION_ID = '123-0'
75
+ CLOUDFUNCTIONS_CLUSTER_NAME = 'cluster-1'
76
+ CLOUDFUNCTIONS_NAMESPACE_NAME = 'default'
77
+ CLOUDFUNCTIONS_POD_NAME = "#{CLOUDFUNCTIONS_FUNCTION_NAME}-c0l82"
78
+ CLOUDFUNCTIONS_CONTAINER_NAME = 'worker'
79
+
71
80
  # Parameters used for authentication
72
81
  AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
73
82
  FAKE_AUTH_TOKEN = 'abc123'
@@ -135,6 +144,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
135
144
  COMPUTE_SERVICE_NAME = 'compute.googleapis.com'
136
145
  APPENGINE_SERVICE_NAME = 'appengine.googleapis.com'
137
146
  CONTAINER_SERVICE_NAME = 'container.googleapis.com'
147
+ CLOUDFUNCTIONS_SERVICE_NAME = 'cloudfunctions.googleapis.com'
138
148
  EC2_SERVICE_NAME = 'ec2.amazonaws.com'
139
149
 
140
150
  COMPUTE_PARAMS = {
@@ -203,6 +213,55 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
203
213
  }
204
214
  }
205
215
 
216
+ CLOUDFUNCTIONS_TAG = "kubernetes.#{CLOUDFUNCTIONS_POD_NAME}_" \
217
+ "#{CLOUDFUNCTIONS_NAMESPACE_NAME}_" \
218
+ "#{CLOUDFUNCTIONS_CONTAINER_NAME}"
219
+
220
+ CLOUDFUNCTIONS_PARAMS = {
221
+ 'service_name' => CLOUDFUNCTIONS_SERVICE_NAME,
222
+ 'log_name' => 'cloud-functions',
223
+ 'project_id' => PROJECT_ID,
224
+ 'zone' => ZONE,
225
+ 'labels' => {
226
+ 'execution_id' => CLOUDFUNCTIONS_EXECUTION_ID,
227
+ "#{CLOUDFUNCTIONS_SERVICE_NAME}/function_name" =>
228
+ CLOUDFUNCTIONS_FUNCTION_NAME,
229
+ "#{CLOUDFUNCTIONS_SERVICE_NAME}/region" => CLOUDFUNCTIONS_REGION,
230
+ "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
231
+ "#{CONTAINER_SERVICE_NAME}/cluster_name" => CLOUDFUNCTIONS_CLUSTER_NAME,
232
+ "#{CONTAINER_SERVICE_NAME}/namespace_name" =>
233
+ CLOUDFUNCTIONS_NAMESPACE_NAME,
234
+ "#{CONTAINER_SERVICE_NAME}/pod_name" => CLOUDFUNCTIONS_POD_NAME,
235
+ "#{CONTAINER_SERVICE_NAME}/container_name" =>
236
+ CLOUDFUNCTIONS_CONTAINER_NAME,
237
+ "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
238
+ "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
239
+ "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
240
+ }
241
+ }
242
+
243
+ CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS = {
244
+ 'service_name' => CLOUDFUNCTIONS_SERVICE_NAME,
245
+ 'log_name' => 'cloud-functions',
246
+ 'project_id' => PROJECT_ID,
247
+ 'zone' => ZONE,
248
+ 'labels' => {
249
+ "#{CLOUDFUNCTIONS_SERVICE_NAME}/function_name" =>
250
+ CLOUDFUNCTIONS_FUNCTION_NAME,
251
+ "#{CLOUDFUNCTIONS_SERVICE_NAME}/region" => CLOUDFUNCTIONS_REGION,
252
+ "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
253
+ "#{CONTAINER_SERVICE_NAME}/cluster_name" => CLOUDFUNCTIONS_CLUSTER_NAME,
254
+ "#{CONTAINER_SERVICE_NAME}/namespace_name" =>
255
+ CLOUDFUNCTIONS_NAMESPACE_NAME,
256
+ "#{CONTAINER_SERVICE_NAME}/pod_name" => CLOUDFUNCTIONS_POD_NAME,
257
+ "#{CONTAINER_SERVICE_NAME}/container_name" =>
258
+ CLOUDFUNCTIONS_CONTAINER_NAME,
259
+ "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
260
+ "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
261
+ "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
262
+ }
263
+ }
264
+
206
265
  CUSTOM_PARAMS = {
207
266
  'service_name' => COMPUTE_SERVICE_NAME,
208
267
  'log_name' => 'test',
@@ -839,6 +898,105 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
839
898
  end
840
899
  end
841
900
 
901
+ def test_one_cloudfunctions_log
902
+ setup_gce_metadata_stubs
903
+ setup_cloudfunctions_metadata_stubs
904
+ setup_logging_stubs
905
+ d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
906
+ d.emit(cloudfunctions_log_entry(0))
907
+ d.run
908
+ verify_log_entries(1, CLOUDFUNCTIONS_PARAMS) do |entry|
909
+ assert_equal 'DEBUG', entry['metadata']['severity'], entry
910
+ end
911
+ end
912
+
913
+ def test_multiple_cloudfunctions_logs
914
+ setup_gce_metadata_stubs
915
+ setup_cloudfunctions_metadata_stubs
916
+ setup_logging_stubs
917
+ d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
918
+ [2, 3, 5, 11, 50].each do |n|
919
+ # The test driver doesn't clear its buffer of entries after running, so
920
+ # do it manually here.
921
+ d.instance_variable_get('@entries').clear
922
+ @logs_sent = []
923
+ n.times { |i| d.emit(cloudfunctions_log_entry(i)) }
924
+ d.run
925
+ verify_log_entries(n, CLOUDFUNCTIONS_PARAMS) do |entry|
926
+ assert_equal 'DEBUG', entry['metadata']['severity'], entry
927
+ end
928
+ end
929
+ end
930
+
931
+ def test_one_cloudfunctions_log_text_not_matched
932
+ setup_gce_metadata_stubs
933
+ setup_cloudfunctions_metadata_stubs
934
+ setup_logging_stubs
935
+ d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
936
+ d.emit(cloudfunctions_log_entry_text_not_matched(0))
937
+ d.run
938
+ verify_log_entries(1, CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS) do |entry|
939
+ assert_equal 'INFO', entry['metadata']['severity'], entry
940
+ end
941
+ end
942
+
943
+ def test_multiple_cloudfunctions_logs_text_not_matched
944
+ setup_gce_metadata_stubs
945
+ setup_cloudfunctions_metadata_stubs
946
+ setup_logging_stubs
947
+ d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
948
+ [2, 3, 5, 11, 50].each do |n|
949
+ # The test driver doesn't clear its buffer of entries after running, so
950
+ # do it manually here.
951
+ d.instance_variable_get('@entries').clear
952
+ @logs_sent = []
953
+ n.times { |i| d.emit(cloudfunctions_log_entry_text_not_matched(i)) }
954
+ d.run
955
+ verify_log_entries(n, CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS) do |entry|
956
+ assert_equal 'INFO', entry['metadata']['severity'], entry
957
+ end
958
+ end
959
+ end
960
+
961
+ def test_one_cloudfunctions_log_tag_not_matched
962
+ setup_gce_metadata_stubs
963
+ setup_cloudfunctions_metadata_stubs
964
+ setup_logging_stubs
965
+ d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
966
+ d.emit(cloudfunctions_log_entry(0))
967
+ d.run
968
+ verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS, 'structPayload') do |entry|
969
+ assert_equal 2, entry['structPayload'].size, entry
970
+ assert_equal '[D][2015-09-25T12:34:56.789Z][123-0] test log entry 0',
971
+ entry['structPayload']['log'], entry
972
+ assert_equal 'stdout', entry['structPayload']['stream'], entry
973
+ end
974
+ end
975
+
976
+ def test_multiple_cloudfunctions_logs_tag_not_matched
977
+ setup_gce_metadata_stubs
978
+ setup_cloudfunctions_metadata_stubs
979
+ setup_logging_stubs
980
+ d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
981
+ [2, 3, 5, 11, 50].each do |n|
982
+ # The test driver doesn't clear its buffer of entries after running, so
983
+ # do it manually here.
984
+ d.instance_variable_get('@entries').clear
985
+ @logs_sent = []
986
+ n.times { |i| d.emit(cloudfunctions_log_entry(i)) }
987
+ d.run
988
+ i = 0
989
+ params = CONTAINER_FROM_TAG_PARAMS
990
+ verify_log_entries(n, params, 'structPayload') do |entry|
991
+ assert_equal 2, entry['structPayload'].size, entry
992
+ assert_equal "[D][2015-09-25T12:34:56.789Z][123-0] test log entry #{i}",
993
+ entry['structPayload']['log'], entry
994
+ assert_equal 'stdout', entry['structPayload']['stream'], entry
995
+ i += 1
996
+ end
997
+ end
998
+ end
999
+
842
1000
  # Make parse_severity public so we can test it.
843
1001
  class Fluent::GoogleCloudOutput # rubocop:disable Style/ClassAndModuleChildren
844
1002
  public :parse_severity
@@ -972,7 +1130,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
972
1130
 
973
1131
  def setup_logging_stubs
974
1132
  [COMPUTE_PARAMS, VMENGINE_PARAMS, CONTAINER_FROM_TAG_PARAMS,
975
- CONTAINER_FROM_METADATA_PARAMS, CUSTOM_PARAMS, EC2_PARAMS].each do |params|
1133
+ CONTAINER_FROM_METADATA_PARAMS, CLOUDFUNCTIONS_PARAMS, CUSTOM_PARAMS,
1134
+ EC2_PARAMS].each do |params|
976
1135
  stub_request(:post, uri_for_log(params)).to_return do |request|
977
1136
  @logs_sent << JSON.parse(request.body)
978
1137
  { body: '' }
@@ -1024,6 +1183,17 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
1024
1183
  'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
1025
1184
  end
1026
1185
 
1186
+ def setup_cloudfunctions_metadata_stubs
1187
+ stub_metadata_request(
1188
+ 'instance/attributes/',
1189
+ "attribute1\nkube-env\ngcf_region\nlast_attribute")
1190
+ stub_metadata_request('instance/attributes/kube-env',
1191
+ "ENABLE_NODE_LOGGING: \"true\"\n"\
1192
+ "INSTANCE_PREFIX: gke-cluster-1-740fdafa\n"\
1193
+ 'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
1194
+ stub_metadata_request('instance/attributes/gcf_region', 'us-central1')
1195
+ end
1196
+
1027
1197
  def container_log_entry_with_metadata(i)
1028
1198
  {
1029
1199
  message: log_entry(i),
@@ -1037,6 +1207,20 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
1037
1207
  }
1038
1208
  end
1039
1209
 
1210
+ def cloudfunctions_log_entry(i)
1211
+ {
1212
+ stream: 'stdout',
1213
+ log: '[D][2015-09-25T12:34:56.789Z][123-0] ' + log_entry(i)
1214
+ }
1215
+ end
1216
+
1217
+ def cloudfunctions_log_entry_text_not_matched(i)
1218
+ {
1219
+ stream: 'stdout',
1220
+ log: log_entry(i)
1221
+ }
1222
+ end
1223
+
1040
1224
  def log_entry(i)
1041
1225
  'test log entry ' + i.to_s
1042
1226
  end
@@ -1049,7 +1233,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
1049
1233
  assert value.is_a?(String), "Value #{value} for label #{key} " \
1050
1234
  'is not a string: ' + value.class.name
1051
1235
  assert expected_labels.key?(key), "Unexpected label #{key} => #{value}"
1052
- assert_equal value, expected_labels[key], 'Value mismatch - expected ' \
1236
+ assert_equal expected_labels[key], value, 'Value mismatch - expected ' \
1053
1237
  "#{expected_labels[key]} in #{key} => #{value}"
1054
1238
  end
1055
1239
  assert_equal expected_labels.length, all_labels.length, 'Expected ' \
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-google-cloud
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.7
4
+ version: 0.4.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Todd Derr
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2015-09-15 00:00:00.000000000 Z
12
+ date: 2015-09-30 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd