fluent-plugin-google-cloud 0.6.12 → 0.6.13.pre.memory.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 4fbc5121ee932ffffd68d89d4c907f836b82edbc
4
- data.tar.gz: d0e361c403ad696830514899b295709f0ef7c99c
3
+ metadata.gz: b26e153a8191c18cf52ed1efdc552d5323045683
4
+ data.tar.gz: 3111b02b15f0a930f7cbd0d28b5517ffbe56a0ab
5
5
  SHA512:
6
- metadata.gz: e5059d786227bd2d15224a359c002a309a6960117eeee63c1e4690e8606b2ac8de64aec1f6cf654847deab41e1fd524c82910361faea29eb2651c8f1167571ed
7
- data.tar.gz: 1576bea68999170b5bb78d22190c050cb060baebc5cb2e075f0d58cedad44667efa3a3b0b6b60ebcb78209f005218f1929d2dec08a0ac58d5e64569ce130a9a5
6
+ metadata.gz: 4e569bbf17acbb4c997c3321bee3d7bac9cdbaa21c5e8b6d6f40bf996906ac3cc41f4ae7e7ae2f8085251369cefa5a0be4261d3529c294a305c7e0bb4aba2d1c
7
+ data.tar.gz: 28562ed528b840677aca44c5f5982740e0e936a1eb9bb15ed8ca35d36f821fbc0931e8888ffe53a99cc4c73e622b10dbda51b8146c8ec2f1ecec97b6b671d042
data/Gemfile.lock CHANGED
@@ -1,20 +1,20 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- fluent-plugin-google-cloud (0.6.12)
4
+ fluent-plugin-google-cloud (0.6.13.pre.memory.1)
5
5
  fluentd (~> 0.10)
6
6
  google-api-client (~> 0.14)
7
7
  google-cloud-logging (~> 1.2.3)
8
8
  googleapis-common-protos (~> 1.3)
9
9
  googleauth (~> 0.5)
10
- grpc (~> 1.2.5)
10
+ grpc (~> 1.8)
11
11
  json (~> 1.8)
12
12
 
13
13
  GEM
14
14
  remote: https://rubygems.org/
15
15
  specs:
16
- addressable (2.5.1)
17
- public_suffix (~> 2.0, >= 2.0.2)
16
+ addressable (2.5.2)
17
+ public_suffix (>= 2.0.2, < 4.0)
18
18
  ast (2.3.0)
19
19
  cool.io (1.5.3)
20
20
  crack (0.4.3)
@@ -34,7 +34,7 @@ GEM
34
34
  tzinfo (~> 1.0)
35
35
  tzinfo-data (~> 1.0)
36
36
  yajl-ruby (~> 1.0)
37
- google-api-client (0.17.4)
37
+ google-api-client (0.19.3)
38
38
  addressable (~> 2.5, >= 2.5.1)
39
39
  googleauth (>= 0.5, < 0.7.0)
40
40
  httpclient (>= 2.8.1, < 3.0)
@@ -55,7 +55,7 @@ GEM
55
55
  googleauth (~> 0.5.1)
56
56
  grpc (~> 1.0)
57
57
  rly (~> 0.2.3)
58
- google-protobuf (3.5.0)
58
+ google-protobuf (3.5.1)
59
59
  googleapis-common-protos (1.3.7)
60
60
  google-protobuf (~> 3.0)
61
61
  googleapis-common-protos-types (~> 1.0)
@@ -70,10 +70,11 @@ GEM
70
70
  multi_json (~> 1.11)
71
71
  os (~> 0.9)
72
72
  signet (~> 0.7)
73
- grpc (1.2.5)
73
+ grpc (1.8.3)
74
74
  google-protobuf (~> 3.1)
75
- googleauth (~> 0.5.1)
76
- hashdiff (0.3.4)
75
+ googleapis-common-protos-types (~> 1.0.0)
76
+ googleauth (>= 0.5.1, < 0.7)
77
+ hashdiff (0.3.7)
77
78
  http_parser.rb (0.6.0)
78
79
  httpclient (2.8.3)
79
80
  json (1.8.6)
@@ -87,19 +88,19 @@ GEM
87
88
  mime-types (3.1)
88
89
  mime-types-data (~> 3.2015)
89
90
  mime-types-data (3.2016.0521)
90
- mocha (1.2.1)
91
+ mocha (1.3.0)
91
92
  metaclass (~> 0.0.1)
92
- msgpack (1.2.0)
93
- multi_json (1.12.2)
93
+ msgpack (1.2.2)
94
+ multi_json (1.13.1)
94
95
  multipart-post (2.0.0)
95
96
  os (0.9.6)
96
97
  parser (2.4.0.2)
97
98
  ast (~> 2.3)
98
- power_assert (1.0.1)
99
+ power_assert (1.1.1)
99
100
  powerpack (0.1.1)
100
101
  prometheus-client (0.7.1)
101
102
  quantile (~> 0.2.0)
102
- public_suffix (2.0.5)
103
+ public_suffix (3.0.1)
103
104
  quantile (0.2.0)
104
105
  rainbow (2.2.2)
105
106
  rake
@@ -119,7 +120,7 @@ GEM
119
120
  ruby-progressbar (1.9.0)
120
121
  ruby_dig (0.0.2)
121
122
  safe_yaml (1.0.4)
122
- serverengine (2.0.5)
123
+ serverengine (2.0.6)
123
124
  sigdump (~> 0.2.2)
124
125
  sigdump (0.2.4)
125
126
  signet (0.8.1)
@@ -129,7 +130,7 @@ GEM
129
130
  multi_json (~> 1.10)
130
131
  stackdriver-core (1.2.0)
131
132
  strptime (0.1.9)
132
- test-unit (3.2.3)
133
+ test-unit (3.2.7)
133
134
  power_assert
134
135
  thread_safe (0.3.6)
135
136
  tzinfo (1.2.4)
@@ -157,4 +158,4 @@ DEPENDENCIES
157
158
  webmock (~> 2.3.1)
158
159
 
159
160
  BUNDLED WITH
160
- 1.16.0
161
+ 1.16.1
@@ -10,7 +10,7 @@ eos
10
10
  gem.homepage =
11
11
  'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
12
12
  gem.license = 'Apache-2.0'
13
- gem.version = '0.6.12'
13
+ gem.version = '0.6.13.pre.memory.1'
14
14
  gem.authors = ['Todd Derr', 'Alex Robinson']
15
15
  gem.email = ['salty@google.com']
16
16
  gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
@@ -24,7 +24,7 @@ eos
24
24
  gem.add_runtime_dependency 'google-api-client', '~> 0.14'
25
25
  gem.add_runtime_dependency 'google-cloud-logging', '~> 1.2.3'
26
26
  gem.add_runtime_dependency 'googleauth', '~> 0.5'
27
- gem.add_runtime_dependency 'grpc', '~> 1.2.5'
27
+ gem.add_runtime_dependency 'grpc', '~> 1.8'
28
28
  gem.add_runtime_dependency 'json', '~> 1.8'
29
29
 
30
30
  gem.add_development_dependency 'mocha', '~> 1.1'
@@ -195,7 +195,7 @@ module Fluent
195
195
  Fluent::Plugin.register_output('google_cloud', self)
196
196
 
197
197
  PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'.freeze
198
- PLUGIN_VERSION = '0.6.12'.freeze
198
+ PLUGIN_VERSION = '0.6.13.pre.memory.1'.freeze
199
199
 
200
200
  # Name of the the Google cloud logging write scope.
201
201
  LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'.freeze
@@ -295,13 +295,13 @@ module Fluent
295
295
  config_param :labels, :hash, :default => nil
296
296
 
297
297
  # Whether to use gRPC instead of REST/JSON to communicate to the
298
- # Cloud Logging API.
298
+ # Stackdriver Logging API.
299
299
  config_param :use_grpc, :bool, :default => false
300
300
 
301
301
  # Whether valid entries should be written even if some other entries fail
302
302
  # due to INVALID_ARGUMENT or PERMISSION_DENIED errors when communicating to
303
- # the Cloud Logging API. This is highly recommended. Right now this only
304
- # works with the REST path (use_grpc = false).
303
+ # the Stackdriver Logging API. This is highly recommended. Right now this
304
+ # only works with the REST path (use_grpc = false).
305
305
  config_param :partial_success, :bool, :default => false
306
306
 
307
307
  # Whether to allow non-UTF-8 characters in user logs. If set to true, any
@@ -453,6 +453,14 @@ module Fluent
453
453
  @resource.labels.freeze
454
454
  @common_labels.freeze
455
455
 
456
+ if @use_grpc
457
+ @construct_log_entry = method(:construct_log_entry_in_grpc_format)
458
+ @write_request = method(:write_request_via_grpc)
459
+ else
460
+ @construct_log_entry = method(:construct_log_entry_in_rest_format)
461
+ @write_request = method(:write_request_via_rest)
462
+ end
463
+
456
464
  # Log an informational message containing the Logs viewer URL
457
465
  @log.info 'Logs viewer address: https://console.cloud.google.com/logs/',
458
466
  "viewer?project=#{@project_id}&resource=#{@resource.type}/",
@@ -473,6 +481,7 @@ module Fluent
473
481
  def write(chunk)
474
482
  grouped_entries = group_log_entries_by_tag_and_local_resource_id(chunk)
475
483
 
484
+ requests_to_send = []
476
485
  grouped_entries.each do |(tag, local_resource_id), arr|
477
486
  entries = []
478
487
  group_level_resource, group_level_common_labels =
@@ -517,40 +526,11 @@ module Fluent
517
526
  severity = compute_severity(
518
527
  entry_level_resource.type, record, entry_level_common_labels)
519
528
 
520
- if @use_grpc
521
- entry = Google::Logging::V2::LogEntry.new(
522
- labels: entry_level_common_labels,
523
- resource: Google::Api::MonitoredResource.new(
524
- type: entry_level_resource.type,
525
- labels: entry_level_resource.labels.to_h
526
- ),
527
- severity: grpc_severity(severity)
528
- )
529
- # If "seconds" is null or not an integer, we will omit the timestamp
530
- # field and defer the decision on how to handle it to the downstream
531
- # Logging API. If "nanos" is null or not an integer, it will be set
532
- # to 0.
533
- if ts_secs.is_a?(Integer)
534
- ts_nanos = 0 unless ts_nanos.is_a?(Integer)
535
- entry.timestamp = Google::Protobuf::Timestamp.new(
536
- seconds: ts_secs,
537
- nanos: ts_nanos
538
- )
539
- end
540
- else
541
- # Remove the labels if we didn't populate them with anything.
542
- entry_level_resource.labels = nil if
543
- entry_level_resource.labels.empty?
544
- entry = Google::Apis::LoggingV2::LogEntry.new(
545
- labels: entry_level_common_labels,
546
- resource: entry_level_resource,
547
- severity: severity,
548
- timestamp: {
549
- seconds: ts_secs,
550
- nanos: ts_nanos
551
- }
552
- )
553
- end
529
+ entry = @construct_log_entry.call(entry_level_common_labels,
530
+ entry_level_resource,
531
+ severity,
532
+ ts_secs,
533
+ ts_nanos)
554
534
 
555
535
  # Get fully-qualified trace id for LogEntry "trace" field.
556
536
  fq_trace_id = record.delete(@trace_key)
@@ -567,165 +547,226 @@ module Fluent
567
547
  log_name = "projects/#{@project_id}/logs/#{log_name(
568
548
  tag, group_level_resource)}"
569
549
 
570
- # Does the actual write to the cloud logging api.
571
- client = api_client
572
- if @use_grpc
573
- begin
574
- labels_utf8_pairs = group_level_common_labels.map do |k, v|
575
- [k.encode('utf-8'), convert_to_utf8(v)]
576
- end
550
+ requests_to_send << {
551
+ entries: entries,
552
+ log_name: log_name,
553
+ resource: group_level_resource,
554
+ labels: group_level_common_labels
555
+ }
556
+ end
577
557
 
578
- entries_count = entries.length
579
- client.write_log_entries(
580
- entries,
581
- log_name: log_name,
582
- resource: Google::Api::MonitoredResource.new(
583
- type: group_level_resource.type,
584
- labels: group_level_resource.labels.to_h
585
- ),
586
- labels: labels_utf8_pairs.to_h
587
- )
588
- increment_successful_requests_count
589
- increment_ingested_entries_count(entries_count)
590
-
591
- # Let the user explicitly know when the first call succeeded, to aid
592
- # with verification and troubleshooting.
593
- unless @successful_call
594
- @successful_call = true
595
- @log.info 'Successfully sent gRPC to Stackdriver Logging API.'
596
- end
558
+ requests_to_send.each do |request|
559
+ @write_request.call(request)
560
+ end
561
+ end
597
562
 
598
- rescue Google::Gax::GaxError => gax_error
599
- # GRPC::BadStatus is wrapped in error.cause.
600
- error = gax_error.cause
601
-
602
- # See the mapping between HTTP status and gRPC status code at:
603
- # https://github.com/grpc/grpc/blob/master/src/core/lib/transport/status_conversion.cc
604
- case error
605
- # Server error, so retry via re-raising the error.
606
- when \
607
- # HTTP status 500 (Internal Server Error).
608
- GRPC::Internal,
609
- # HTTP status 501 (Not Implemented).
610
- GRPC::Unimplemented,
611
- # HTTP status 503 (Service Unavailable).
612
- GRPC::Unavailable,
613
- # HTTP status 504 (Gateway Timeout).
614
- GRPC::DeadlineExceeded
615
- increment_retried_entries_count(entries_count, error.code)
616
- @log.debug "Retrying #{entries_count} log message(s) later.",
617
- error: error.to_s, error_code: error.code.to_s
618
- raise error
619
-
620
- # Most client errors indicate a problem with the request itself and
621
- # should not be retried.
622
- when \
623
- # HTTP status 400 (Bad Request).
624
- GRPC::InvalidArgument,
625
- # HTTP status 401 (Unauthorized).
626
- # These are usually solved via a `gcloud auth` call, or by
627
- # modifying the permissions on the Google Cloud project.
628
- GRPC::Unauthenticated,
629
- # HTTP status 403 (Forbidden).
630
- GRPC::PermissionDenied,
631
- # HTTP status 404 (Not Found).
632
- GRPC::NotFound,
633
- # HTTP status 409 (Conflict).
634
- GRPC::Aborted,
635
- # HTTP status 412 (Precondition Failed).
636
- GRPC::FailedPrecondition,
637
- # HTTP status 429 (Too Many Requests).
638
- GRPC::ResourceExhausted,
639
- # HTTP status 499 (Client Closed Request).
640
- GRPC::Cancelled,
641
- # the remaining http codes in both 4xx and 5xx category.
642
- # It's debatable whether to retry or drop these log entries.
643
- # This decision is made to avoid retrying forever due to client
644
- # errors.
645
- GRPC::Unknown
646
- increment_failed_requests_count(error.code)
647
- increment_dropped_entries_count(entries_count, error.code)
648
- @log.warn "Dropping #{entries_count} log message(s)",
649
- error: error.to_s, error_code: error.code.to_s
650
-
651
- else
652
- # Assume it is a problem with the request itself and don't retry.
653
- increment_failed_requests_count(error.code)
654
- increment_dropped_entries_count(entries_count, error.code)
655
- @log.error "Unknown response code #{error.code} from the "\
656
- "server, dropping #{entries_count} log message(s)",
657
- error: error.to_s, error_code: error.code.to_s
658
- end
659
- end
660
- else
661
- begin
662
- write_request =
663
- Google::Apis::LoggingV2::WriteLogEntriesRequest.new(
664
- log_name: log_name,
665
- resource: group_level_resource,
666
- labels: group_level_common_labels,
667
- partial_success: @partial_success,
668
- entries: entries)
669
- entries_count = entries.length
670
- client.write_entry_log_entries(
671
- write_request,
672
- options: { api_format_version: '2' }
673
- )
674
-
675
- increment_successful_requests_count
676
- increment_ingested_entries_count(entries_count)
677
-
678
- # Let the user explicitly know when the first call succeeded, to aid
679
- # with verification and troubleshooting.
680
- unless @successful_call
681
- @successful_call = true
682
- @log.info 'Successfully sent to Stackdriver Logging API.'
683
- end
563
+ private
684
564
 
685
- rescue Google::Apis::ServerError => error
686
- # 5xx server errors. Retry via re-raising the error.
687
- increment_retried_entries_count(entries_count, error.status_code)
688
- @log.debug "Retrying #{entries_count} log message(s) later.",
689
- error: error.to_s, error_code: error.status_code.to_s
690
- raise error
691
-
692
- rescue Google::Apis::AuthorizationError => error
693
- # 401 authorization error.
694
- # These are usually solved via a `gcloud auth` call, or by modifying
695
- # the permissions on the Google Cloud project.
696
- increment_failed_requests_count(error.status_code)
697
- increment_dropped_entries_count(entries_count, error.status_code)
698
- @log.warn "Dropping #{entries_count} log message(s)",
699
- error_class: error.class.to_s, error: error.to_s
700
-
701
- rescue Google::Apis::ClientError => error
702
- # 4xx client errors. Most client errors indicate a problem with the
703
- # request itself and should not be retried.
704
- error_details_map = construct_error_details_map(error)
705
- if error_details_map.empty?
706
- increment_failed_requests_count(error.status_code)
707
- increment_dropped_entries_count(entries_count, error.status_code)
708
- @log.warn "Dropping #{entries_count} log message(s)",
709
- error_class: error.class.to_s, error: error.to_s
710
- else
711
- error_details_map.each do |(error_code, error_message), indexes|
712
- partial_error_count = indexes.length
713
- increment_dropped_entries_count(partial_error_count, error_code)
714
- entries_count -= partial_error_count
715
- @log.warn "Dropping #{partial_error_count} log message(s)",
716
- error_code: "google.rpc.Code[#{error_code}]",
717
- error: error_message
718
- end
719
- # Consider partially successful requests successful.
720
- increment_successful_requests_count
721
- increment_ingested_entries_count(entries_count)
722
- end
723
- end
724
- end
565
+ def construct_log_entry_in_grpc_format(labels,
566
+ resource,
567
+ severity,
568
+ ts_secs,
569
+ ts_nanos)
570
+ entry = Google::Logging::V2::LogEntry.new(
571
+ labels: labels,
572
+ resource: Google::Api::MonitoredResource.new(
573
+ type: resource.type,
574
+ labels: resource.labels.to_h
575
+ ),
576
+ severity: grpc_severity(severity)
577
+ )
578
+ # If "seconds" is null or not an integer, we will omit the timestamp
579
+ # field and defer the decision on how to handle it to the downstream
580
+ # Logging API. If "nanos" is null or not an integer, it will be set
581
+ # to 0.
582
+ if ts_secs.is_a?(Integer)
583
+ ts_nanos = 0 unless ts_nanos.is_a?(Integer)
584
+ entry.timestamp = Google::Protobuf::Timestamp.new(
585
+ seconds: ts_secs,
586
+ nanos: ts_nanos
587
+ )
725
588
  end
589
+ entry
726
590
  end
727
591
 
728
- private
592
+ def construct_log_entry_in_rest_format(labels,
593
+ resource,
594
+ severity,
595
+ ts_secs,
596
+ ts_nanos)
597
+ # Remove the labels if we didn't populate them with anything.
598
+ resource.labels = nil if resource.labels.empty?
599
+ Google::Apis::LoggingV2::LogEntry.new(
600
+ labels: labels,
601
+ resource: resource,
602
+ severity: severity,
603
+ timestamp: {
604
+ seconds: ts_secs,
605
+ nanos: ts_nanos
606
+ }
607
+ )
608
+ end
609
+
610
+ def write_request_via_grpc(entries:, log_name:, resource:, labels:)
611
+ client = api_client
612
+ entries_count = entries.length
613
+ client.write_log_entries(
614
+ # Ignore partial_success for gRPC path.
615
+ entries,
616
+ log_name: log_name,
617
+ resource: Google::Api::MonitoredResource.new(
618
+ type: resource.type,
619
+ labels: resource.labels.to_h
620
+ ),
621
+ labels: labels.map do |k, v|
622
+ [k.encode('utf-8'), convert_to_utf8(v)]
623
+ end.to_h
624
+ )
625
+ increment_successful_requests_count
626
+ increment_ingested_entries_count(entries_count)
627
+
628
+ # Let the user explicitly know when the first call succeeded, to
629
+ # aid with verification and troubleshooting.
630
+ unless @successful_call
631
+ @successful_call = true
632
+ @log.info 'Successfully sent gRPC to Stackdriver Logging API.'
633
+ end
634
+
635
+ rescue Google::Gax::GaxError => gax_error
636
+ # GRPC::BadStatus is wrapped in error.cause.
637
+ error = gax_error.cause
638
+
639
+ # See the mapping between HTTP status and gRPC status code at:
640
+ # https://github.com/grpc/grpc/blob/master/src/core/lib/transport/status_conversion.cc
641
+ case error
642
+ # Server error, so retry via re-raising the error.
643
+ when \
644
+ # HTTP status 500 (Internal Server Error).
645
+ GRPC::Internal,
646
+ # HTTP status 501 (Not Implemented).
647
+ GRPC::Unimplemented,
648
+ # HTTP status 503 (Service Unavailable).
649
+ GRPC::Unavailable,
650
+ # HTTP status 504 (Gateway Timeout).
651
+ GRPC::DeadlineExceeded
652
+ increment_retried_entries_count(entries_count, error.code)
653
+ @log.debug "Retrying #{entries_count} log message(s) later.",
654
+ error: error.to_s, error_code: error.code.to_s
655
+ raise error
656
+
657
+ # Most client errors indicate a problem with the request itself and
658
+ # should not be retried.
659
+ when \
660
+ # HTTP status 400 (Bad Request).
661
+ GRPC::InvalidArgument,
662
+ # HTTP status 401 (Unauthorized).
663
+ # These are usually solved via a `gcloud auth` call, or by
664
+ # modifying the permissions on the Google Cloud project.
665
+ GRPC::Unauthenticated,
666
+ # HTTP status 403 (Forbidden).
667
+ GRPC::PermissionDenied,
668
+ # HTTP status 404 (Not Found).
669
+ GRPC::NotFound,
670
+ # HTTP status 409 (Conflict).
671
+ GRPC::Aborted,
672
+ # HTTP status 412 (Precondition Failed).
673
+ GRPC::FailedPrecondition,
674
+ # HTTP status 429 (Too Many Requests).
675
+ GRPC::ResourceExhausted,
676
+ # HTTP status 499 (Client Closed Request).
677
+ GRPC::Cancelled,
678
+ # the remaining http codes in both 4xx and 5xx category.
679
+ # It's debatable whether to retry or drop these log entries.
680
+ # This decision is made to avoid retrying forever due to
681
+ # client errors.
682
+ GRPC::Unknown
683
+ increment_failed_requests_count(error.code)
684
+ increment_dropped_entries_count(entries_count, error.code)
685
+ @log.warn "Dropping #{entries_count} log message(s)",
686
+ error: error.to_s, error_code: error.code.to_s
687
+
688
+ else
689
+ # Assume it's a problem with the request itself and don't retry.
690
+ increment_failed_requests_count(error.code)
691
+ increment_dropped_entries_count(entries_count, error.code)
692
+ @log.error "Unknown response code #{error.code} from the server," \
693
+ " dropping #{entries_count} log message(s)",
694
+ error: error.to_s, error_code: error.code.to_s
695
+ end
696
+
697
+ # Got an unexpected error (not Google::Gax::GaxError) from the
698
+ # google-cloud-logging lib.
699
+ rescue StandardError => error
700
+ increment_failed_requests_count(GRPC::Core::StatusCodes::UNKNOWN)
701
+ increment_dropped_entries_count(entries_count,
702
+ GRPC::Core::StatusCodes::UNKNOWN)
703
+ @log.error "Unexpected error type #{error.class.name} from the client" \
704
+ " library, dropping #{entries_count} log message(s)",
705
+ error: error.to_s
706
+ end
707
+
708
+ def write_request_via_rest(entries:, log_name:, resource:, labels:)
709
+ client = api_client
710
+ entries_count = entries.length
711
+ client.write_entry_log_entries(
712
+ Google::Apis::LoggingV2::WriteLogEntriesRequest.new(
713
+ entries: entries,
714
+ log_name: log_name,
715
+ resource: resource,
716
+ labels: labels,
717
+ partial_success: @partial_success
718
+ ),
719
+ options: { api_format_version: '2' }
720
+ )
721
+ increment_successful_requests_count
722
+ increment_ingested_entries_count(entries_count)
723
+
724
+ # Let the user explicitly know when the first call succeeded, to aid
725
+ # with verification and troubleshooting.
726
+ unless @successful_call
727
+ @successful_call = true
728
+ @log.info 'Successfully sent to Stackdriver Logging API.'
729
+ end
730
+
731
+ rescue Google::Apis::ServerError => error
732
+ # 5xx server errors. Retry via re-raising the error.
733
+ increment_retried_entries_count(entries_count, error.status_code)
734
+ @log.debug "Retrying #{entries_count} log message(s) later.",
735
+ error: error.to_s, error_code: error.status_code.to_s
736
+ raise error
737
+
738
+ rescue Google::Apis::AuthorizationError => error
739
+ # 401 authorization error.
740
+ # These are usually solved via a `gcloud auth` call, or by modifying
741
+ # the permissions on the Google Cloud project.
742
+ increment_failed_requests_count(error.status_code)
743
+ increment_dropped_entries_count(entries_count, error.status_code)
744
+ @log.warn "Dropping #{entries_count} log message(s)",
745
+ error_class: error.class.to_s, error: error.to_s
746
+
747
+ rescue Google::Apis::ClientError => error
748
+ # 4xx client errors. Most client errors indicate a problem with the
749
+ # request itself and should not be retried.
750
+ error_details_map = construct_error_details_map(error)
751
+ if error_details_map.empty?
752
+ increment_failed_requests_count(error.status_code)
753
+ increment_dropped_entries_count(entries_count, error.status_code)
754
+ @log.warn "Dropping #{entries_count} log message(s)",
755
+ error_class: error.class.to_s, error: error.to_s
756
+ else
757
+ error_details_map.each do |(error_code, error_message), indexes|
758
+ partial_error_count = indexes.length
759
+ increment_dropped_entries_count(partial_error_count, error_code)
760
+ entries_count -= partial_error_count
761
+ @log.warn "Dropping #{partial_error_count} log message(s)",
762
+ error_code: "google.rpc.Code[#{error_code}]",
763
+ error: error_message
764
+ end
765
+ # Consider partially successful requests successful.
766
+ increment_successful_requests_count
767
+ increment_ingested_entries_count(entries_count)
768
+ end
769
+ end
729
770
 
730
771
  def parse_json_or_nil(input)
731
772
  # Only here to please rubocop...
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-google-cloud
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.12
4
+ version: 0.6.13.pre.memory.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Todd Derr
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-12-07 00:00:00.000000000 Z
12
+ date: 2018-01-16 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
@@ -87,14 +87,14 @@ dependencies:
87
87
  requirements:
88
88
  - - "~>"
89
89
  - !ruby/object:Gem::Version
90
- version: 1.2.5
90
+ version: '1.8'
91
91
  type: :runtime
92
92
  prerelease: false
93
93
  version_requirements: !ruby/object:Gem::Requirement
94
94
  requirements:
95
95
  - - "~>"
96
96
  - !ruby/object:Gem::Version
97
- version: 1.2.5
97
+ version: '1.8'
98
98
  - !ruby/object:Gem::Dependency
99
99
  name: json
100
100
  requirement: !ruby/object:Gem::Requirement
@@ -237,12 +237,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
237
237
  version: '2.0'
238
238
  required_rubygems_version: !ruby/object:Gem::Requirement
239
239
  requirements:
240
- - - ">="
240
+ - - ">"
241
241
  - !ruby/object:Gem::Version
242
- version: '0'
242
+ version: 1.3.1
243
243
  requirements: []
244
244
  rubyforge_project:
245
- rubygems_version: 2.6.12
245
+ rubygems_version: 2.4.8
246
246
  signing_key:
247
247
  specification_version: 4
248
248
  summary: fluentd output plugin for the Stackdriver Logging API