fluent-plugin-google-cloud 0.6.12 → 0.6.13.pre.memory.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +18 -17
- data/fluent-plugin-google-cloud.gemspec +2 -2
- data/lib/fluent/plugin/out_google_cloud.rb +232 -191
- metadata +7 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b26e153a8191c18cf52ed1efdc552d5323045683
|
4
|
+
data.tar.gz: 3111b02b15f0a930f7cbd0d28b5517ffbe56a0ab
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4e569bbf17acbb4c997c3321bee3d7bac9cdbaa21c5e8b6d6f40bf996906ac3cc41f4ae7e7ae2f8085251369cefa5a0be4261d3529c294a305c7e0bb4aba2d1c
|
7
|
+
data.tar.gz: 28562ed528b840677aca44c5f5982740e0e936a1eb9bb15ed8ca35d36f821fbc0931e8888ffe53a99cc4c73e622b10dbda51b8146c8ec2f1ecec97b6b671d042
|
data/Gemfile.lock
CHANGED
@@ -1,20 +1,20 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-google-cloud (0.6.
|
4
|
+
fluent-plugin-google-cloud (0.6.13.pre.memory.1)
|
5
5
|
fluentd (~> 0.10)
|
6
6
|
google-api-client (~> 0.14)
|
7
7
|
google-cloud-logging (~> 1.2.3)
|
8
8
|
googleapis-common-protos (~> 1.3)
|
9
9
|
googleauth (~> 0.5)
|
10
|
-
grpc (~> 1.
|
10
|
+
grpc (~> 1.8)
|
11
11
|
json (~> 1.8)
|
12
12
|
|
13
13
|
GEM
|
14
14
|
remote: https://rubygems.org/
|
15
15
|
specs:
|
16
|
-
addressable (2.5.
|
17
|
-
public_suffix (
|
16
|
+
addressable (2.5.2)
|
17
|
+
public_suffix (>= 2.0.2, < 4.0)
|
18
18
|
ast (2.3.0)
|
19
19
|
cool.io (1.5.3)
|
20
20
|
crack (0.4.3)
|
@@ -34,7 +34,7 @@ GEM
|
|
34
34
|
tzinfo (~> 1.0)
|
35
35
|
tzinfo-data (~> 1.0)
|
36
36
|
yajl-ruby (~> 1.0)
|
37
|
-
google-api-client (0.
|
37
|
+
google-api-client (0.19.3)
|
38
38
|
addressable (~> 2.5, >= 2.5.1)
|
39
39
|
googleauth (>= 0.5, < 0.7.0)
|
40
40
|
httpclient (>= 2.8.1, < 3.0)
|
@@ -55,7 +55,7 @@ GEM
|
|
55
55
|
googleauth (~> 0.5.1)
|
56
56
|
grpc (~> 1.0)
|
57
57
|
rly (~> 0.2.3)
|
58
|
-
google-protobuf (3.5.
|
58
|
+
google-protobuf (3.5.1)
|
59
59
|
googleapis-common-protos (1.3.7)
|
60
60
|
google-protobuf (~> 3.0)
|
61
61
|
googleapis-common-protos-types (~> 1.0)
|
@@ -70,10 +70,11 @@ GEM
|
|
70
70
|
multi_json (~> 1.11)
|
71
71
|
os (~> 0.9)
|
72
72
|
signet (~> 0.7)
|
73
|
-
grpc (1.
|
73
|
+
grpc (1.8.3)
|
74
74
|
google-protobuf (~> 3.1)
|
75
|
-
|
76
|
-
|
75
|
+
googleapis-common-protos-types (~> 1.0.0)
|
76
|
+
googleauth (>= 0.5.1, < 0.7)
|
77
|
+
hashdiff (0.3.7)
|
77
78
|
http_parser.rb (0.6.0)
|
78
79
|
httpclient (2.8.3)
|
79
80
|
json (1.8.6)
|
@@ -87,19 +88,19 @@ GEM
|
|
87
88
|
mime-types (3.1)
|
88
89
|
mime-types-data (~> 3.2015)
|
89
90
|
mime-types-data (3.2016.0521)
|
90
|
-
mocha (1.
|
91
|
+
mocha (1.3.0)
|
91
92
|
metaclass (~> 0.0.1)
|
92
|
-
msgpack (1.2.
|
93
|
-
multi_json (1.
|
93
|
+
msgpack (1.2.2)
|
94
|
+
multi_json (1.13.1)
|
94
95
|
multipart-post (2.0.0)
|
95
96
|
os (0.9.6)
|
96
97
|
parser (2.4.0.2)
|
97
98
|
ast (~> 2.3)
|
98
|
-
power_assert (1.
|
99
|
+
power_assert (1.1.1)
|
99
100
|
powerpack (0.1.1)
|
100
101
|
prometheus-client (0.7.1)
|
101
102
|
quantile (~> 0.2.0)
|
102
|
-
public_suffix (
|
103
|
+
public_suffix (3.0.1)
|
103
104
|
quantile (0.2.0)
|
104
105
|
rainbow (2.2.2)
|
105
106
|
rake
|
@@ -119,7 +120,7 @@ GEM
|
|
119
120
|
ruby-progressbar (1.9.0)
|
120
121
|
ruby_dig (0.0.2)
|
121
122
|
safe_yaml (1.0.4)
|
122
|
-
serverengine (2.0.
|
123
|
+
serverengine (2.0.6)
|
123
124
|
sigdump (~> 0.2.2)
|
124
125
|
sigdump (0.2.4)
|
125
126
|
signet (0.8.1)
|
@@ -129,7 +130,7 @@ GEM
|
|
129
130
|
multi_json (~> 1.10)
|
130
131
|
stackdriver-core (1.2.0)
|
131
132
|
strptime (0.1.9)
|
132
|
-
test-unit (3.2.
|
133
|
+
test-unit (3.2.7)
|
133
134
|
power_assert
|
134
135
|
thread_safe (0.3.6)
|
135
136
|
tzinfo (1.2.4)
|
@@ -157,4 +158,4 @@ DEPENDENCIES
|
|
157
158
|
webmock (~> 2.3.1)
|
158
159
|
|
159
160
|
BUNDLED WITH
|
160
|
-
1.16.
|
161
|
+
1.16.1
|
@@ -10,7 +10,7 @@ eos
|
|
10
10
|
gem.homepage =
|
11
11
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
12
12
|
gem.license = 'Apache-2.0'
|
13
|
-
gem.version = '0.6.
|
13
|
+
gem.version = '0.6.13.pre.memory.1'
|
14
14
|
gem.authors = ['Todd Derr', 'Alex Robinson']
|
15
15
|
gem.email = ['salty@google.com']
|
16
16
|
gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
|
@@ -24,7 +24,7 @@ eos
|
|
24
24
|
gem.add_runtime_dependency 'google-api-client', '~> 0.14'
|
25
25
|
gem.add_runtime_dependency 'google-cloud-logging', '~> 1.2.3'
|
26
26
|
gem.add_runtime_dependency 'googleauth', '~> 0.5'
|
27
|
-
gem.add_runtime_dependency 'grpc', '~> 1.
|
27
|
+
gem.add_runtime_dependency 'grpc', '~> 1.8'
|
28
28
|
gem.add_runtime_dependency 'json', '~> 1.8'
|
29
29
|
|
30
30
|
gem.add_development_dependency 'mocha', '~> 1.1'
|
@@ -195,7 +195,7 @@ module Fluent
|
|
195
195
|
Fluent::Plugin.register_output('google_cloud', self)
|
196
196
|
|
197
197
|
PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'.freeze
|
198
|
-
PLUGIN_VERSION = '0.6.
|
198
|
+
PLUGIN_VERSION = '0.6.13.pre.memory.1'.freeze
|
199
199
|
|
200
200
|
# Name of the the Google cloud logging write scope.
|
201
201
|
LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'.freeze
|
@@ -295,13 +295,13 @@ module Fluent
|
|
295
295
|
config_param :labels, :hash, :default => nil
|
296
296
|
|
297
297
|
# Whether to use gRPC instead of REST/JSON to communicate to the
|
298
|
-
#
|
298
|
+
# Stackdriver Logging API.
|
299
299
|
config_param :use_grpc, :bool, :default => false
|
300
300
|
|
301
301
|
# Whether valid entries should be written even if some other entries fail
|
302
302
|
# due to INVALID_ARGUMENT or PERMISSION_DENIED errors when communicating to
|
303
|
-
# the
|
304
|
-
# works with the REST path (use_grpc = false).
|
303
|
+
# the Stackdriver Logging API. This is highly recommended. Right now this
|
304
|
+
# only works with the REST path (use_grpc = false).
|
305
305
|
config_param :partial_success, :bool, :default => false
|
306
306
|
|
307
307
|
# Whether to allow non-UTF-8 characters in user logs. If set to true, any
|
@@ -453,6 +453,14 @@ module Fluent
|
|
453
453
|
@resource.labels.freeze
|
454
454
|
@common_labels.freeze
|
455
455
|
|
456
|
+
if @use_grpc
|
457
|
+
@construct_log_entry = method(:construct_log_entry_in_grpc_format)
|
458
|
+
@write_request = method(:write_request_via_grpc)
|
459
|
+
else
|
460
|
+
@construct_log_entry = method(:construct_log_entry_in_rest_format)
|
461
|
+
@write_request = method(:write_request_via_rest)
|
462
|
+
end
|
463
|
+
|
456
464
|
# Log an informational message containing the Logs viewer URL
|
457
465
|
@log.info 'Logs viewer address: https://console.cloud.google.com/logs/',
|
458
466
|
"viewer?project=#{@project_id}&resource=#{@resource.type}/",
|
@@ -473,6 +481,7 @@ module Fluent
|
|
473
481
|
def write(chunk)
|
474
482
|
grouped_entries = group_log_entries_by_tag_and_local_resource_id(chunk)
|
475
483
|
|
484
|
+
requests_to_send = []
|
476
485
|
grouped_entries.each do |(tag, local_resource_id), arr|
|
477
486
|
entries = []
|
478
487
|
group_level_resource, group_level_common_labels =
|
@@ -517,40 +526,11 @@ module Fluent
|
|
517
526
|
severity = compute_severity(
|
518
527
|
entry_level_resource.type, record, entry_level_common_labels)
|
519
528
|
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
labels: entry_level_resource.labels.to_h
|
526
|
-
),
|
527
|
-
severity: grpc_severity(severity)
|
528
|
-
)
|
529
|
-
# If "seconds" is null or not an integer, we will omit the timestamp
|
530
|
-
# field and defer the decision on how to handle it to the downstream
|
531
|
-
# Logging API. If "nanos" is null or not an integer, it will be set
|
532
|
-
# to 0.
|
533
|
-
if ts_secs.is_a?(Integer)
|
534
|
-
ts_nanos = 0 unless ts_nanos.is_a?(Integer)
|
535
|
-
entry.timestamp = Google::Protobuf::Timestamp.new(
|
536
|
-
seconds: ts_secs,
|
537
|
-
nanos: ts_nanos
|
538
|
-
)
|
539
|
-
end
|
540
|
-
else
|
541
|
-
# Remove the labels if we didn't populate them with anything.
|
542
|
-
entry_level_resource.labels = nil if
|
543
|
-
entry_level_resource.labels.empty?
|
544
|
-
entry = Google::Apis::LoggingV2::LogEntry.new(
|
545
|
-
labels: entry_level_common_labels,
|
546
|
-
resource: entry_level_resource,
|
547
|
-
severity: severity,
|
548
|
-
timestamp: {
|
549
|
-
seconds: ts_secs,
|
550
|
-
nanos: ts_nanos
|
551
|
-
}
|
552
|
-
)
|
553
|
-
end
|
529
|
+
entry = @construct_log_entry.call(entry_level_common_labels,
|
530
|
+
entry_level_resource,
|
531
|
+
severity,
|
532
|
+
ts_secs,
|
533
|
+
ts_nanos)
|
554
534
|
|
555
535
|
# Get fully-qualified trace id for LogEntry "trace" field.
|
556
536
|
fq_trace_id = record.delete(@trace_key)
|
@@ -567,165 +547,226 @@ module Fluent
|
|
567
547
|
log_name = "projects/#{@project_id}/logs/#{log_name(
|
568
548
|
tag, group_level_resource)}"
|
569
549
|
|
570
|
-
|
571
|
-
|
572
|
-
|
573
|
-
|
574
|
-
|
575
|
-
|
576
|
-
|
550
|
+
requests_to_send << {
|
551
|
+
entries: entries,
|
552
|
+
log_name: log_name,
|
553
|
+
resource: group_level_resource,
|
554
|
+
labels: group_level_common_labels
|
555
|
+
}
|
556
|
+
end
|
577
557
|
|
578
|
-
|
579
|
-
|
580
|
-
|
581
|
-
|
582
|
-
resource: Google::Api::MonitoredResource.new(
|
583
|
-
type: group_level_resource.type,
|
584
|
-
labels: group_level_resource.labels.to_h
|
585
|
-
),
|
586
|
-
labels: labels_utf8_pairs.to_h
|
587
|
-
)
|
588
|
-
increment_successful_requests_count
|
589
|
-
increment_ingested_entries_count(entries_count)
|
590
|
-
|
591
|
-
# Let the user explicitly know when the first call succeeded, to aid
|
592
|
-
# with verification and troubleshooting.
|
593
|
-
unless @successful_call
|
594
|
-
@successful_call = true
|
595
|
-
@log.info 'Successfully sent gRPC to Stackdriver Logging API.'
|
596
|
-
end
|
558
|
+
requests_to_send.each do |request|
|
559
|
+
@write_request.call(request)
|
560
|
+
end
|
561
|
+
end
|
597
562
|
|
598
|
-
|
599
|
-
# GRPC::BadStatus is wrapped in error.cause.
|
600
|
-
error = gax_error.cause
|
601
|
-
|
602
|
-
# See the mapping between HTTP status and gRPC status code at:
|
603
|
-
# https://github.com/grpc/grpc/blob/master/src/core/lib/transport/status_conversion.cc
|
604
|
-
case error
|
605
|
-
# Server error, so retry via re-raising the error.
|
606
|
-
when \
|
607
|
-
# HTTP status 500 (Internal Server Error).
|
608
|
-
GRPC::Internal,
|
609
|
-
# HTTP status 501 (Not Implemented).
|
610
|
-
GRPC::Unimplemented,
|
611
|
-
# HTTP status 503 (Service Unavailable).
|
612
|
-
GRPC::Unavailable,
|
613
|
-
# HTTP status 504 (Gateway Timeout).
|
614
|
-
GRPC::DeadlineExceeded
|
615
|
-
increment_retried_entries_count(entries_count, error.code)
|
616
|
-
@log.debug "Retrying #{entries_count} log message(s) later.",
|
617
|
-
error: error.to_s, error_code: error.code.to_s
|
618
|
-
raise error
|
619
|
-
|
620
|
-
# Most client errors indicate a problem with the request itself and
|
621
|
-
# should not be retried.
|
622
|
-
when \
|
623
|
-
# HTTP status 400 (Bad Request).
|
624
|
-
GRPC::InvalidArgument,
|
625
|
-
# HTTP status 401 (Unauthorized).
|
626
|
-
# These are usually solved via a `gcloud auth` call, or by
|
627
|
-
# modifying the permissions on the Google Cloud project.
|
628
|
-
GRPC::Unauthenticated,
|
629
|
-
# HTTP status 403 (Forbidden).
|
630
|
-
GRPC::PermissionDenied,
|
631
|
-
# HTTP status 404 (Not Found).
|
632
|
-
GRPC::NotFound,
|
633
|
-
# HTTP status 409 (Conflict).
|
634
|
-
GRPC::Aborted,
|
635
|
-
# HTTP status 412 (Precondition Failed).
|
636
|
-
GRPC::FailedPrecondition,
|
637
|
-
# HTTP status 429 (Too Many Requests).
|
638
|
-
GRPC::ResourceExhausted,
|
639
|
-
# HTTP status 499 (Client Closed Request).
|
640
|
-
GRPC::Cancelled,
|
641
|
-
# the remaining http codes in both 4xx and 5xx category.
|
642
|
-
# It's debatable whether to retry or drop these log entries.
|
643
|
-
# This decision is made to avoid retrying forever due to client
|
644
|
-
# errors.
|
645
|
-
GRPC::Unknown
|
646
|
-
increment_failed_requests_count(error.code)
|
647
|
-
increment_dropped_entries_count(entries_count, error.code)
|
648
|
-
@log.warn "Dropping #{entries_count} log message(s)",
|
649
|
-
error: error.to_s, error_code: error.code.to_s
|
650
|
-
|
651
|
-
else
|
652
|
-
# Assume it is a problem with the request itself and don't retry.
|
653
|
-
increment_failed_requests_count(error.code)
|
654
|
-
increment_dropped_entries_count(entries_count, error.code)
|
655
|
-
@log.error "Unknown response code #{error.code} from the "\
|
656
|
-
"server, dropping #{entries_count} log message(s)",
|
657
|
-
error: error.to_s, error_code: error.code.to_s
|
658
|
-
end
|
659
|
-
end
|
660
|
-
else
|
661
|
-
begin
|
662
|
-
write_request =
|
663
|
-
Google::Apis::LoggingV2::WriteLogEntriesRequest.new(
|
664
|
-
log_name: log_name,
|
665
|
-
resource: group_level_resource,
|
666
|
-
labels: group_level_common_labels,
|
667
|
-
partial_success: @partial_success,
|
668
|
-
entries: entries)
|
669
|
-
entries_count = entries.length
|
670
|
-
client.write_entry_log_entries(
|
671
|
-
write_request,
|
672
|
-
options: { api_format_version: '2' }
|
673
|
-
)
|
674
|
-
|
675
|
-
increment_successful_requests_count
|
676
|
-
increment_ingested_entries_count(entries_count)
|
677
|
-
|
678
|
-
# Let the user explicitly know when the first call succeeded, to aid
|
679
|
-
# with verification and troubleshooting.
|
680
|
-
unless @successful_call
|
681
|
-
@successful_call = true
|
682
|
-
@log.info 'Successfully sent to Stackdriver Logging API.'
|
683
|
-
end
|
563
|
+
private
|
684
564
|
|
685
|
-
|
686
|
-
|
687
|
-
|
688
|
-
|
689
|
-
|
690
|
-
|
691
|
-
|
692
|
-
|
693
|
-
|
694
|
-
|
695
|
-
|
696
|
-
|
697
|
-
|
698
|
-
|
699
|
-
|
700
|
-
|
701
|
-
|
702
|
-
|
703
|
-
|
704
|
-
|
705
|
-
|
706
|
-
|
707
|
-
|
708
|
-
@log.warn "Dropping #{entries_count} log message(s)",
|
709
|
-
error_class: error.class.to_s, error: error.to_s
|
710
|
-
else
|
711
|
-
error_details_map.each do |(error_code, error_message), indexes|
|
712
|
-
partial_error_count = indexes.length
|
713
|
-
increment_dropped_entries_count(partial_error_count, error_code)
|
714
|
-
entries_count -= partial_error_count
|
715
|
-
@log.warn "Dropping #{partial_error_count} log message(s)",
|
716
|
-
error_code: "google.rpc.Code[#{error_code}]",
|
717
|
-
error: error_message
|
718
|
-
end
|
719
|
-
# Consider partially successful requests successful.
|
720
|
-
increment_successful_requests_count
|
721
|
-
increment_ingested_entries_count(entries_count)
|
722
|
-
end
|
723
|
-
end
|
724
|
-
end
|
565
|
+
def construct_log_entry_in_grpc_format(labels,
|
566
|
+
resource,
|
567
|
+
severity,
|
568
|
+
ts_secs,
|
569
|
+
ts_nanos)
|
570
|
+
entry = Google::Logging::V2::LogEntry.new(
|
571
|
+
labels: labels,
|
572
|
+
resource: Google::Api::MonitoredResource.new(
|
573
|
+
type: resource.type,
|
574
|
+
labels: resource.labels.to_h
|
575
|
+
),
|
576
|
+
severity: grpc_severity(severity)
|
577
|
+
)
|
578
|
+
# If "seconds" is null or not an integer, we will omit the timestamp
|
579
|
+
# field and defer the decision on how to handle it to the downstream
|
580
|
+
# Logging API. If "nanos" is null or not an integer, it will be set
|
581
|
+
# to 0.
|
582
|
+
if ts_secs.is_a?(Integer)
|
583
|
+
ts_nanos = 0 unless ts_nanos.is_a?(Integer)
|
584
|
+
entry.timestamp = Google::Protobuf::Timestamp.new(
|
585
|
+
seconds: ts_secs,
|
586
|
+
nanos: ts_nanos
|
587
|
+
)
|
725
588
|
end
|
589
|
+
entry
|
726
590
|
end
|
727
591
|
|
728
|
-
|
592
|
+
def construct_log_entry_in_rest_format(labels,
|
593
|
+
resource,
|
594
|
+
severity,
|
595
|
+
ts_secs,
|
596
|
+
ts_nanos)
|
597
|
+
# Remove the labels if we didn't populate them with anything.
|
598
|
+
resource.labels = nil if resource.labels.empty?
|
599
|
+
Google::Apis::LoggingV2::LogEntry.new(
|
600
|
+
labels: labels,
|
601
|
+
resource: resource,
|
602
|
+
severity: severity,
|
603
|
+
timestamp: {
|
604
|
+
seconds: ts_secs,
|
605
|
+
nanos: ts_nanos
|
606
|
+
}
|
607
|
+
)
|
608
|
+
end
|
609
|
+
|
610
|
+
def write_request_via_grpc(entries:, log_name:, resource:, labels:)
|
611
|
+
client = api_client
|
612
|
+
entries_count = entries.length
|
613
|
+
client.write_log_entries(
|
614
|
+
# Ignore partial_success for gRPC path.
|
615
|
+
entries,
|
616
|
+
log_name: log_name,
|
617
|
+
resource: Google::Api::MonitoredResource.new(
|
618
|
+
type: resource.type,
|
619
|
+
labels: resource.labels.to_h
|
620
|
+
),
|
621
|
+
labels: labels.map do |k, v|
|
622
|
+
[k.encode('utf-8'), convert_to_utf8(v)]
|
623
|
+
end.to_h
|
624
|
+
)
|
625
|
+
increment_successful_requests_count
|
626
|
+
increment_ingested_entries_count(entries_count)
|
627
|
+
|
628
|
+
# Let the user explicitly know when the first call succeeded, to
|
629
|
+
# aid with verification and troubleshooting.
|
630
|
+
unless @successful_call
|
631
|
+
@successful_call = true
|
632
|
+
@log.info 'Successfully sent gRPC to Stackdriver Logging API.'
|
633
|
+
end
|
634
|
+
|
635
|
+
rescue Google::Gax::GaxError => gax_error
|
636
|
+
# GRPC::BadStatus is wrapped in error.cause.
|
637
|
+
error = gax_error.cause
|
638
|
+
|
639
|
+
# See the mapping between HTTP status and gRPC status code at:
|
640
|
+
# https://github.com/grpc/grpc/blob/master/src/core/lib/transport/status_conversion.cc
|
641
|
+
case error
|
642
|
+
# Server error, so retry via re-raising the error.
|
643
|
+
when \
|
644
|
+
# HTTP status 500 (Internal Server Error).
|
645
|
+
GRPC::Internal,
|
646
|
+
# HTTP status 501 (Not Implemented).
|
647
|
+
GRPC::Unimplemented,
|
648
|
+
# HTTP status 503 (Service Unavailable).
|
649
|
+
GRPC::Unavailable,
|
650
|
+
# HTTP status 504 (Gateway Timeout).
|
651
|
+
GRPC::DeadlineExceeded
|
652
|
+
increment_retried_entries_count(entries_count, error.code)
|
653
|
+
@log.debug "Retrying #{entries_count} log message(s) later.",
|
654
|
+
error: error.to_s, error_code: error.code.to_s
|
655
|
+
raise error
|
656
|
+
|
657
|
+
# Most client errors indicate a problem with the request itself and
|
658
|
+
# should not be retried.
|
659
|
+
when \
|
660
|
+
# HTTP status 400 (Bad Request).
|
661
|
+
GRPC::InvalidArgument,
|
662
|
+
# HTTP status 401 (Unauthorized).
|
663
|
+
# These are usually solved via a `gcloud auth` call, or by
|
664
|
+
# modifying the permissions on the Google Cloud project.
|
665
|
+
GRPC::Unauthenticated,
|
666
|
+
# HTTP status 403 (Forbidden).
|
667
|
+
GRPC::PermissionDenied,
|
668
|
+
# HTTP status 404 (Not Found).
|
669
|
+
GRPC::NotFound,
|
670
|
+
# HTTP status 409 (Conflict).
|
671
|
+
GRPC::Aborted,
|
672
|
+
# HTTP status 412 (Precondition Failed).
|
673
|
+
GRPC::FailedPrecondition,
|
674
|
+
# HTTP status 429 (Too Many Requests).
|
675
|
+
GRPC::ResourceExhausted,
|
676
|
+
# HTTP status 499 (Client Closed Request).
|
677
|
+
GRPC::Cancelled,
|
678
|
+
# the remaining http codes in both 4xx and 5xx category.
|
679
|
+
# It's debatable whether to retry or drop these log entries.
|
680
|
+
# This decision is made to avoid retrying forever due to
|
681
|
+
# client errors.
|
682
|
+
GRPC::Unknown
|
683
|
+
increment_failed_requests_count(error.code)
|
684
|
+
increment_dropped_entries_count(entries_count, error.code)
|
685
|
+
@log.warn "Dropping #{entries_count} log message(s)",
|
686
|
+
error: error.to_s, error_code: error.code.to_s
|
687
|
+
|
688
|
+
else
|
689
|
+
# Assume it's a problem with the request itself and don't retry.
|
690
|
+
increment_failed_requests_count(error.code)
|
691
|
+
increment_dropped_entries_count(entries_count, error.code)
|
692
|
+
@log.error "Unknown response code #{error.code} from the server," \
|
693
|
+
" dropping #{entries_count} log message(s)",
|
694
|
+
error: error.to_s, error_code: error.code.to_s
|
695
|
+
end
|
696
|
+
|
697
|
+
# Got an unexpected error (not Google::Gax::GaxError) from the
|
698
|
+
# google-cloud-logging lib.
|
699
|
+
rescue StandardError => error
|
700
|
+
increment_failed_requests_count(GRPC::Core::StatusCodes::UNKNOWN)
|
701
|
+
increment_dropped_entries_count(entries_count,
|
702
|
+
GRPC::Core::StatusCodes::UNKNOWN)
|
703
|
+
@log.error "Unexpected error type #{error.class.name} from the client" \
|
704
|
+
" library, dropping #{entries_count} log message(s)",
|
705
|
+
error: error.to_s
|
706
|
+
end
|
707
|
+
|
708
|
+
def write_request_via_rest(entries:, log_name:, resource:, labels:)
|
709
|
+
client = api_client
|
710
|
+
entries_count = entries.length
|
711
|
+
client.write_entry_log_entries(
|
712
|
+
Google::Apis::LoggingV2::WriteLogEntriesRequest.new(
|
713
|
+
entries: entries,
|
714
|
+
log_name: log_name,
|
715
|
+
resource: resource,
|
716
|
+
labels: labels,
|
717
|
+
partial_success: @partial_success
|
718
|
+
),
|
719
|
+
options: { api_format_version: '2' }
|
720
|
+
)
|
721
|
+
increment_successful_requests_count
|
722
|
+
increment_ingested_entries_count(entries_count)
|
723
|
+
|
724
|
+
# Let the user explicitly know when the first call succeeded, to aid
|
725
|
+
# with verification and troubleshooting.
|
726
|
+
unless @successful_call
|
727
|
+
@successful_call = true
|
728
|
+
@log.info 'Successfully sent to Stackdriver Logging API.'
|
729
|
+
end
|
730
|
+
|
731
|
+
rescue Google::Apis::ServerError => error
|
732
|
+
# 5xx server errors. Retry via re-raising the error.
|
733
|
+
increment_retried_entries_count(entries_count, error.status_code)
|
734
|
+
@log.debug "Retrying #{entries_count} log message(s) later.",
|
735
|
+
error: error.to_s, error_code: error.status_code.to_s
|
736
|
+
raise error
|
737
|
+
|
738
|
+
rescue Google::Apis::AuthorizationError => error
|
739
|
+
# 401 authorization error.
|
740
|
+
# These are usually solved via a `gcloud auth` call, or by modifying
|
741
|
+
# the permissions on the Google Cloud project.
|
742
|
+
increment_failed_requests_count(error.status_code)
|
743
|
+
increment_dropped_entries_count(entries_count, error.status_code)
|
744
|
+
@log.warn "Dropping #{entries_count} log message(s)",
|
745
|
+
error_class: error.class.to_s, error: error.to_s
|
746
|
+
|
747
|
+
rescue Google::Apis::ClientError => error
|
748
|
+
# 4xx client errors. Most client errors indicate a problem with the
|
749
|
+
# request itself and should not be retried.
|
750
|
+
error_details_map = construct_error_details_map(error)
|
751
|
+
if error_details_map.empty?
|
752
|
+
increment_failed_requests_count(error.status_code)
|
753
|
+
increment_dropped_entries_count(entries_count, error.status_code)
|
754
|
+
@log.warn "Dropping #{entries_count} log message(s)",
|
755
|
+
error_class: error.class.to_s, error: error.to_s
|
756
|
+
else
|
757
|
+
error_details_map.each do |(error_code, error_message), indexes|
|
758
|
+
partial_error_count = indexes.length
|
759
|
+
increment_dropped_entries_count(partial_error_count, error_code)
|
760
|
+
entries_count -= partial_error_count
|
761
|
+
@log.warn "Dropping #{partial_error_count} log message(s)",
|
762
|
+
error_code: "google.rpc.Code[#{error_code}]",
|
763
|
+
error: error_message
|
764
|
+
end
|
765
|
+
# Consider partially successful requests successful.
|
766
|
+
increment_successful_requests_count
|
767
|
+
increment_ingested_entries_count(entries_count)
|
768
|
+
end
|
769
|
+
end
|
729
770
|
|
730
771
|
def parse_json_or_nil(input)
|
731
772
|
# Only here to please rubocop...
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-google-cloud
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.13.pre.memory.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Todd Derr
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2018-01-16 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -87,14 +87,14 @@ dependencies:
|
|
87
87
|
requirements:
|
88
88
|
- - "~>"
|
89
89
|
- !ruby/object:Gem::Version
|
90
|
-
version: 1.
|
90
|
+
version: '1.8'
|
91
91
|
type: :runtime
|
92
92
|
prerelease: false
|
93
93
|
version_requirements: !ruby/object:Gem::Requirement
|
94
94
|
requirements:
|
95
95
|
- - "~>"
|
96
96
|
- !ruby/object:Gem::Version
|
97
|
-
version: 1.
|
97
|
+
version: '1.8'
|
98
98
|
- !ruby/object:Gem::Dependency
|
99
99
|
name: json
|
100
100
|
requirement: !ruby/object:Gem::Requirement
|
@@ -237,12 +237,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
237
237
|
version: '2.0'
|
238
238
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
239
239
|
requirements:
|
240
|
-
- - "
|
240
|
+
- - ">"
|
241
241
|
- !ruby/object:Gem::Version
|
242
|
-
version:
|
242
|
+
version: 1.3.1
|
243
243
|
requirements: []
|
244
244
|
rubyforge_project:
|
245
|
-
rubygems_version: 2.
|
245
|
+
rubygems_version: 2.4.8
|
246
246
|
signing_key:
|
247
247
|
specification_version: 4
|
248
248
|
summary: fluentd output plugin for the Stackdriver Logging API
|