fluent-plugin-google-cloud 0.6.12 → 0.6.13.pre.memory.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +18 -17
- data/fluent-plugin-google-cloud.gemspec +2 -2
- data/lib/fluent/plugin/out_google_cloud.rb +232 -191
- metadata +7 -7
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b26e153a8191c18cf52ed1efdc552d5323045683
|
|
4
|
+
data.tar.gz: 3111b02b15f0a930f7cbd0d28b5517ffbe56a0ab
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4e569bbf17acbb4c997c3321bee3d7bac9cdbaa21c5e8b6d6f40bf996906ac3cc41f4ae7e7ae2f8085251369cefa5a0be4261d3529c294a305c7e0bb4aba2d1c
|
|
7
|
+
data.tar.gz: 28562ed528b840677aca44c5f5982740e0e936a1eb9bb15ed8ca35d36f821fbc0931e8888ffe53a99cc4c73e622b10dbda51b8146c8ec2f1ecec97b6b671d042
|
data/Gemfile.lock
CHANGED
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
fluent-plugin-google-cloud (0.6.
|
|
4
|
+
fluent-plugin-google-cloud (0.6.13.pre.memory.1)
|
|
5
5
|
fluentd (~> 0.10)
|
|
6
6
|
google-api-client (~> 0.14)
|
|
7
7
|
google-cloud-logging (~> 1.2.3)
|
|
8
8
|
googleapis-common-protos (~> 1.3)
|
|
9
9
|
googleauth (~> 0.5)
|
|
10
|
-
grpc (~> 1.
|
|
10
|
+
grpc (~> 1.8)
|
|
11
11
|
json (~> 1.8)
|
|
12
12
|
|
|
13
13
|
GEM
|
|
14
14
|
remote: https://rubygems.org/
|
|
15
15
|
specs:
|
|
16
|
-
addressable (2.5.
|
|
17
|
-
public_suffix (
|
|
16
|
+
addressable (2.5.2)
|
|
17
|
+
public_suffix (>= 2.0.2, < 4.0)
|
|
18
18
|
ast (2.3.0)
|
|
19
19
|
cool.io (1.5.3)
|
|
20
20
|
crack (0.4.3)
|
|
@@ -34,7 +34,7 @@ GEM
|
|
|
34
34
|
tzinfo (~> 1.0)
|
|
35
35
|
tzinfo-data (~> 1.0)
|
|
36
36
|
yajl-ruby (~> 1.0)
|
|
37
|
-
google-api-client (0.
|
|
37
|
+
google-api-client (0.19.3)
|
|
38
38
|
addressable (~> 2.5, >= 2.5.1)
|
|
39
39
|
googleauth (>= 0.5, < 0.7.0)
|
|
40
40
|
httpclient (>= 2.8.1, < 3.0)
|
|
@@ -55,7 +55,7 @@ GEM
|
|
|
55
55
|
googleauth (~> 0.5.1)
|
|
56
56
|
grpc (~> 1.0)
|
|
57
57
|
rly (~> 0.2.3)
|
|
58
|
-
google-protobuf (3.5.
|
|
58
|
+
google-protobuf (3.5.1)
|
|
59
59
|
googleapis-common-protos (1.3.7)
|
|
60
60
|
google-protobuf (~> 3.0)
|
|
61
61
|
googleapis-common-protos-types (~> 1.0)
|
|
@@ -70,10 +70,11 @@ GEM
|
|
|
70
70
|
multi_json (~> 1.11)
|
|
71
71
|
os (~> 0.9)
|
|
72
72
|
signet (~> 0.7)
|
|
73
|
-
grpc (1.
|
|
73
|
+
grpc (1.8.3)
|
|
74
74
|
google-protobuf (~> 3.1)
|
|
75
|
-
|
|
76
|
-
|
|
75
|
+
googleapis-common-protos-types (~> 1.0.0)
|
|
76
|
+
googleauth (>= 0.5.1, < 0.7)
|
|
77
|
+
hashdiff (0.3.7)
|
|
77
78
|
http_parser.rb (0.6.0)
|
|
78
79
|
httpclient (2.8.3)
|
|
79
80
|
json (1.8.6)
|
|
@@ -87,19 +88,19 @@ GEM
|
|
|
87
88
|
mime-types (3.1)
|
|
88
89
|
mime-types-data (~> 3.2015)
|
|
89
90
|
mime-types-data (3.2016.0521)
|
|
90
|
-
mocha (1.
|
|
91
|
+
mocha (1.3.0)
|
|
91
92
|
metaclass (~> 0.0.1)
|
|
92
|
-
msgpack (1.2.
|
|
93
|
-
multi_json (1.
|
|
93
|
+
msgpack (1.2.2)
|
|
94
|
+
multi_json (1.13.1)
|
|
94
95
|
multipart-post (2.0.0)
|
|
95
96
|
os (0.9.6)
|
|
96
97
|
parser (2.4.0.2)
|
|
97
98
|
ast (~> 2.3)
|
|
98
|
-
power_assert (1.
|
|
99
|
+
power_assert (1.1.1)
|
|
99
100
|
powerpack (0.1.1)
|
|
100
101
|
prometheus-client (0.7.1)
|
|
101
102
|
quantile (~> 0.2.0)
|
|
102
|
-
public_suffix (
|
|
103
|
+
public_suffix (3.0.1)
|
|
103
104
|
quantile (0.2.0)
|
|
104
105
|
rainbow (2.2.2)
|
|
105
106
|
rake
|
|
@@ -119,7 +120,7 @@ GEM
|
|
|
119
120
|
ruby-progressbar (1.9.0)
|
|
120
121
|
ruby_dig (0.0.2)
|
|
121
122
|
safe_yaml (1.0.4)
|
|
122
|
-
serverengine (2.0.
|
|
123
|
+
serverengine (2.0.6)
|
|
123
124
|
sigdump (~> 0.2.2)
|
|
124
125
|
sigdump (0.2.4)
|
|
125
126
|
signet (0.8.1)
|
|
@@ -129,7 +130,7 @@ GEM
|
|
|
129
130
|
multi_json (~> 1.10)
|
|
130
131
|
stackdriver-core (1.2.0)
|
|
131
132
|
strptime (0.1.9)
|
|
132
|
-
test-unit (3.2.
|
|
133
|
+
test-unit (3.2.7)
|
|
133
134
|
power_assert
|
|
134
135
|
thread_safe (0.3.6)
|
|
135
136
|
tzinfo (1.2.4)
|
|
@@ -157,4 +158,4 @@ DEPENDENCIES
|
|
|
157
158
|
webmock (~> 2.3.1)
|
|
158
159
|
|
|
159
160
|
BUNDLED WITH
|
|
160
|
-
1.16.
|
|
161
|
+
1.16.1
|
|
@@ -10,7 +10,7 @@ eos
|
|
|
10
10
|
gem.homepage =
|
|
11
11
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
|
12
12
|
gem.license = 'Apache-2.0'
|
|
13
|
-
gem.version = '0.6.
|
|
13
|
+
gem.version = '0.6.13.pre.memory.1'
|
|
14
14
|
gem.authors = ['Todd Derr', 'Alex Robinson']
|
|
15
15
|
gem.email = ['salty@google.com']
|
|
16
16
|
gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
|
|
@@ -24,7 +24,7 @@ eos
|
|
|
24
24
|
gem.add_runtime_dependency 'google-api-client', '~> 0.14'
|
|
25
25
|
gem.add_runtime_dependency 'google-cloud-logging', '~> 1.2.3'
|
|
26
26
|
gem.add_runtime_dependency 'googleauth', '~> 0.5'
|
|
27
|
-
gem.add_runtime_dependency 'grpc', '~> 1.
|
|
27
|
+
gem.add_runtime_dependency 'grpc', '~> 1.8'
|
|
28
28
|
gem.add_runtime_dependency 'json', '~> 1.8'
|
|
29
29
|
|
|
30
30
|
gem.add_development_dependency 'mocha', '~> 1.1'
|
|
@@ -195,7 +195,7 @@ module Fluent
|
|
|
195
195
|
Fluent::Plugin.register_output('google_cloud', self)
|
|
196
196
|
|
|
197
197
|
PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'.freeze
|
|
198
|
-
PLUGIN_VERSION = '0.6.
|
|
198
|
+
PLUGIN_VERSION = '0.6.13.pre.memory.1'.freeze
|
|
199
199
|
|
|
200
200
|
# Name of the the Google cloud logging write scope.
|
|
201
201
|
LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'.freeze
|
|
@@ -295,13 +295,13 @@ module Fluent
|
|
|
295
295
|
config_param :labels, :hash, :default => nil
|
|
296
296
|
|
|
297
297
|
# Whether to use gRPC instead of REST/JSON to communicate to the
|
|
298
|
-
#
|
|
298
|
+
# Stackdriver Logging API.
|
|
299
299
|
config_param :use_grpc, :bool, :default => false
|
|
300
300
|
|
|
301
301
|
# Whether valid entries should be written even if some other entries fail
|
|
302
302
|
# due to INVALID_ARGUMENT or PERMISSION_DENIED errors when communicating to
|
|
303
|
-
# the
|
|
304
|
-
# works with the REST path (use_grpc = false).
|
|
303
|
+
# the Stackdriver Logging API. This is highly recommended. Right now this
|
|
304
|
+
# only works with the REST path (use_grpc = false).
|
|
305
305
|
config_param :partial_success, :bool, :default => false
|
|
306
306
|
|
|
307
307
|
# Whether to allow non-UTF-8 characters in user logs. If set to true, any
|
|
@@ -453,6 +453,14 @@ module Fluent
|
|
|
453
453
|
@resource.labels.freeze
|
|
454
454
|
@common_labels.freeze
|
|
455
455
|
|
|
456
|
+
if @use_grpc
|
|
457
|
+
@construct_log_entry = method(:construct_log_entry_in_grpc_format)
|
|
458
|
+
@write_request = method(:write_request_via_grpc)
|
|
459
|
+
else
|
|
460
|
+
@construct_log_entry = method(:construct_log_entry_in_rest_format)
|
|
461
|
+
@write_request = method(:write_request_via_rest)
|
|
462
|
+
end
|
|
463
|
+
|
|
456
464
|
# Log an informational message containing the Logs viewer URL
|
|
457
465
|
@log.info 'Logs viewer address: https://console.cloud.google.com/logs/',
|
|
458
466
|
"viewer?project=#{@project_id}&resource=#{@resource.type}/",
|
|
@@ -473,6 +481,7 @@ module Fluent
|
|
|
473
481
|
def write(chunk)
|
|
474
482
|
grouped_entries = group_log_entries_by_tag_and_local_resource_id(chunk)
|
|
475
483
|
|
|
484
|
+
requests_to_send = []
|
|
476
485
|
grouped_entries.each do |(tag, local_resource_id), arr|
|
|
477
486
|
entries = []
|
|
478
487
|
group_level_resource, group_level_common_labels =
|
|
@@ -517,40 +526,11 @@ module Fluent
|
|
|
517
526
|
severity = compute_severity(
|
|
518
527
|
entry_level_resource.type, record, entry_level_common_labels)
|
|
519
528
|
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
labels: entry_level_resource.labels.to_h
|
|
526
|
-
),
|
|
527
|
-
severity: grpc_severity(severity)
|
|
528
|
-
)
|
|
529
|
-
# If "seconds" is null or not an integer, we will omit the timestamp
|
|
530
|
-
# field and defer the decision on how to handle it to the downstream
|
|
531
|
-
# Logging API. If "nanos" is null or not an integer, it will be set
|
|
532
|
-
# to 0.
|
|
533
|
-
if ts_secs.is_a?(Integer)
|
|
534
|
-
ts_nanos = 0 unless ts_nanos.is_a?(Integer)
|
|
535
|
-
entry.timestamp = Google::Protobuf::Timestamp.new(
|
|
536
|
-
seconds: ts_secs,
|
|
537
|
-
nanos: ts_nanos
|
|
538
|
-
)
|
|
539
|
-
end
|
|
540
|
-
else
|
|
541
|
-
# Remove the labels if we didn't populate them with anything.
|
|
542
|
-
entry_level_resource.labels = nil if
|
|
543
|
-
entry_level_resource.labels.empty?
|
|
544
|
-
entry = Google::Apis::LoggingV2::LogEntry.new(
|
|
545
|
-
labels: entry_level_common_labels,
|
|
546
|
-
resource: entry_level_resource,
|
|
547
|
-
severity: severity,
|
|
548
|
-
timestamp: {
|
|
549
|
-
seconds: ts_secs,
|
|
550
|
-
nanos: ts_nanos
|
|
551
|
-
}
|
|
552
|
-
)
|
|
553
|
-
end
|
|
529
|
+
entry = @construct_log_entry.call(entry_level_common_labels,
|
|
530
|
+
entry_level_resource,
|
|
531
|
+
severity,
|
|
532
|
+
ts_secs,
|
|
533
|
+
ts_nanos)
|
|
554
534
|
|
|
555
535
|
# Get fully-qualified trace id for LogEntry "trace" field.
|
|
556
536
|
fq_trace_id = record.delete(@trace_key)
|
|
@@ -567,165 +547,226 @@ module Fluent
|
|
|
567
547
|
log_name = "projects/#{@project_id}/logs/#{log_name(
|
|
568
548
|
tag, group_level_resource)}"
|
|
569
549
|
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
550
|
+
requests_to_send << {
|
|
551
|
+
entries: entries,
|
|
552
|
+
log_name: log_name,
|
|
553
|
+
resource: group_level_resource,
|
|
554
|
+
labels: group_level_common_labels
|
|
555
|
+
}
|
|
556
|
+
end
|
|
577
557
|
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
resource: Google::Api::MonitoredResource.new(
|
|
583
|
-
type: group_level_resource.type,
|
|
584
|
-
labels: group_level_resource.labels.to_h
|
|
585
|
-
),
|
|
586
|
-
labels: labels_utf8_pairs.to_h
|
|
587
|
-
)
|
|
588
|
-
increment_successful_requests_count
|
|
589
|
-
increment_ingested_entries_count(entries_count)
|
|
590
|
-
|
|
591
|
-
# Let the user explicitly know when the first call succeeded, to aid
|
|
592
|
-
# with verification and troubleshooting.
|
|
593
|
-
unless @successful_call
|
|
594
|
-
@successful_call = true
|
|
595
|
-
@log.info 'Successfully sent gRPC to Stackdriver Logging API.'
|
|
596
|
-
end
|
|
558
|
+
requests_to_send.each do |request|
|
|
559
|
+
@write_request.call(request)
|
|
560
|
+
end
|
|
561
|
+
end
|
|
597
562
|
|
|
598
|
-
|
|
599
|
-
# GRPC::BadStatus is wrapped in error.cause.
|
|
600
|
-
error = gax_error.cause
|
|
601
|
-
|
|
602
|
-
# See the mapping between HTTP status and gRPC status code at:
|
|
603
|
-
# https://github.com/grpc/grpc/blob/master/src/core/lib/transport/status_conversion.cc
|
|
604
|
-
case error
|
|
605
|
-
# Server error, so retry via re-raising the error.
|
|
606
|
-
when \
|
|
607
|
-
# HTTP status 500 (Internal Server Error).
|
|
608
|
-
GRPC::Internal,
|
|
609
|
-
# HTTP status 501 (Not Implemented).
|
|
610
|
-
GRPC::Unimplemented,
|
|
611
|
-
# HTTP status 503 (Service Unavailable).
|
|
612
|
-
GRPC::Unavailable,
|
|
613
|
-
# HTTP status 504 (Gateway Timeout).
|
|
614
|
-
GRPC::DeadlineExceeded
|
|
615
|
-
increment_retried_entries_count(entries_count, error.code)
|
|
616
|
-
@log.debug "Retrying #{entries_count} log message(s) later.",
|
|
617
|
-
error: error.to_s, error_code: error.code.to_s
|
|
618
|
-
raise error
|
|
619
|
-
|
|
620
|
-
# Most client errors indicate a problem with the request itself and
|
|
621
|
-
# should not be retried.
|
|
622
|
-
when \
|
|
623
|
-
# HTTP status 400 (Bad Request).
|
|
624
|
-
GRPC::InvalidArgument,
|
|
625
|
-
# HTTP status 401 (Unauthorized).
|
|
626
|
-
# These are usually solved via a `gcloud auth` call, or by
|
|
627
|
-
# modifying the permissions on the Google Cloud project.
|
|
628
|
-
GRPC::Unauthenticated,
|
|
629
|
-
# HTTP status 403 (Forbidden).
|
|
630
|
-
GRPC::PermissionDenied,
|
|
631
|
-
# HTTP status 404 (Not Found).
|
|
632
|
-
GRPC::NotFound,
|
|
633
|
-
# HTTP status 409 (Conflict).
|
|
634
|
-
GRPC::Aborted,
|
|
635
|
-
# HTTP status 412 (Precondition Failed).
|
|
636
|
-
GRPC::FailedPrecondition,
|
|
637
|
-
# HTTP status 429 (Too Many Requests).
|
|
638
|
-
GRPC::ResourceExhausted,
|
|
639
|
-
# HTTP status 499 (Client Closed Request).
|
|
640
|
-
GRPC::Cancelled,
|
|
641
|
-
# the remaining http codes in both 4xx and 5xx category.
|
|
642
|
-
# It's debatable whether to retry or drop these log entries.
|
|
643
|
-
# This decision is made to avoid retrying forever due to client
|
|
644
|
-
# errors.
|
|
645
|
-
GRPC::Unknown
|
|
646
|
-
increment_failed_requests_count(error.code)
|
|
647
|
-
increment_dropped_entries_count(entries_count, error.code)
|
|
648
|
-
@log.warn "Dropping #{entries_count} log message(s)",
|
|
649
|
-
error: error.to_s, error_code: error.code.to_s
|
|
650
|
-
|
|
651
|
-
else
|
|
652
|
-
# Assume it is a problem with the request itself and don't retry.
|
|
653
|
-
increment_failed_requests_count(error.code)
|
|
654
|
-
increment_dropped_entries_count(entries_count, error.code)
|
|
655
|
-
@log.error "Unknown response code #{error.code} from the "\
|
|
656
|
-
"server, dropping #{entries_count} log message(s)",
|
|
657
|
-
error: error.to_s, error_code: error.code.to_s
|
|
658
|
-
end
|
|
659
|
-
end
|
|
660
|
-
else
|
|
661
|
-
begin
|
|
662
|
-
write_request =
|
|
663
|
-
Google::Apis::LoggingV2::WriteLogEntriesRequest.new(
|
|
664
|
-
log_name: log_name,
|
|
665
|
-
resource: group_level_resource,
|
|
666
|
-
labels: group_level_common_labels,
|
|
667
|
-
partial_success: @partial_success,
|
|
668
|
-
entries: entries)
|
|
669
|
-
entries_count = entries.length
|
|
670
|
-
client.write_entry_log_entries(
|
|
671
|
-
write_request,
|
|
672
|
-
options: { api_format_version: '2' }
|
|
673
|
-
)
|
|
674
|
-
|
|
675
|
-
increment_successful_requests_count
|
|
676
|
-
increment_ingested_entries_count(entries_count)
|
|
677
|
-
|
|
678
|
-
# Let the user explicitly know when the first call succeeded, to aid
|
|
679
|
-
# with verification and troubleshooting.
|
|
680
|
-
unless @successful_call
|
|
681
|
-
@successful_call = true
|
|
682
|
-
@log.info 'Successfully sent to Stackdriver Logging API.'
|
|
683
|
-
end
|
|
563
|
+
private
|
|
684
564
|
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
@log.warn "Dropping #{entries_count} log message(s)",
|
|
709
|
-
error_class: error.class.to_s, error: error.to_s
|
|
710
|
-
else
|
|
711
|
-
error_details_map.each do |(error_code, error_message), indexes|
|
|
712
|
-
partial_error_count = indexes.length
|
|
713
|
-
increment_dropped_entries_count(partial_error_count, error_code)
|
|
714
|
-
entries_count -= partial_error_count
|
|
715
|
-
@log.warn "Dropping #{partial_error_count} log message(s)",
|
|
716
|
-
error_code: "google.rpc.Code[#{error_code}]",
|
|
717
|
-
error: error_message
|
|
718
|
-
end
|
|
719
|
-
# Consider partially successful requests successful.
|
|
720
|
-
increment_successful_requests_count
|
|
721
|
-
increment_ingested_entries_count(entries_count)
|
|
722
|
-
end
|
|
723
|
-
end
|
|
724
|
-
end
|
|
565
|
+
def construct_log_entry_in_grpc_format(labels,
|
|
566
|
+
resource,
|
|
567
|
+
severity,
|
|
568
|
+
ts_secs,
|
|
569
|
+
ts_nanos)
|
|
570
|
+
entry = Google::Logging::V2::LogEntry.new(
|
|
571
|
+
labels: labels,
|
|
572
|
+
resource: Google::Api::MonitoredResource.new(
|
|
573
|
+
type: resource.type,
|
|
574
|
+
labels: resource.labels.to_h
|
|
575
|
+
),
|
|
576
|
+
severity: grpc_severity(severity)
|
|
577
|
+
)
|
|
578
|
+
# If "seconds" is null or not an integer, we will omit the timestamp
|
|
579
|
+
# field and defer the decision on how to handle it to the downstream
|
|
580
|
+
# Logging API. If "nanos" is null or not an integer, it will be set
|
|
581
|
+
# to 0.
|
|
582
|
+
if ts_secs.is_a?(Integer)
|
|
583
|
+
ts_nanos = 0 unless ts_nanos.is_a?(Integer)
|
|
584
|
+
entry.timestamp = Google::Protobuf::Timestamp.new(
|
|
585
|
+
seconds: ts_secs,
|
|
586
|
+
nanos: ts_nanos
|
|
587
|
+
)
|
|
725
588
|
end
|
|
589
|
+
entry
|
|
726
590
|
end
|
|
727
591
|
|
|
728
|
-
|
|
592
|
+
def construct_log_entry_in_rest_format(labels,
|
|
593
|
+
resource,
|
|
594
|
+
severity,
|
|
595
|
+
ts_secs,
|
|
596
|
+
ts_nanos)
|
|
597
|
+
# Remove the labels if we didn't populate them with anything.
|
|
598
|
+
resource.labels = nil if resource.labels.empty?
|
|
599
|
+
Google::Apis::LoggingV2::LogEntry.new(
|
|
600
|
+
labels: labels,
|
|
601
|
+
resource: resource,
|
|
602
|
+
severity: severity,
|
|
603
|
+
timestamp: {
|
|
604
|
+
seconds: ts_secs,
|
|
605
|
+
nanos: ts_nanos
|
|
606
|
+
}
|
|
607
|
+
)
|
|
608
|
+
end
|
|
609
|
+
|
|
610
|
+
def write_request_via_grpc(entries:, log_name:, resource:, labels:)
|
|
611
|
+
client = api_client
|
|
612
|
+
entries_count = entries.length
|
|
613
|
+
client.write_log_entries(
|
|
614
|
+
# Ignore partial_success for gRPC path.
|
|
615
|
+
entries,
|
|
616
|
+
log_name: log_name,
|
|
617
|
+
resource: Google::Api::MonitoredResource.new(
|
|
618
|
+
type: resource.type,
|
|
619
|
+
labels: resource.labels.to_h
|
|
620
|
+
),
|
|
621
|
+
labels: labels.map do |k, v|
|
|
622
|
+
[k.encode('utf-8'), convert_to_utf8(v)]
|
|
623
|
+
end.to_h
|
|
624
|
+
)
|
|
625
|
+
increment_successful_requests_count
|
|
626
|
+
increment_ingested_entries_count(entries_count)
|
|
627
|
+
|
|
628
|
+
# Let the user explicitly know when the first call succeeded, to
|
|
629
|
+
# aid with verification and troubleshooting.
|
|
630
|
+
unless @successful_call
|
|
631
|
+
@successful_call = true
|
|
632
|
+
@log.info 'Successfully sent gRPC to Stackdriver Logging API.'
|
|
633
|
+
end
|
|
634
|
+
|
|
635
|
+
rescue Google::Gax::GaxError => gax_error
|
|
636
|
+
# GRPC::BadStatus is wrapped in error.cause.
|
|
637
|
+
error = gax_error.cause
|
|
638
|
+
|
|
639
|
+
# See the mapping between HTTP status and gRPC status code at:
|
|
640
|
+
# https://github.com/grpc/grpc/blob/master/src/core/lib/transport/status_conversion.cc
|
|
641
|
+
case error
|
|
642
|
+
# Server error, so retry via re-raising the error.
|
|
643
|
+
when \
|
|
644
|
+
# HTTP status 500 (Internal Server Error).
|
|
645
|
+
GRPC::Internal,
|
|
646
|
+
# HTTP status 501 (Not Implemented).
|
|
647
|
+
GRPC::Unimplemented,
|
|
648
|
+
# HTTP status 503 (Service Unavailable).
|
|
649
|
+
GRPC::Unavailable,
|
|
650
|
+
# HTTP status 504 (Gateway Timeout).
|
|
651
|
+
GRPC::DeadlineExceeded
|
|
652
|
+
increment_retried_entries_count(entries_count, error.code)
|
|
653
|
+
@log.debug "Retrying #{entries_count} log message(s) later.",
|
|
654
|
+
error: error.to_s, error_code: error.code.to_s
|
|
655
|
+
raise error
|
|
656
|
+
|
|
657
|
+
# Most client errors indicate a problem with the request itself and
|
|
658
|
+
# should not be retried.
|
|
659
|
+
when \
|
|
660
|
+
# HTTP status 400 (Bad Request).
|
|
661
|
+
GRPC::InvalidArgument,
|
|
662
|
+
# HTTP status 401 (Unauthorized).
|
|
663
|
+
# These are usually solved via a `gcloud auth` call, or by
|
|
664
|
+
# modifying the permissions on the Google Cloud project.
|
|
665
|
+
GRPC::Unauthenticated,
|
|
666
|
+
# HTTP status 403 (Forbidden).
|
|
667
|
+
GRPC::PermissionDenied,
|
|
668
|
+
# HTTP status 404 (Not Found).
|
|
669
|
+
GRPC::NotFound,
|
|
670
|
+
# HTTP status 409 (Conflict).
|
|
671
|
+
GRPC::Aborted,
|
|
672
|
+
# HTTP status 412 (Precondition Failed).
|
|
673
|
+
GRPC::FailedPrecondition,
|
|
674
|
+
# HTTP status 429 (Too Many Requests).
|
|
675
|
+
GRPC::ResourceExhausted,
|
|
676
|
+
# HTTP status 499 (Client Closed Request).
|
|
677
|
+
GRPC::Cancelled,
|
|
678
|
+
# the remaining http codes in both 4xx and 5xx category.
|
|
679
|
+
# It's debatable whether to retry or drop these log entries.
|
|
680
|
+
# This decision is made to avoid retrying forever due to
|
|
681
|
+
# client errors.
|
|
682
|
+
GRPC::Unknown
|
|
683
|
+
increment_failed_requests_count(error.code)
|
|
684
|
+
increment_dropped_entries_count(entries_count, error.code)
|
|
685
|
+
@log.warn "Dropping #{entries_count} log message(s)",
|
|
686
|
+
error: error.to_s, error_code: error.code.to_s
|
|
687
|
+
|
|
688
|
+
else
|
|
689
|
+
# Assume it's a problem with the request itself and don't retry.
|
|
690
|
+
increment_failed_requests_count(error.code)
|
|
691
|
+
increment_dropped_entries_count(entries_count, error.code)
|
|
692
|
+
@log.error "Unknown response code #{error.code} from the server," \
|
|
693
|
+
" dropping #{entries_count} log message(s)",
|
|
694
|
+
error: error.to_s, error_code: error.code.to_s
|
|
695
|
+
end
|
|
696
|
+
|
|
697
|
+
# Got an unexpected error (not Google::Gax::GaxError) from the
|
|
698
|
+
# google-cloud-logging lib.
|
|
699
|
+
rescue StandardError => error
|
|
700
|
+
increment_failed_requests_count(GRPC::Core::StatusCodes::UNKNOWN)
|
|
701
|
+
increment_dropped_entries_count(entries_count,
|
|
702
|
+
GRPC::Core::StatusCodes::UNKNOWN)
|
|
703
|
+
@log.error "Unexpected error type #{error.class.name} from the client" \
|
|
704
|
+
" library, dropping #{entries_count} log message(s)",
|
|
705
|
+
error: error.to_s
|
|
706
|
+
end
|
|
707
|
+
|
|
708
|
+
def write_request_via_rest(entries:, log_name:, resource:, labels:)
|
|
709
|
+
client = api_client
|
|
710
|
+
entries_count = entries.length
|
|
711
|
+
client.write_entry_log_entries(
|
|
712
|
+
Google::Apis::LoggingV2::WriteLogEntriesRequest.new(
|
|
713
|
+
entries: entries,
|
|
714
|
+
log_name: log_name,
|
|
715
|
+
resource: resource,
|
|
716
|
+
labels: labels,
|
|
717
|
+
partial_success: @partial_success
|
|
718
|
+
),
|
|
719
|
+
options: { api_format_version: '2' }
|
|
720
|
+
)
|
|
721
|
+
increment_successful_requests_count
|
|
722
|
+
increment_ingested_entries_count(entries_count)
|
|
723
|
+
|
|
724
|
+
# Let the user explicitly know when the first call succeeded, to aid
|
|
725
|
+
# with verification and troubleshooting.
|
|
726
|
+
unless @successful_call
|
|
727
|
+
@successful_call = true
|
|
728
|
+
@log.info 'Successfully sent to Stackdriver Logging API.'
|
|
729
|
+
end
|
|
730
|
+
|
|
731
|
+
rescue Google::Apis::ServerError => error
|
|
732
|
+
# 5xx server errors. Retry via re-raising the error.
|
|
733
|
+
increment_retried_entries_count(entries_count, error.status_code)
|
|
734
|
+
@log.debug "Retrying #{entries_count} log message(s) later.",
|
|
735
|
+
error: error.to_s, error_code: error.status_code.to_s
|
|
736
|
+
raise error
|
|
737
|
+
|
|
738
|
+
rescue Google::Apis::AuthorizationError => error
|
|
739
|
+
# 401 authorization error.
|
|
740
|
+
# These are usually solved via a `gcloud auth` call, or by modifying
|
|
741
|
+
# the permissions on the Google Cloud project.
|
|
742
|
+
increment_failed_requests_count(error.status_code)
|
|
743
|
+
increment_dropped_entries_count(entries_count, error.status_code)
|
|
744
|
+
@log.warn "Dropping #{entries_count} log message(s)",
|
|
745
|
+
error_class: error.class.to_s, error: error.to_s
|
|
746
|
+
|
|
747
|
+
rescue Google::Apis::ClientError => error
|
|
748
|
+
# 4xx client errors. Most client errors indicate a problem with the
|
|
749
|
+
# request itself and should not be retried.
|
|
750
|
+
error_details_map = construct_error_details_map(error)
|
|
751
|
+
if error_details_map.empty?
|
|
752
|
+
increment_failed_requests_count(error.status_code)
|
|
753
|
+
increment_dropped_entries_count(entries_count, error.status_code)
|
|
754
|
+
@log.warn "Dropping #{entries_count} log message(s)",
|
|
755
|
+
error_class: error.class.to_s, error: error.to_s
|
|
756
|
+
else
|
|
757
|
+
error_details_map.each do |(error_code, error_message), indexes|
|
|
758
|
+
partial_error_count = indexes.length
|
|
759
|
+
increment_dropped_entries_count(partial_error_count, error_code)
|
|
760
|
+
entries_count -= partial_error_count
|
|
761
|
+
@log.warn "Dropping #{partial_error_count} log message(s)",
|
|
762
|
+
error_code: "google.rpc.Code[#{error_code}]",
|
|
763
|
+
error: error_message
|
|
764
|
+
end
|
|
765
|
+
# Consider partially successful requests successful.
|
|
766
|
+
increment_successful_requests_count
|
|
767
|
+
increment_ingested_entries_count(entries_count)
|
|
768
|
+
end
|
|
769
|
+
end
|
|
729
770
|
|
|
730
771
|
def parse_json_or_nil(input)
|
|
731
772
|
# Only here to please rubocop...
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-google-cloud
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.6.
|
|
4
|
+
version: 0.6.13.pre.memory.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Todd Derr
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date:
|
|
12
|
+
date: 2018-01-16 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: fluentd
|
|
@@ -87,14 +87,14 @@ dependencies:
|
|
|
87
87
|
requirements:
|
|
88
88
|
- - "~>"
|
|
89
89
|
- !ruby/object:Gem::Version
|
|
90
|
-
version: 1.
|
|
90
|
+
version: '1.8'
|
|
91
91
|
type: :runtime
|
|
92
92
|
prerelease: false
|
|
93
93
|
version_requirements: !ruby/object:Gem::Requirement
|
|
94
94
|
requirements:
|
|
95
95
|
- - "~>"
|
|
96
96
|
- !ruby/object:Gem::Version
|
|
97
|
-
version: 1.
|
|
97
|
+
version: '1.8'
|
|
98
98
|
- !ruby/object:Gem::Dependency
|
|
99
99
|
name: json
|
|
100
100
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -237,12 +237,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
237
237
|
version: '2.0'
|
|
238
238
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
239
239
|
requirements:
|
|
240
|
-
- - "
|
|
240
|
+
- - ">"
|
|
241
241
|
- !ruby/object:Gem::Version
|
|
242
|
-
version:
|
|
242
|
+
version: 1.3.1
|
|
243
243
|
requirements: []
|
|
244
244
|
rubyforge_project:
|
|
245
|
-
rubygems_version: 2.
|
|
245
|
+
rubygems_version: 2.4.8
|
|
246
246
|
signing_key:
|
|
247
247
|
specification_version: 4
|
|
248
248
|
summary: fluentd output plugin for the Stackdriver Logging API
|