fluent-plugin-google-cloud 0.6.25.pre.2 → 0.6.25.pre.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +3 -3
- data/fluent-plugin-google-cloud.gemspec +1 -1
- data/lib/fluent/plugin/filter_add_insert_ids.rb +68 -70
- data/test/plugin/test_filter_add_insert_ids.rb +2 -2
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 219e76eaed4425f0075746b4f95fbe5673970281
|
|
4
|
+
data.tar.gz: 5c5f0c9e4dc6da1b58ac3252a05c891f0a40c9f0
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 223aefacf4ab26c6b6d0c1df418ba12e6d540b592c669d68a8c1fbfb6dcba12f50a7a9574e1e2a53f27760cd2bfecbfa1ff2a603be9223f0e9848da4c250c6d6
|
|
7
|
+
data.tar.gz: e6d95ce23a5eee04730cfeedc64f983290143dd093405c7cc68afdf08ff0984130744d00a948e7fd95eab8632b56b80616cd2b3405bcd4526ea93f10f51ef43b
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
fluent-plugin-google-cloud (0.6.25.pre.
|
|
4
|
+
fluent-plugin-google-cloud (0.6.25.pre.3)
|
|
5
5
|
fluentd (~> 0.10)
|
|
6
6
|
google-api-client (~> 0.17)
|
|
7
7
|
google-cloud-logging (~> 1.3, >= 1.3.2)
|
|
@@ -58,7 +58,7 @@ GEM
|
|
|
58
58
|
googleauth (~> 0.6.2)
|
|
59
59
|
grpc (>= 1.7.2, < 2.0)
|
|
60
60
|
rly (~> 0.2.3)
|
|
61
|
-
google-protobuf (3.6.1)
|
|
61
|
+
google-protobuf (3.6.1-x86_64-linux)
|
|
62
62
|
googleapis-common-protos (1.3.7)
|
|
63
63
|
google-protobuf (~> 3.0)
|
|
64
64
|
googleapis-common-protos-types (~> 1.0)
|
|
@@ -72,7 +72,7 @@ GEM
|
|
|
72
72
|
multi_json (~> 1.11)
|
|
73
73
|
os (>= 0.9, < 2.0)
|
|
74
74
|
signet (~> 0.7)
|
|
75
|
-
grpc (1.15.0)
|
|
75
|
+
grpc (1.15.0-x86_64-linux)
|
|
76
76
|
google-protobuf (~> 3.1)
|
|
77
77
|
googleapis-common-protos-types (~> 1.0.0)
|
|
78
78
|
hashdiff (0.3.7)
|
|
@@ -10,7 +10,7 @@ eos
|
|
|
10
10
|
gem.homepage =
|
|
11
11
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
|
12
12
|
gem.license = 'Apache-2.0'
|
|
13
|
-
gem.version = '0.6.25.pre.
|
|
13
|
+
gem.version = '0.6.25.pre.3'
|
|
14
14
|
gem.authors = ['Stackdriver Agents Team']
|
|
15
15
|
gem.email = ['stackdriver-agents@google.com']
|
|
16
16
|
gem.required_ruby_version = Gem::Requirement.new('>= 2.2')
|
|
@@ -15,89 +15,87 @@
|
|
|
15
15
|
require 'fluent/plugin/filter'
|
|
16
16
|
|
|
17
17
|
module Fluent
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
Fluent::Plugin.register_filter('add_insert_ids', self)
|
|
18
|
+
# Fluentd filter plugin for adding insertIds to guarantee log entry order
|
|
19
|
+
# and uniqueness.
|
|
20
|
+
# Sample log entries enriched by this plugin:
|
|
21
|
+
# {
|
|
22
|
+
# "timestamp": "2017-08-22 13:35:28",
|
|
23
|
+
# "message": "1",
|
|
24
|
+
# "logging.googleapis.com/insertId": "aye7eakuf23h41aef0"
|
|
25
|
+
# }
|
|
26
|
+
# {
|
|
27
|
+
# "timestamp": "2017-08-22 13:35:28",
|
|
28
|
+
# "message": "2",
|
|
29
|
+
# "logging.googleapis.com/insertId": "aye7eakuf23h41aef1"
|
|
30
|
+
# }
|
|
31
|
+
# {
|
|
32
|
+
# "timestamp": "2017-08-22 13:35:28",
|
|
33
|
+
# "message": "3",
|
|
34
|
+
# "logging.googleapis.com/insertId": "aye7eakuf23h41aef2"
|
|
35
|
+
# }
|
|
36
|
+
class AddInsertIdsFilter < Filter
|
|
37
|
+
Fluent::Plugin.register_filter('add_insert_ids', self)
|
|
39
38
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
39
|
+
# Constants for configuration.
|
|
40
|
+
module ConfigConstants
|
|
41
|
+
# The default field name of insertIds in the log entry.
|
|
42
|
+
DEFAULT_INSERT_ID_KEY = 'logging.googleapis.com/insertId'.freeze
|
|
43
|
+
# The character size of the insertIds. This matches the setup in the
|
|
44
|
+
# Stackdriver Logging backend.
|
|
45
|
+
INSERT_ID_SIZE = 17
|
|
46
|
+
# The characters that are allowed in the insertIds. This matches the
|
|
47
|
+
# allowed collection by the Stackdriver Logging Backend.
|
|
48
|
+
ALLOWED_CHARS = (Array(0..9) + Array('a'..'z')).freeze
|
|
49
|
+
end
|
|
51
50
|
|
|
52
|
-
|
|
51
|
+
include self::ConfigConstants
|
|
53
52
|
|
|
54
|
-
|
|
55
|
-
|
|
53
|
+
desc 'The field name for insertIds in the log record.'
|
|
54
|
+
config_param :insert_id_key, :string, default: DEFAULT_INSERT_ID_KEY
|
|
56
55
|
|
|
57
|
-
|
|
58
|
-
|
|
56
|
+
# Expose attr_readers for testing.
|
|
57
|
+
attr_reader :insert_id_key
|
|
59
58
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
59
|
+
def start
|
|
60
|
+
super
|
|
61
|
+
@log = $log # rubocop:disable Style/GlobalVars
|
|
63
62
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
63
|
+
# Initialize the insertID.
|
|
64
|
+
@log.info "Started the add_insert_ids plugin with #{@insert_id_key}" \
|
|
65
|
+
' as the insert ID key.'
|
|
66
|
+
@insert_id = generate_initial_insert_id
|
|
67
|
+
@log.info "Initialized the insert ID key to #{@insert_id}."
|
|
68
|
+
end
|
|
70
69
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
70
|
+
def configure(conf)
|
|
71
|
+
super
|
|
72
|
+
end
|
|
74
73
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
74
|
+
def shutdown
|
|
75
|
+
super
|
|
76
|
+
end
|
|
78
77
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
end
|
|
86
|
-
record
|
|
78
|
+
# rubocop:disable Style/UnusedMethodArgument
|
|
79
|
+
def filter(tag, time, record)
|
|
80
|
+
# Only generate and add an insertId field if the record is a hash and
|
|
81
|
+
# the insert ID field is not already set (or set to an empty string).
|
|
82
|
+
if record.is_a?(Hash) && record[@insert_id_key].to_s.empty?
|
|
83
|
+
record[@insert_id_key] = increment_insert_id
|
|
87
84
|
end
|
|
88
|
-
|
|
85
|
+
record
|
|
86
|
+
end
|
|
87
|
+
# rubocop:enable Style/UnusedMethodArgument
|
|
89
88
|
|
|
90
|
-
|
|
89
|
+
private
|
|
91
90
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
91
|
+
# Generate a random string as the initial insertId.
|
|
92
|
+
def generate_initial_insert_id
|
|
93
|
+
Array.new(INSERT_ID_SIZE) { ALLOWED_CHARS.sample }.join
|
|
94
|
+
end
|
|
96
95
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
end
|
|
96
|
+
# Increment the insertId and return the new value.
|
|
97
|
+
def increment_insert_id
|
|
98
|
+
@insert_id = @insert_id.next
|
|
101
99
|
end
|
|
102
100
|
end
|
|
103
101
|
end
|
|
@@ -19,7 +19,7 @@ require 'fluent/plugin/filter_add_insert_ids'
|
|
|
19
19
|
|
|
20
20
|
# Unit tests for filter_add_insert_ids plugin.
|
|
21
21
|
class FilterAddInsertIdsTest < Test::Unit::TestCase
|
|
22
|
-
include Fluent::
|
|
22
|
+
include Fluent::AddInsertIdsFilter::ConfigConstants
|
|
23
23
|
|
|
24
24
|
CUSTOM_INSERT_ID_KEY = 'custom_insert_id_key'.freeze
|
|
25
25
|
INSERT_ID = 'aeyr82r92h249gh9h'.freeze
|
|
@@ -123,7 +123,7 @@ class FilterAddInsertIdsTest < Test::Unit::TestCase
|
|
|
123
123
|
|
|
124
124
|
def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
|
|
125
125
|
Fluent::Test::FilterTestDriver.new(
|
|
126
|
-
Fluent::
|
|
126
|
+
Fluent::AddInsertIdsFilter).configure(conf, true)
|
|
127
127
|
end
|
|
128
128
|
|
|
129
129
|
def log_entry(index)
|