fluent-plugin-google-cloud 0.6.25.pre.2 → 0.6.25.pre.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +3 -3
- data/fluent-plugin-google-cloud.gemspec +1 -1
- data/lib/fluent/plugin/filter_add_insert_ids.rb +68 -70
- data/test/plugin/test_filter_add_insert_ids.rb +2 -2
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 219e76eaed4425f0075746b4f95fbe5673970281
|
4
|
+
data.tar.gz: 5c5f0c9e4dc6da1b58ac3252a05c891f0a40c9f0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 223aefacf4ab26c6b6d0c1df418ba12e6d540b592c669d68a8c1fbfb6dcba12f50a7a9574e1e2a53f27760cd2bfecbfa1ff2a603be9223f0e9848da4c250c6d6
|
7
|
+
data.tar.gz: e6d95ce23a5eee04730cfeedc64f983290143dd093405c7cc68afdf08ff0984130744d00a948e7fd95eab8632b56b80616cd2b3405bcd4526ea93f10f51ef43b
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-google-cloud (0.6.25.pre.
|
4
|
+
fluent-plugin-google-cloud (0.6.25.pre.3)
|
5
5
|
fluentd (~> 0.10)
|
6
6
|
google-api-client (~> 0.17)
|
7
7
|
google-cloud-logging (~> 1.3, >= 1.3.2)
|
@@ -58,7 +58,7 @@ GEM
|
|
58
58
|
googleauth (~> 0.6.2)
|
59
59
|
grpc (>= 1.7.2, < 2.0)
|
60
60
|
rly (~> 0.2.3)
|
61
|
-
google-protobuf (3.6.1)
|
61
|
+
google-protobuf (3.6.1-x86_64-linux)
|
62
62
|
googleapis-common-protos (1.3.7)
|
63
63
|
google-protobuf (~> 3.0)
|
64
64
|
googleapis-common-protos-types (~> 1.0)
|
@@ -72,7 +72,7 @@ GEM
|
|
72
72
|
multi_json (~> 1.11)
|
73
73
|
os (>= 0.9, < 2.0)
|
74
74
|
signet (~> 0.7)
|
75
|
-
grpc (1.15.0)
|
75
|
+
grpc (1.15.0-x86_64-linux)
|
76
76
|
google-protobuf (~> 3.1)
|
77
77
|
googleapis-common-protos-types (~> 1.0.0)
|
78
78
|
hashdiff (0.3.7)
|
@@ -10,7 +10,7 @@ eos
|
|
10
10
|
gem.homepage =
|
11
11
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
12
12
|
gem.license = 'Apache-2.0'
|
13
|
-
gem.version = '0.6.25.pre.
|
13
|
+
gem.version = '0.6.25.pre.3'
|
14
14
|
gem.authors = ['Stackdriver Agents Team']
|
15
15
|
gem.email = ['stackdriver-agents@google.com']
|
16
16
|
gem.required_ruby_version = Gem::Requirement.new('>= 2.2')
|
@@ -15,89 +15,87 @@
|
|
15
15
|
require 'fluent/plugin/filter'
|
16
16
|
|
17
17
|
module Fluent
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
Fluent::Plugin.register_filter('add_insert_ids', self)
|
18
|
+
# Fluentd filter plugin for adding insertIds to guarantee log entry order
|
19
|
+
# and uniqueness.
|
20
|
+
# Sample log entries enriched by this plugin:
|
21
|
+
# {
|
22
|
+
# "timestamp": "2017-08-22 13:35:28",
|
23
|
+
# "message": "1",
|
24
|
+
# "logging.googleapis.com/insertId": "aye7eakuf23h41aef0"
|
25
|
+
# }
|
26
|
+
# {
|
27
|
+
# "timestamp": "2017-08-22 13:35:28",
|
28
|
+
# "message": "2",
|
29
|
+
# "logging.googleapis.com/insertId": "aye7eakuf23h41aef1"
|
30
|
+
# }
|
31
|
+
# {
|
32
|
+
# "timestamp": "2017-08-22 13:35:28",
|
33
|
+
# "message": "3",
|
34
|
+
# "logging.googleapis.com/insertId": "aye7eakuf23h41aef2"
|
35
|
+
# }
|
36
|
+
class AddInsertIdsFilter < Filter
|
37
|
+
Fluent::Plugin.register_filter('add_insert_ids', self)
|
39
38
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
39
|
+
# Constants for configuration.
|
40
|
+
module ConfigConstants
|
41
|
+
# The default field name of insertIds in the log entry.
|
42
|
+
DEFAULT_INSERT_ID_KEY = 'logging.googleapis.com/insertId'.freeze
|
43
|
+
# The character size of the insertIds. This matches the setup in the
|
44
|
+
# Stackdriver Logging backend.
|
45
|
+
INSERT_ID_SIZE = 17
|
46
|
+
# The characters that are allowed in the insertIds. This matches the
|
47
|
+
# allowed collection by the Stackdriver Logging Backend.
|
48
|
+
ALLOWED_CHARS = (Array(0..9) + Array('a'..'z')).freeze
|
49
|
+
end
|
51
50
|
|
52
|
-
|
51
|
+
include self::ConfigConstants
|
53
52
|
|
54
|
-
|
55
|
-
|
53
|
+
desc 'The field name for insertIds in the log record.'
|
54
|
+
config_param :insert_id_key, :string, default: DEFAULT_INSERT_ID_KEY
|
56
55
|
|
57
|
-
|
58
|
-
|
56
|
+
# Expose attr_readers for testing.
|
57
|
+
attr_reader :insert_id_key
|
59
58
|
|
60
|
-
|
61
|
-
|
62
|
-
|
59
|
+
def start
|
60
|
+
super
|
61
|
+
@log = $log # rubocop:disable Style/GlobalVars
|
63
62
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
63
|
+
# Initialize the insertID.
|
64
|
+
@log.info "Started the add_insert_ids plugin with #{@insert_id_key}" \
|
65
|
+
' as the insert ID key.'
|
66
|
+
@insert_id = generate_initial_insert_id
|
67
|
+
@log.info "Initialized the insert ID key to #{@insert_id}."
|
68
|
+
end
|
70
69
|
|
71
|
-
|
72
|
-
|
73
|
-
|
70
|
+
def configure(conf)
|
71
|
+
super
|
72
|
+
end
|
74
73
|
|
75
|
-
|
76
|
-
|
77
|
-
|
74
|
+
def shutdown
|
75
|
+
super
|
76
|
+
end
|
78
77
|
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
end
|
86
|
-
record
|
78
|
+
# rubocop:disable Style/UnusedMethodArgument
|
79
|
+
def filter(tag, time, record)
|
80
|
+
# Only generate and add an insertId field if the record is a hash and
|
81
|
+
# the insert ID field is not already set (or set to an empty string).
|
82
|
+
if record.is_a?(Hash) && record[@insert_id_key].to_s.empty?
|
83
|
+
record[@insert_id_key] = increment_insert_id
|
87
84
|
end
|
88
|
-
|
85
|
+
record
|
86
|
+
end
|
87
|
+
# rubocop:enable Style/UnusedMethodArgument
|
89
88
|
|
90
|
-
|
89
|
+
private
|
91
90
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
91
|
+
# Generate a random string as the initial insertId.
|
92
|
+
def generate_initial_insert_id
|
93
|
+
Array.new(INSERT_ID_SIZE) { ALLOWED_CHARS.sample }.join
|
94
|
+
end
|
96
95
|
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
end
|
96
|
+
# Increment the insertId and return the new value.
|
97
|
+
def increment_insert_id
|
98
|
+
@insert_id = @insert_id.next
|
101
99
|
end
|
102
100
|
end
|
103
101
|
end
|
@@ -19,7 +19,7 @@ require 'fluent/plugin/filter_add_insert_ids'
|
|
19
19
|
|
20
20
|
# Unit tests for filter_add_insert_ids plugin.
|
21
21
|
class FilterAddInsertIdsTest < Test::Unit::TestCase
|
22
|
-
include Fluent::
|
22
|
+
include Fluent::AddInsertIdsFilter::ConfigConstants
|
23
23
|
|
24
24
|
CUSTOM_INSERT_ID_KEY = 'custom_insert_id_key'.freeze
|
25
25
|
INSERT_ID = 'aeyr82r92h249gh9h'.freeze
|
@@ -123,7 +123,7 @@ class FilterAddInsertIdsTest < Test::Unit::TestCase
|
|
123
123
|
|
124
124
|
def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
|
125
125
|
Fluent::Test::FilterTestDriver.new(
|
126
|
-
Fluent::
|
126
|
+
Fluent::AddInsertIdsFilter).configure(conf, true)
|
127
127
|
end
|
128
128
|
|
129
129
|
def log_entry(index)
|