fluent-plugin-cloudwatch-logs 0.0.7 → 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 92dbd2c06330d3511fdc20c7ed1668c8b3984522
|
4
|
+
data.tar.gz: 7fb872f59507e8513201aba54f90089a22023fed
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0ff3b9b908e63926e095391b91d60df659d37dbeead2725f3e726bc9a4482839cbcda17f41d0377aea14fefd8a6af9fc66770f5a3df4d1aa48ce7debaaed8b25
|
7
|
+
data.tar.gz: 49e564f50e9fc4d3d19f11761d1c9f77ab1d8b780b5de006786da03d2c16a796c25e871960f47e9e0965d91eb935ec424a21620c40a3b4ad212aa26203d2af75
|
@@ -2,14 +2,15 @@ module Fluent
|
|
2
2
|
class CloudwatchLogsInput < Input
|
3
3
|
Plugin.register_input('cloudwatch_logs', self)
|
4
4
|
|
5
|
-
config_param :aws_key_id, :string, :default => nil
|
6
|
-
config_param :aws_sec_key, :string, :default => nil
|
5
|
+
config_param :aws_key_id, :string, :default => nil, :secret => true
|
6
|
+
config_param :aws_sec_key, :string, :default => nil, :secret => true
|
7
7
|
config_param :region, :string, :default => nil
|
8
8
|
config_param :tag, :string
|
9
9
|
config_param :log_group_name, :string
|
10
10
|
config_param :log_stream_name, :string
|
11
11
|
config_param :state_file, :string
|
12
12
|
config_param :fetch_interval, :time, default: 60
|
13
|
+
config_param :http_proxy, :string, default: nil
|
13
14
|
|
14
15
|
def initialize
|
15
16
|
super
|
@@ -17,10 +18,16 @@ module Fluent
|
|
17
18
|
require 'aws-sdk-core'
|
18
19
|
end
|
19
20
|
|
21
|
+
def configure(conf)
|
22
|
+
super
|
23
|
+
configure_parser(conf)
|
24
|
+
end
|
25
|
+
|
20
26
|
def start
|
21
27
|
options = {}
|
22
28
|
options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
|
23
29
|
options[:region] = @region if @region
|
30
|
+
options[:http_proxy] = @http_proxy if @http_proxy
|
24
31
|
@logs = Aws::CloudWatchLogs::Client.new(options)
|
25
32
|
|
26
33
|
@finished = false
|
@@ -33,6 +40,13 @@ module Fluent
|
|
33
40
|
end
|
34
41
|
|
35
42
|
private
|
43
|
+
def configure_parser(conf)
|
44
|
+
if conf['format']
|
45
|
+
@parser = TextParser.new
|
46
|
+
@parser.configure(conf)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
36
50
|
def next_token
|
37
51
|
return nil unless File.exist?(@state_file)
|
38
52
|
File.read(@state_file).chomp
|
@@ -53,9 +67,14 @@ module Fluent
|
|
53
67
|
|
54
68
|
events = get_events
|
55
69
|
events.each do |event|
|
56
|
-
|
57
|
-
|
58
|
-
|
70
|
+
if @parser
|
71
|
+
record = @parser.parse(event.message)
|
72
|
+
Engine.emit(@tag, record[0], record[1])
|
73
|
+
else
|
74
|
+
time = (event.timestamp / 1000).floor
|
75
|
+
record = JSON.parse(event.message)
|
76
|
+
Engine.emit(@tag, time, record)
|
77
|
+
end
|
59
78
|
end
|
60
79
|
end
|
61
80
|
sleep 1
|
@@ -2,8 +2,8 @@ module Fluent
|
|
2
2
|
class CloudwatchLogsOutput < BufferedOutput
|
3
3
|
Plugin.register_output('cloudwatch_logs', self)
|
4
4
|
|
5
|
-
config_param :aws_key_id, :string, :default => nil
|
6
|
-
config_param :aws_sec_key, :string, :default => nil
|
5
|
+
config_param :aws_key_id, :string, :default => nil, :secret => true
|
6
|
+
config_param :aws_sec_key, :string, :default => nil, :secret => true
|
7
7
|
config_param :region, :string, :default => nil
|
8
8
|
config_param :log_group_name, :string, :default => nil
|
9
9
|
config_param :log_stream_name, :string
|
@@ -11,8 +11,9 @@ module Fluent
|
|
11
11
|
config_param :message_keys, :string, :default => nil
|
12
12
|
config_param :max_message_length, :integer, :default => nil
|
13
13
|
config_param :use_tag_as_group, :bool, :default => false
|
14
|
+
config_param :http_proxy, :string, default: nil
|
14
15
|
|
15
|
-
MAX_EVENTS_SIZE =
|
16
|
+
MAX_EVENTS_SIZE = 1_048_576
|
16
17
|
|
17
18
|
unless method_defined?(:log)
|
18
19
|
define_method(:log) { $log }
|
@@ -30,6 +31,7 @@ module Fluent
|
|
30
31
|
options = {}
|
31
32
|
options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
|
32
33
|
options[:region] = @region if @region
|
34
|
+
options[:http_proxy] = @http_proxy if @http_proxy
|
33
35
|
@logs = Aws::CloudWatchLogs::Client.new(options)
|
34
36
|
@sequence_tokens = {}
|
35
37
|
end
|
@@ -99,10 +101,10 @@ module Fluent
|
|
99
101
|
def put_events_by_chunk(group_name, events)
|
100
102
|
chunk = []
|
101
103
|
|
102
|
-
# The maximum batch size is
|
104
|
+
# The maximum batch size is 1,048,576 bytes, and this size is calculated as the sum of all event messages in UTF-8, plus 26 bytes for each log event.
|
103
105
|
# http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html
|
104
106
|
while event = events.shift
|
105
|
-
if (chunk + [event]).inject(0) {|sum, e| sum + e[:message].length } > MAX_EVENTS_SIZE
|
107
|
+
if (chunk + [event]).inject(0) {|sum, e| sum + e[:message].length + 26 } > MAX_EVENTS_SIZE
|
106
108
|
put_events(group_name, chunk)
|
107
109
|
chunk = [event]
|
108
110
|
else
|
@@ -162,12 +164,27 @@ module Fluent
|
|
162
164
|
false
|
163
165
|
elsif @sequence_tokens[group_name].has_key?(stream_name)
|
164
166
|
true
|
165
|
-
elsif (log_stream =
|
167
|
+
elsif (log_stream = find_log_stream(group_name, stream_name))
|
166
168
|
@sequence_tokens[group_name][stream_name] = log_stream.upload_sequence_token
|
167
169
|
true
|
168
170
|
else
|
169
171
|
false
|
170
172
|
end
|
171
173
|
end
|
174
|
+
|
175
|
+
def find_log_stream(group_name, stream_name)
|
176
|
+
next_token = nil
|
177
|
+
loop do
|
178
|
+
response = @logs.describe_log_streams(log_group_name: group_name, next_token: next_token)
|
179
|
+
if (log_stream = response.log_streams.find {|i| i.log_stream_name == stream_name })
|
180
|
+
return log_stream
|
181
|
+
end
|
182
|
+
if response.next_token.nil?
|
183
|
+
break
|
184
|
+
end
|
185
|
+
next_token = response.next_token
|
186
|
+
end
|
187
|
+
nil
|
188
|
+
end
|
172
189
|
end
|
173
190
|
end
|
@@ -56,6 +56,43 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
56
56
|
assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
|
57
57
|
end
|
58
58
|
|
59
|
+
def test_emit_width_format
|
60
|
+
create_log_stream
|
61
|
+
|
62
|
+
time_ms = (Time.now.to_f * 1000).floor
|
63
|
+
put_log_events([
|
64
|
+
{timestamp: time_ms, message: 'logs1'},
|
65
|
+
{timestamp: time_ms, message: 'logs2'},
|
66
|
+
])
|
67
|
+
|
68
|
+
sleep 5
|
69
|
+
|
70
|
+
d = create_driver(<<-EOC)
|
71
|
+
tag test
|
72
|
+
type cloudwatch_logs
|
73
|
+
log_group_name #{log_group_name}
|
74
|
+
log_stream_name #{log_stream_name}
|
75
|
+
state_file /tmp/state
|
76
|
+
format /^(?<cloudwatch>[^ ]*)?/
|
77
|
+
#{aws_key_id}
|
78
|
+
#{aws_sec_key}
|
79
|
+
#{region}
|
80
|
+
EOC
|
81
|
+
|
82
|
+
d.run do
|
83
|
+
sleep 5
|
84
|
+
end
|
85
|
+
|
86
|
+
emits = d.emits
|
87
|
+
assert_equal(2, emits.size)
|
88
|
+
assert_equal('test', emits[0][0])
|
89
|
+
assert_in_delta((time_ms / 1000).floor, emits[0][1], 10)
|
90
|
+
assert_equal({'cloudwatch' => 'logs1'}, emits[0][2])
|
91
|
+
assert_equal('test', emits[1][0])
|
92
|
+
assert_in_delta((time_ms / 1000).floor, emits[1][1], 10)
|
93
|
+
assert_equal({'cloudwatch' => 'logs2'}, emits[1][2])
|
94
|
+
end
|
95
|
+
|
59
96
|
private
|
60
97
|
def default_config
|
61
98
|
<<-EOC
|
data/test/test_helper.rb
CHANGED
@@ -9,6 +9,7 @@ module CloudwatchLogsTestHelper
|
|
9
9
|
options = {}
|
10
10
|
options[:credentials] = Aws::Credentials.new(ENV['aws_key_id'], ENV['aws_sec_key']) if ENV['aws_key_id'] && ENV['aws_sec_key']
|
11
11
|
options[:region] = ENV['region'] if ENV['region']
|
12
|
+
options[:http_proxy] = ENV['http_proxy'] if ENV['http_proxy']
|
12
13
|
@logs ||= Aws::CloudWatchLogs.new(options)
|
13
14
|
end
|
14
15
|
|
metadata
CHANGED
@@ -1,69 +1,69 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryota Arai
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-09-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: aws-sdk-core
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - ~>
|
31
|
+
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: 2.0.7
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - ~>
|
38
|
+
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: 2.0.7
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: bundler
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - ~>
|
45
|
+
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '1.6'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - ~>
|
52
|
+
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '1.6'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: rake
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- -
|
59
|
+
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '0'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- -
|
66
|
+
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
description:
|
@@ -73,7 +73,7 @@ executables: []
|
|
73
73
|
extensions: []
|
74
74
|
extra_rdoc_files: []
|
75
75
|
files:
|
76
|
-
- .gitignore
|
76
|
+
- ".gitignore"
|
77
77
|
- Gemfile
|
78
78
|
- LICENSE.txt
|
79
79
|
- README.md
|
@@ -97,17 +97,17 @@ require_paths:
|
|
97
97
|
- lib
|
98
98
|
required_ruby_version: !ruby/object:Gem::Requirement
|
99
99
|
requirements:
|
100
|
-
- -
|
100
|
+
- - ">="
|
101
101
|
- !ruby/object:Gem::Version
|
102
102
|
version: '0'
|
103
103
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
104
104
|
requirements:
|
105
|
-
- -
|
105
|
+
- - ">="
|
106
106
|
- !ruby/object:Gem::Version
|
107
107
|
version: '0'
|
108
108
|
requirements: []
|
109
109
|
rubyforge_project:
|
110
|
-
rubygems_version: 2.
|
110
|
+
rubygems_version: 2.4.5
|
111
111
|
signing_key:
|
112
112
|
specification_version: 4
|
113
113
|
summary: CloudWatch Logs Plugin for Fluentd
|