fluent-plugin-cloudwatch-logs 0.0.9 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d63e33f244adf82bff43836bd21d935eebb5fee8
|
4
|
+
data.tar.gz: 6e14fdd4432c7b21c0b0cf6b9c1f6574565a34b9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 87332e167639ac1d7a43404849f05bb1820faac2190e2e30ce73672159b2e52f91aa68c7bb3433f815d3969c87fcd0d46c9408a84612bed7243c66afc0e53d60
|
7
|
+
data.tar.gz: 52698378e0c3d7a935aec5117eceba6ae3799dedc09e125bc99dd9f2f0608448cc640c08138a3a4d6703884371e868e1252af97773bfbb30e3c117baa02dc429
|
data/README.md
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# fluent-plugin-cloudwatch-logs
|
2
2
|
|
3
|
+
[![Gem Version](https://badge.fury.io/rb/fluent-plugin-cloudwatch-logs.svg)](http://badge.fury.io/rb/fluent-plugin-cloudwatch-logs)
|
4
|
+
|
3
5
|
[CloudWatch Logs](http://aws.amazon.com/blogs/aws/cloudwatch-log-service/) Plugin for Fluentd
|
4
6
|
|
5
7
|
## Installation
|
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
|
|
18
18
|
spec.require_paths = ["lib"]
|
19
19
|
|
20
20
|
spec.add_dependency 'fluentd'
|
21
|
-
spec.add_dependency 'aws-sdk-core', '
|
21
|
+
spec.add_dependency 'aws-sdk-core', '>= 2.0.7'
|
22
22
|
|
23
23
|
spec.add_development_dependency "bundler", "~> 1.6"
|
24
24
|
spec.add_development_dependency "rake"
|
@@ -6,11 +6,12 @@ module Fluent
|
|
6
6
|
config_param :aws_sec_key, :string, :default => nil, :secret => true
|
7
7
|
config_param :region, :string, :default => nil
|
8
8
|
config_param :log_group_name, :string, :default => nil
|
9
|
-
config_param :log_stream_name, :string
|
9
|
+
config_param :log_stream_name, :string, :default => nil
|
10
10
|
config_param :auto_create_stream, :bool, default: false
|
11
11
|
config_param :message_keys, :string, :default => nil
|
12
12
|
config_param :max_message_length, :integer, :default => nil
|
13
13
|
config_param :use_tag_as_group, :bool, :default => false
|
14
|
+
config_param :use_tag_as_stream, :bool, :default => false
|
14
15
|
config_param :http_proxy, :string, default: nil
|
15
16
|
|
16
17
|
MAX_EVENTS_SIZE = 1_048_576
|
@@ -46,6 +47,7 @@ module Fluent
|
|
46
47
|
tag
|
47
48
|
}.each {|tag, rs|
|
48
49
|
group_name = @use_tag_as_group ? tag : @log_group_name
|
50
|
+
stream_name = @use_tag_as_stream ? tag : @log_stream_name
|
49
51
|
|
50
52
|
unless log_group_exists?(group_name)
|
51
53
|
if @auto_create_stream
|
@@ -56,11 +58,11 @@ module Fluent
|
|
56
58
|
end
|
57
59
|
end
|
58
60
|
|
59
|
-
unless log_stream_exists?(group_name,
|
61
|
+
unless log_stream_exists?(group_name, stream_name)
|
60
62
|
if @auto_create_stream
|
61
|
-
create_log_stream(group_name,
|
63
|
+
create_log_stream(group_name, stream_name)
|
62
64
|
else
|
63
|
-
log.warn "Log stream '#{
|
65
|
+
log.warn "Log stream '#{stream_name}' dose not exists"
|
64
66
|
next
|
65
67
|
end
|
66
68
|
end
|
@@ -85,7 +87,7 @@ module Fluent
|
|
85
87
|
# The log events in the batch must be in chronological ordered by their timestamp.
|
86
88
|
# http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html
|
87
89
|
events = events.sort_by {|e| e[:timestamp] }
|
88
|
-
put_events_by_chunk(group_name, events)
|
90
|
+
put_events_by_chunk(group_name, stream_name, events)
|
89
91
|
}
|
90
92
|
end
|
91
93
|
|
@@ -98,14 +100,14 @@ module Fluent
|
|
98
100
|
@sequence_tokens[group_name][stream_name] = token
|
99
101
|
end
|
100
102
|
|
101
|
-
def put_events_by_chunk(group_name, events)
|
103
|
+
def put_events_by_chunk(group_name, stream_name, events)
|
102
104
|
chunk = []
|
103
105
|
|
104
106
|
# The maximum batch size is 1,048,576 bytes, and this size is calculated as the sum of all event messages in UTF-8, plus 26 bytes for each log event.
|
105
107
|
# http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html
|
106
108
|
while event = events.shift
|
107
109
|
if (chunk + [event]).inject(0) {|sum, e| sum + e[:message].length + 26 } > MAX_EVENTS_SIZE
|
108
|
-
put_events(group_name, chunk)
|
110
|
+
put_events(group_name, stream_name, chunk)
|
109
111
|
chunk = [event]
|
110
112
|
else
|
111
113
|
chunk << event
|
@@ -113,21 +115,21 @@ module Fluent
|
|
113
115
|
end
|
114
116
|
|
115
117
|
unless chunk.empty?
|
116
|
-
put_events(group_name, chunk)
|
118
|
+
put_events(group_name, stream_name, chunk)
|
117
119
|
end
|
118
120
|
end
|
119
121
|
|
120
|
-
def put_events(group_name, events)
|
122
|
+
def put_events(group_name, stream_name, events)
|
121
123
|
args = {
|
122
124
|
log_events: events,
|
123
125
|
log_group_name: group_name,
|
124
|
-
log_stream_name:
|
126
|
+
log_stream_name: stream_name,
|
125
127
|
}
|
126
|
-
token = next_sequence_token(group_name,
|
128
|
+
token = next_sequence_token(group_name, stream_name)
|
127
129
|
args[:sequence_token] = token if token
|
128
130
|
|
129
131
|
response = @logs.put_log_events(args)
|
130
|
-
store_next_sequence_token(group_name,
|
132
|
+
store_next_sequence_token(group_name, stream_name, response.next_sequence_token)
|
131
133
|
end
|
132
134
|
|
133
135
|
def create_log_group(group_name)
|
@@ -125,6 +125,30 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
125
125
|
assert_equal('message2 logs2', events[1].message)
|
126
126
|
end
|
127
127
|
|
128
|
+
def test_write_use_tag_as_stream
|
129
|
+
new_log_stream
|
130
|
+
|
131
|
+
d = create_driver(<<-EOC)
|
132
|
+
#{default_config}
|
133
|
+
message_keys message,cloudwatch
|
134
|
+
use_tag_as_stream true
|
135
|
+
EOC
|
136
|
+
|
137
|
+
time = Time.now
|
138
|
+
d.emit({'cloudwatch' => 'logs1', 'message' => 'message1'}, time.to_i)
|
139
|
+
d.emit({'cloudwatch' => 'logs2', 'message' => 'message2'}, time.to_i + 1)
|
140
|
+
d.run
|
141
|
+
|
142
|
+
sleep 20
|
143
|
+
|
144
|
+
events = get_log_events(log_group_name, fluentd_tag)
|
145
|
+
assert_equal(2, events.size)
|
146
|
+
assert_equal(time.to_i * 1000, events[0].timestamp)
|
147
|
+
assert_equal('message1 logs1', events[0].message)
|
148
|
+
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
149
|
+
assert_equal('message2 logs2', events[1].message)
|
150
|
+
end
|
151
|
+
|
128
152
|
private
|
129
153
|
def default_config
|
130
154
|
<<-EOC
|
data/test/test_helper.rb
CHANGED
@@ -10,7 +10,7 @@ module CloudwatchLogsTestHelper
|
|
10
10
|
options[:credentials] = Aws::Credentials.new(ENV['aws_key_id'], ENV['aws_sec_key']) if ENV['aws_key_id'] && ENV['aws_sec_key']
|
11
11
|
options[:region] = ENV['region'] if ENV['region']
|
12
12
|
options[:http_proxy] = ENV['http_proxy'] if ENV['http_proxy']
|
13
|
-
@logs ||= Aws::CloudWatchLogs.new(options)
|
13
|
+
@logs ||= Aws::CloudWatchLogs::Client.new(options)
|
14
14
|
end
|
15
15
|
|
16
16
|
def log_group_name
|
@@ -68,8 +68,8 @@ module CloudwatchLogsTestHelper
|
|
68
68
|
end
|
69
69
|
end
|
70
70
|
|
71
|
-
def get_log_events(
|
72
|
-
logs.get_log_events(log_group_name:
|
71
|
+
def get_log_events(group = log_group_name, stream = log_stream_name)
|
72
|
+
logs.get_log_events(log_group_name: group, log_stream_name: stream).events
|
73
73
|
end
|
74
74
|
|
75
75
|
def put_log_events(events)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryota Arai
|
@@ -28,14 +28,14 @@ dependencies:
|
|
28
28
|
name: aws-sdk-core
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - "
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: 2.0.7
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - "
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: 2.0.7
|
41
41
|
- !ruby/object:Gem::Dependency
|