fluent-plugin-logzio 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +17 -15
- data/fluent-plugin-logzio.gemspec +4 -4
- data/lib/fluent/plugin/out_logzio_buffered.rb +55 -8
- metadata +25 -21
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 11d3f4a159acd08d6095600f0797a728286d5506
|
|
4
|
+
data.tar.gz: cd557a5cc9b16d7f8a1a3dbda0ba49f9905141f3
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 974b08d3a578522f7d481d936cb9b13637a3380f8fcb4dc7834aa5841608f8d7c0104d469eff5a170b8670d790300a769733289b9ece14698f167e356dce5482
|
|
7
|
+
data.tar.gz: 6bf09502c2396b3e59167af13c80e1411e88c1b2c4a4d19b29b11b874528fb89b7038193b01203fafcbfe937a5da533a34bef83b3859590e425f55fce6374780
|
data/README.md
CHANGED
|
@@ -7,27 +7,29 @@ With fluent-plugin-logzio you will be able to use [Logz.io](http://logz.io) as o
|
|
|
7
7
|
* gem install fluent-plugin-logzio
|
|
8
8
|
* Make sure you have an account with Logz.io.
|
|
9
9
|
* Configure Fluentd as below:
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
type logzio
|
|
13
|
-
endpoint_url http://listener.logz.io:8090?token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
14
|
-
</match>
|
|
15
|
-
~~~~
|
|
16
|
-
or if you want to use buffered plugin:
|
|
17
|
-
~~~~
|
|
10
|
+
|
|
11
|
+
```
|
|
18
12
|
<match your_match>
|
|
19
13
|
type logzio_buffered
|
|
20
|
-
endpoint_url
|
|
21
|
-
output_include_time true
|
|
14
|
+
endpoint_url https://listener.logz.io:8071?token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&type=my_type
|
|
15
|
+
output_include_time true
|
|
16
|
+
output_include_tags true
|
|
22
17
|
buffer_type file
|
|
23
18
|
buffer_path /path/to/buffer/file
|
|
24
19
|
flush_interval 10s
|
|
20
|
+
buffer_chunk_limit 1m # Logz.io has bulk limit of 10M. We recommend set this to 1M, to avoid oversized bulks
|
|
25
21
|
</match>
|
|
26
|
-
|
|
22
|
+
```
|
|
27
23
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
24
|
+
If you absolutly must, use the non-buffered plugin (we really recommend using the buffered)
|
|
25
|
+
```
|
|
26
|
+
<match your_match>
|
|
27
|
+
type logzio
|
|
28
|
+
endpoint_url http://listener.logz.io:8090?token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
29
|
+
</match>
|
|
30
|
+
```
|
|
31
31
|
|
|
32
32
|
## Parameters
|
|
33
|
-
**endpoint_url** the url to your Logz.io
|
|
33
|
+
* **endpoint_url** the url to Logz.io input where `xxx-xxxx...` is your Logz.io access token, and `my_type` is the type of your logs in logz.io
|
|
34
|
+
* **output_include_time** should the appender add a timestamp to your logs on their process time. (recommended)
|
|
35
|
+
* **output_include_tags** should the appender add the fluentd tag to the document, called "fluentd_tag"
|
|
@@ -4,9 +4,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
4
4
|
|
|
5
5
|
Gem::Specification.new do |s|
|
|
6
6
|
s.name = 'fluent-plugin-logzio'
|
|
7
|
-
s.version = '0.0.
|
|
8
|
-
s.authors = ['Yury Kotov']
|
|
9
|
-
s.email = ['bairkan@gmail.com']
|
|
7
|
+
s.version = '0.0.3'
|
|
8
|
+
s.authors = ['Yury Kotov', 'Roi Rav-Hon']
|
|
9
|
+
s.email = ['bairkan@gmail.com', 'roi@logz.io']
|
|
10
10
|
s.homepage = 'https://github.com/logzio/fluent-plugin-logzio'
|
|
11
11
|
s.summary = %q{Fluentd plugin for output to Logz.io}
|
|
12
12
|
s.description = %q{Fluentd pluging (fluent.org) for output to Logz.io (logz.io)}
|
|
@@ -14,7 +14,7 @@ Gem::Specification.new do |s|
|
|
|
14
14
|
s.rubyforge_project = 'fluent-plugin-logzio'
|
|
15
15
|
|
|
16
16
|
s.files = `git ls-files`.split("\n")
|
|
17
|
-
s.test_files = `git ls-files -- {test,spec,features}/*`.split(
|
|
17
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
|
18
18
|
s.require_paths = ['lib']
|
|
19
19
|
s.required_ruby_version = Gem::Requirement.new('>= 1.9.3')
|
|
20
20
|
|
|
@@ -2,7 +2,9 @@ module Fluent
|
|
|
2
2
|
class LogzioOutputBuffered < Fluent::BufferedOutput
|
|
3
3
|
Fluent::Plugin.register_output('logzio_buffered', self)
|
|
4
4
|
config_param :endpoint_url, :string, default: nil
|
|
5
|
-
config_param :output_include_time, :bool, default: true
|
|
5
|
+
config_param :output_include_time, :bool, default: true
|
|
6
|
+
config_param :output_include_tags, :bool, default: true
|
|
7
|
+
config_param :retry_count, :integer, default: 3 # How many times to resend failed bulks. Undocumented because not suppose to be changed
|
|
6
8
|
|
|
7
9
|
unless method_defined?(:log)
|
|
8
10
|
define_method('log') { $log }
|
|
@@ -18,7 +20,8 @@ module Fluent
|
|
|
18
20
|
require 'net/http/persistent'
|
|
19
21
|
@uri = URI @endpoint_url
|
|
20
22
|
@http = Net::HTTP::Persistent.new 'fluent-plugin-logzio', :ENV
|
|
21
|
-
@http.headers['Content-Type'] = 'text'
|
|
23
|
+
@http.headers['Content-Type'] = 'text/plain'
|
|
24
|
+
$log.debug "Started logzio shipper.."
|
|
22
25
|
end
|
|
23
26
|
|
|
24
27
|
def shutdown
|
|
@@ -31,19 +34,63 @@ module Fluent
|
|
|
31
34
|
|
|
32
35
|
def write(chunk)
|
|
33
36
|
records = []
|
|
37
|
+
|
|
34
38
|
chunk.msgpack_each {|tag,time,record|
|
|
35
|
-
record['timestamp'] ||= Time.at(time).iso8601 if @output_include_time
|
|
39
|
+
record['@timestamp'] ||= Time.at(time).iso8601 if @output_include_time
|
|
40
|
+
record['fluentd_tags'] ||= tag.to_s if @output_include_tags
|
|
36
41
|
records.push(record.to_json)
|
|
37
42
|
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
43
|
+
|
|
44
|
+
$log.debug "Got flush timeout, containing #{records.length} chunks"
|
|
45
|
+
|
|
46
|
+
# Setting our request
|
|
47
|
+
post = Net::HTTP::Post.new @uri.request_uri
|
|
48
|
+
|
|
49
|
+
# Logz.io bulk http endpoint expecting log line with \n delimiter
|
|
50
|
+
post.body = records.join("\n")
|
|
51
|
+
|
|
41
52
|
begin
|
|
42
53
|
response = @http.request @uri, post
|
|
43
54
|
$log.debug "HTTP Response code #{response.code}"
|
|
44
|
-
|
|
55
|
+
|
|
56
|
+
if response.code != '200'
|
|
57
|
+
|
|
58
|
+
$log.debug "Got HTTP #{response.code} from logz.io, not giving up just yet"
|
|
59
|
+
|
|
60
|
+
# If any other non-200, we will try to resend it after 2, 4 and 8 seconds. Then we will give up
|
|
61
|
+
|
|
62
|
+
sleep_interval = 2
|
|
63
|
+
@retry_count.times do |counter|
|
|
64
|
+
|
|
65
|
+
$log.debug "Sleeping for #{sleep_interval} seconds, and trying again."
|
|
66
|
+
|
|
67
|
+
sleep(sleep_interval)
|
|
68
|
+
|
|
69
|
+
# Retry
|
|
70
|
+
response = @http.request @uri, post
|
|
71
|
+
|
|
72
|
+
# Sucecss, no further action is needed
|
|
73
|
+
if response.code == 200
|
|
74
|
+
|
|
75
|
+
$log.debug "Successfuly sent the failed bulk."
|
|
76
|
+
|
|
77
|
+
# Breaking out
|
|
78
|
+
break
|
|
79
|
+
|
|
80
|
+
else
|
|
81
|
+
|
|
82
|
+
# Doubling the sleep interval
|
|
83
|
+
sleep_interval *= 2
|
|
84
|
+
|
|
85
|
+
if counter == @retry_count - 1
|
|
86
|
+
|
|
87
|
+
$log.error "Could not send your bulk after 3 tries. Sorry. Got HTTP #{response.code}"
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
45
92
|
rescue StandardError
|
|
46
|
-
$log.error "Error connecting to logzio verify the url #{@
|
|
93
|
+
$log.error "Error connecting to logzio. verify the url #{@endpoint_url}"
|
|
47
94
|
end
|
|
48
95
|
end
|
|
49
96
|
end
|
metadata
CHANGED
|
@@ -1,108 +1,110 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-logzio
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Yury Kotov
|
|
8
|
+
- Roi Rav-Hon
|
|
8
9
|
autorequire:
|
|
9
10
|
bindir: bin
|
|
10
11
|
cert_chain: []
|
|
11
|
-
date: 2015-
|
|
12
|
+
date: 2015-12-13 00:00:00.000000000 Z
|
|
12
13
|
dependencies:
|
|
13
14
|
- !ruby/object:Gem::Dependency
|
|
14
15
|
name: net-http-persistent
|
|
15
16
|
requirement: !ruby/object:Gem::Requirement
|
|
16
17
|
requirements:
|
|
17
|
-
- -
|
|
18
|
+
- - ~>
|
|
18
19
|
- !ruby/object:Gem::Version
|
|
19
20
|
version: '2.7'
|
|
20
21
|
type: :runtime
|
|
21
22
|
prerelease: false
|
|
22
23
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
24
|
requirements:
|
|
24
|
-
- -
|
|
25
|
+
- - ~>
|
|
25
26
|
- !ruby/object:Gem::Version
|
|
26
27
|
version: '2.7'
|
|
27
28
|
- !ruby/object:Gem::Dependency
|
|
28
29
|
name: fluentd
|
|
29
30
|
requirement: !ruby/object:Gem::Requirement
|
|
30
31
|
requirements:
|
|
31
|
-
- -
|
|
32
|
+
- - ~>
|
|
32
33
|
- !ruby/object:Gem::Version
|
|
33
34
|
version: '0.12'
|
|
34
35
|
type: :runtime
|
|
35
36
|
prerelease: false
|
|
36
37
|
version_requirements: !ruby/object:Gem::Requirement
|
|
37
38
|
requirements:
|
|
38
|
-
- -
|
|
39
|
+
- - ~>
|
|
39
40
|
- !ruby/object:Gem::Version
|
|
40
41
|
version: '0.12'
|
|
41
42
|
- !ruby/object:Gem::Dependency
|
|
42
43
|
name: rake
|
|
43
44
|
requirement: !ruby/object:Gem::Requirement
|
|
44
45
|
requirements:
|
|
45
|
-
- -
|
|
46
|
+
- - ~>
|
|
46
47
|
- !ruby/object:Gem::Version
|
|
47
48
|
version: '10.4'
|
|
48
49
|
type: :development
|
|
49
50
|
prerelease: false
|
|
50
51
|
version_requirements: !ruby/object:Gem::Requirement
|
|
51
52
|
requirements:
|
|
52
|
-
- -
|
|
53
|
+
- - ~>
|
|
53
54
|
- !ruby/object:Gem::Version
|
|
54
55
|
version: '10.4'
|
|
55
56
|
- !ruby/object:Gem::Dependency
|
|
56
57
|
name: bundler
|
|
57
58
|
requirement: !ruby/object:Gem::Requirement
|
|
58
59
|
requirements:
|
|
59
|
-
- -
|
|
60
|
+
- - ~>
|
|
60
61
|
- !ruby/object:Gem::Version
|
|
61
62
|
version: '1.10'
|
|
62
63
|
type: :development
|
|
63
64
|
prerelease: false
|
|
64
65
|
version_requirements: !ruby/object:Gem::Requirement
|
|
65
66
|
requirements:
|
|
66
|
-
- -
|
|
67
|
+
- - ~>
|
|
67
68
|
- !ruby/object:Gem::Version
|
|
68
69
|
version: '1.10'
|
|
69
70
|
- !ruby/object:Gem::Dependency
|
|
70
71
|
name: rspec
|
|
71
72
|
requirement: !ruby/object:Gem::Requirement
|
|
72
73
|
requirements:
|
|
73
|
-
- -
|
|
74
|
+
- - ~>
|
|
74
75
|
- !ruby/object:Gem::Version
|
|
75
76
|
version: '3.3'
|
|
76
77
|
type: :development
|
|
77
78
|
prerelease: false
|
|
78
79
|
version_requirements: !ruby/object:Gem::Requirement
|
|
79
80
|
requirements:
|
|
80
|
-
- -
|
|
81
|
+
- - ~>
|
|
81
82
|
- !ruby/object:Gem::Version
|
|
82
83
|
version: '3.3'
|
|
83
84
|
- !ruby/object:Gem::Dependency
|
|
84
85
|
name: test-unit
|
|
85
86
|
requirement: !ruby/object:Gem::Requirement
|
|
86
87
|
requirements:
|
|
87
|
-
- -
|
|
88
|
+
- - ~>
|
|
88
89
|
- !ruby/object:Gem::Version
|
|
89
90
|
version: '3.1'
|
|
90
91
|
type: :development
|
|
91
92
|
prerelease: false
|
|
92
93
|
version_requirements: !ruby/object:Gem::Requirement
|
|
93
94
|
requirements:
|
|
94
|
-
- -
|
|
95
|
+
- - ~>
|
|
95
96
|
- !ruby/object:Gem::Version
|
|
96
97
|
version: '3.1'
|
|
97
98
|
description: Fluentd pluging (fluent.org) for output to Logz.io (logz.io)
|
|
98
99
|
email:
|
|
99
100
|
- bairkan@gmail.com
|
|
101
|
+
- roi@logz.io
|
|
100
102
|
executables: []
|
|
101
103
|
extensions: []
|
|
102
104
|
extra_rdoc_files: []
|
|
103
105
|
files:
|
|
104
|
-
-
|
|
105
|
-
-
|
|
106
|
+
- .gitignore
|
|
107
|
+
- .rspec
|
|
106
108
|
- Gemfile
|
|
107
109
|
- LICENSE
|
|
108
110
|
- README.md
|
|
@@ -122,19 +124,21 @@ require_paths:
|
|
|
122
124
|
- lib
|
|
123
125
|
required_ruby_version: !ruby/object:Gem::Requirement
|
|
124
126
|
requirements:
|
|
125
|
-
- -
|
|
127
|
+
- - '>='
|
|
126
128
|
- !ruby/object:Gem::Version
|
|
127
129
|
version: 1.9.3
|
|
128
130
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
129
131
|
requirements:
|
|
130
|
-
- -
|
|
132
|
+
- - '>='
|
|
131
133
|
- !ruby/object:Gem::Version
|
|
132
134
|
version: '0'
|
|
133
135
|
requirements: []
|
|
134
136
|
rubyforge_project: fluent-plugin-logzio
|
|
135
|
-
rubygems_version: 2.
|
|
137
|
+
rubygems_version: 2.0.14
|
|
136
138
|
signing_key:
|
|
137
139
|
specification_version: 4
|
|
138
140
|
summary: Fluentd plugin for output to Logz.io
|
|
139
|
-
test_files:
|
|
140
|
-
|
|
141
|
+
test_files:
|
|
142
|
+
- spec/lib/fluent/plugin/out_logzio_buffered_spec.rb
|
|
143
|
+
- spec/lib/fluent/plugin/out_logzio_spec.rb
|
|
144
|
+
- spec/spec_helper.rb
|