fluent-plugin-logzio 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +17 -15
- data/fluent-plugin-logzio.gemspec +4 -4
- data/lib/fluent/plugin/out_logzio_buffered.rb +55 -8
- metadata +25 -21
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 11d3f4a159acd08d6095600f0797a728286d5506
|
|
4
|
+
data.tar.gz: cd557a5cc9b16d7f8a1a3dbda0ba49f9905141f3
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 974b08d3a578522f7d481d936cb9b13637a3380f8fcb4dc7834aa5841608f8d7c0104d469eff5a170b8670d790300a769733289b9ece14698f167e356dce5482
|
|
7
|
+
data.tar.gz: 6bf09502c2396b3e59167af13c80e1411e88c1b2c4a4d19b29b11b874528fb89b7038193b01203fafcbfe937a5da533a34bef83b3859590e425f55fce6374780
|
data/README.md
CHANGED
|
@@ -7,27 +7,29 @@ With fluent-plugin-logzio you will be able to use [Logz.io](http://logz.io) as o
|
|
|
7
7
|
* gem install fluent-plugin-logzio
|
|
8
8
|
* Make sure you have an account with Logz.io.
|
|
9
9
|
* Configure Fluentd as below:
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
type logzio
|
|
13
|
-
endpoint_url http://listener.logz.io:8090?token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
14
|
-
</match>
|
|
15
|
-
~~~~
|
|
16
|
-
or if you want to use buffered plugin:
|
|
17
|
-
~~~~
|
|
10
|
+
|
|
11
|
+
```
|
|
18
12
|
<match your_match>
|
|
19
13
|
type logzio_buffered
|
|
20
|
-
endpoint_url
|
|
21
|
-
output_include_time true
|
|
14
|
+
endpoint_url https://listener.logz.io:8071?token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&type=my_type
|
|
15
|
+
output_include_time true
|
|
16
|
+
output_include_tags true
|
|
22
17
|
buffer_type file
|
|
23
18
|
buffer_path /path/to/buffer/file
|
|
24
19
|
flush_interval 10s
|
|
20
|
+
buffer_chunk_limit 1m # Logz.io has bulk limit of 10M. We recommend set this to 1M, to avoid oversized bulks
|
|
25
21
|
</match>
|
|
26
|
-
|
|
22
|
+
```
|
|
27
23
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
24
|
+
If you absolutly must, use the non-buffered plugin (we really recommend using the buffered)
|
|
25
|
+
```
|
|
26
|
+
<match your_match>
|
|
27
|
+
type logzio
|
|
28
|
+
endpoint_url http://listener.logz.io:8090?token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
29
|
+
</match>
|
|
30
|
+
```
|
|
31
31
|
|
|
32
32
|
## Parameters
|
|
33
|
-
**endpoint_url** the url to your Logz.io
|
|
33
|
+
* **endpoint_url** the url to Logz.io input where `xxx-xxxx...` is your Logz.io access token, and `my_type` is the type of your logs in logz.io
|
|
34
|
+
* **output_include_time** should the appender add a timestamp to your logs on their process time. (recommended)
|
|
35
|
+
* **output_include_tags** should the appender add the fluentd tag to the document, called "fluentd_tag"
|
|
@@ -4,9 +4,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
4
4
|
|
|
5
5
|
Gem::Specification.new do |s|
|
|
6
6
|
s.name = 'fluent-plugin-logzio'
|
|
7
|
-
s.version = '0.0.
|
|
8
|
-
s.authors = ['Yury Kotov']
|
|
9
|
-
s.email = ['bairkan@gmail.com']
|
|
7
|
+
s.version = '0.0.3'
|
|
8
|
+
s.authors = ['Yury Kotov', 'Roi Rav-Hon']
|
|
9
|
+
s.email = ['bairkan@gmail.com', 'roi@logz.io']
|
|
10
10
|
s.homepage = 'https://github.com/logzio/fluent-plugin-logzio'
|
|
11
11
|
s.summary = %q{Fluentd plugin for output to Logz.io}
|
|
12
12
|
s.description = %q{Fluentd pluging (fluent.org) for output to Logz.io (logz.io)}
|
|
@@ -14,7 +14,7 @@ Gem::Specification.new do |s|
|
|
|
14
14
|
s.rubyforge_project = 'fluent-plugin-logzio'
|
|
15
15
|
|
|
16
16
|
s.files = `git ls-files`.split("\n")
|
|
17
|
-
s.test_files = `git ls-files -- {test,spec,features}/*`.split(
|
|
17
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
|
18
18
|
s.require_paths = ['lib']
|
|
19
19
|
s.required_ruby_version = Gem::Requirement.new('>= 1.9.3')
|
|
20
20
|
|
|
@@ -2,7 +2,9 @@ module Fluent
|
|
|
2
2
|
class LogzioOutputBuffered < Fluent::BufferedOutput
|
|
3
3
|
Fluent::Plugin.register_output('logzio_buffered', self)
|
|
4
4
|
config_param :endpoint_url, :string, default: nil
|
|
5
|
-
config_param :output_include_time, :bool, default: true
|
|
5
|
+
config_param :output_include_time, :bool, default: true
|
|
6
|
+
config_param :output_include_tags, :bool, default: true
|
|
7
|
+
config_param :retry_count, :integer, default: 3 # How many times to resend failed bulks. Undocumented because not suppose to be changed
|
|
6
8
|
|
|
7
9
|
unless method_defined?(:log)
|
|
8
10
|
define_method('log') { $log }
|
|
@@ -18,7 +20,8 @@ module Fluent
|
|
|
18
20
|
require 'net/http/persistent'
|
|
19
21
|
@uri = URI @endpoint_url
|
|
20
22
|
@http = Net::HTTP::Persistent.new 'fluent-plugin-logzio', :ENV
|
|
21
|
-
@http.headers['Content-Type'] = 'text'
|
|
23
|
+
@http.headers['Content-Type'] = 'text/plain'
|
|
24
|
+
$log.debug "Started logzio shipper.."
|
|
22
25
|
end
|
|
23
26
|
|
|
24
27
|
def shutdown
|
|
@@ -31,19 +34,63 @@ module Fluent
|
|
|
31
34
|
|
|
32
35
|
def write(chunk)
|
|
33
36
|
records = []
|
|
37
|
+
|
|
34
38
|
chunk.msgpack_each {|tag,time,record|
|
|
35
|
-
record['timestamp'] ||= Time.at(time).iso8601 if @output_include_time
|
|
39
|
+
record['@timestamp'] ||= Time.at(time).iso8601 if @output_include_time
|
|
40
|
+
record['fluentd_tags'] ||= tag.to_s if @output_include_tags
|
|
36
41
|
records.push(record.to_json)
|
|
37
42
|
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
43
|
+
|
|
44
|
+
$log.debug "Got flush timeout, containing #{records.length} chunks"
|
|
45
|
+
|
|
46
|
+
# Setting our request
|
|
47
|
+
post = Net::HTTP::Post.new @uri.request_uri
|
|
48
|
+
|
|
49
|
+
# Logz.io bulk http endpoint expecting log line with \n delimiter
|
|
50
|
+
post.body = records.join("\n")
|
|
51
|
+
|
|
41
52
|
begin
|
|
42
53
|
response = @http.request @uri, post
|
|
43
54
|
$log.debug "HTTP Response code #{response.code}"
|
|
44
|
-
|
|
55
|
+
|
|
56
|
+
if response.code != '200'
|
|
57
|
+
|
|
58
|
+
$log.debug "Got HTTP #{response.code} from logz.io, not giving up just yet"
|
|
59
|
+
|
|
60
|
+
# If any other non-200, we will try to resend it after 2, 4 and 8 seconds. Then we will give up
|
|
61
|
+
|
|
62
|
+
sleep_interval = 2
|
|
63
|
+
@retry_count.times do |counter|
|
|
64
|
+
|
|
65
|
+
$log.debug "Sleeping for #{sleep_interval} seconds, and trying again."
|
|
66
|
+
|
|
67
|
+
sleep(sleep_interval)
|
|
68
|
+
|
|
69
|
+
# Retry
|
|
70
|
+
response = @http.request @uri, post
|
|
71
|
+
|
|
72
|
+
# Sucecss, no further action is needed
|
|
73
|
+
if response.code == 200
|
|
74
|
+
|
|
75
|
+
$log.debug "Successfuly sent the failed bulk."
|
|
76
|
+
|
|
77
|
+
# Breaking out
|
|
78
|
+
break
|
|
79
|
+
|
|
80
|
+
else
|
|
81
|
+
|
|
82
|
+
# Doubling the sleep interval
|
|
83
|
+
sleep_interval *= 2
|
|
84
|
+
|
|
85
|
+
if counter == @retry_count - 1
|
|
86
|
+
|
|
87
|
+
$log.error "Could not send your bulk after 3 tries. Sorry. Got HTTP #{response.code}"
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
45
92
|
rescue StandardError
|
|
46
|
-
$log.error "Error connecting to logzio verify the url #{@
|
|
93
|
+
$log.error "Error connecting to logzio. verify the url #{@endpoint_url}"
|
|
47
94
|
end
|
|
48
95
|
end
|
|
49
96
|
end
|
metadata
CHANGED
|
@@ -1,108 +1,110 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-logzio
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Yury Kotov
|
|
8
|
+
- Roi Rav-Hon
|
|
8
9
|
autorequire:
|
|
9
10
|
bindir: bin
|
|
10
11
|
cert_chain: []
|
|
11
|
-
date: 2015-
|
|
12
|
+
date: 2015-12-13 00:00:00.000000000 Z
|
|
12
13
|
dependencies:
|
|
13
14
|
- !ruby/object:Gem::Dependency
|
|
14
15
|
name: net-http-persistent
|
|
15
16
|
requirement: !ruby/object:Gem::Requirement
|
|
16
17
|
requirements:
|
|
17
|
-
- -
|
|
18
|
+
- - ~>
|
|
18
19
|
- !ruby/object:Gem::Version
|
|
19
20
|
version: '2.7'
|
|
20
21
|
type: :runtime
|
|
21
22
|
prerelease: false
|
|
22
23
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
24
|
requirements:
|
|
24
|
-
- -
|
|
25
|
+
- - ~>
|
|
25
26
|
- !ruby/object:Gem::Version
|
|
26
27
|
version: '2.7'
|
|
27
28
|
- !ruby/object:Gem::Dependency
|
|
28
29
|
name: fluentd
|
|
29
30
|
requirement: !ruby/object:Gem::Requirement
|
|
30
31
|
requirements:
|
|
31
|
-
- -
|
|
32
|
+
- - ~>
|
|
32
33
|
- !ruby/object:Gem::Version
|
|
33
34
|
version: '0.12'
|
|
34
35
|
type: :runtime
|
|
35
36
|
prerelease: false
|
|
36
37
|
version_requirements: !ruby/object:Gem::Requirement
|
|
37
38
|
requirements:
|
|
38
|
-
- -
|
|
39
|
+
- - ~>
|
|
39
40
|
- !ruby/object:Gem::Version
|
|
40
41
|
version: '0.12'
|
|
41
42
|
- !ruby/object:Gem::Dependency
|
|
42
43
|
name: rake
|
|
43
44
|
requirement: !ruby/object:Gem::Requirement
|
|
44
45
|
requirements:
|
|
45
|
-
- -
|
|
46
|
+
- - ~>
|
|
46
47
|
- !ruby/object:Gem::Version
|
|
47
48
|
version: '10.4'
|
|
48
49
|
type: :development
|
|
49
50
|
prerelease: false
|
|
50
51
|
version_requirements: !ruby/object:Gem::Requirement
|
|
51
52
|
requirements:
|
|
52
|
-
- -
|
|
53
|
+
- - ~>
|
|
53
54
|
- !ruby/object:Gem::Version
|
|
54
55
|
version: '10.4'
|
|
55
56
|
- !ruby/object:Gem::Dependency
|
|
56
57
|
name: bundler
|
|
57
58
|
requirement: !ruby/object:Gem::Requirement
|
|
58
59
|
requirements:
|
|
59
|
-
- -
|
|
60
|
+
- - ~>
|
|
60
61
|
- !ruby/object:Gem::Version
|
|
61
62
|
version: '1.10'
|
|
62
63
|
type: :development
|
|
63
64
|
prerelease: false
|
|
64
65
|
version_requirements: !ruby/object:Gem::Requirement
|
|
65
66
|
requirements:
|
|
66
|
-
- -
|
|
67
|
+
- - ~>
|
|
67
68
|
- !ruby/object:Gem::Version
|
|
68
69
|
version: '1.10'
|
|
69
70
|
- !ruby/object:Gem::Dependency
|
|
70
71
|
name: rspec
|
|
71
72
|
requirement: !ruby/object:Gem::Requirement
|
|
72
73
|
requirements:
|
|
73
|
-
- -
|
|
74
|
+
- - ~>
|
|
74
75
|
- !ruby/object:Gem::Version
|
|
75
76
|
version: '3.3'
|
|
76
77
|
type: :development
|
|
77
78
|
prerelease: false
|
|
78
79
|
version_requirements: !ruby/object:Gem::Requirement
|
|
79
80
|
requirements:
|
|
80
|
-
- -
|
|
81
|
+
- - ~>
|
|
81
82
|
- !ruby/object:Gem::Version
|
|
82
83
|
version: '3.3'
|
|
83
84
|
- !ruby/object:Gem::Dependency
|
|
84
85
|
name: test-unit
|
|
85
86
|
requirement: !ruby/object:Gem::Requirement
|
|
86
87
|
requirements:
|
|
87
|
-
- -
|
|
88
|
+
- - ~>
|
|
88
89
|
- !ruby/object:Gem::Version
|
|
89
90
|
version: '3.1'
|
|
90
91
|
type: :development
|
|
91
92
|
prerelease: false
|
|
92
93
|
version_requirements: !ruby/object:Gem::Requirement
|
|
93
94
|
requirements:
|
|
94
|
-
- -
|
|
95
|
+
- - ~>
|
|
95
96
|
- !ruby/object:Gem::Version
|
|
96
97
|
version: '3.1'
|
|
97
98
|
description: Fluentd pluging (fluent.org) for output to Logz.io (logz.io)
|
|
98
99
|
email:
|
|
99
100
|
- bairkan@gmail.com
|
|
101
|
+
- roi@logz.io
|
|
100
102
|
executables: []
|
|
101
103
|
extensions: []
|
|
102
104
|
extra_rdoc_files: []
|
|
103
105
|
files:
|
|
104
|
-
-
|
|
105
|
-
-
|
|
106
|
+
- .gitignore
|
|
107
|
+
- .rspec
|
|
106
108
|
- Gemfile
|
|
107
109
|
- LICENSE
|
|
108
110
|
- README.md
|
|
@@ -122,19 +124,21 @@ require_paths:
|
|
|
122
124
|
- lib
|
|
123
125
|
required_ruby_version: !ruby/object:Gem::Requirement
|
|
124
126
|
requirements:
|
|
125
|
-
- -
|
|
127
|
+
- - '>='
|
|
126
128
|
- !ruby/object:Gem::Version
|
|
127
129
|
version: 1.9.3
|
|
128
130
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
129
131
|
requirements:
|
|
130
|
-
- -
|
|
132
|
+
- - '>='
|
|
131
133
|
- !ruby/object:Gem::Version
|
|
132
134
|
version: '0'
|
|
133
135
|
requirements: []
|
|
134
136
|
rubyforge_project: fluent-plugin-logzio
|
|
135
|
-
rubygems_version: 2.
|
|
137
|
+
rubygems_version: 2.0.14
|
|
136
138
|
signing_key:
|
|
137
139
|
specification_version: 4
|
|
138
140
|
summary: Fluentd plugin for output to Logz.io
|
|
139
|
-
test_files:
|
|
140
|
-
|
|
141
|
+
test_files:
|
|
142
|
+
- spec/lib/fluent/plugin/out_logzio_buffered_spec.rb
|
|
143
|
+
- spec/lib/fluent/plugin/out_logzio_spec.rb
|
|
144
|
+
- spec/spec_helper.rb
|