fluent-plugin-timber 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 496cd7f7702fb5c4753cd01e1f5757956110b873
4
+ data.tar.gz: dcb578e403619c54d9bbd4b631f432d532300d70
5
+ SHA512:
6
+ metadata.gz: 94c65c0c0f837a5f1c0fbbe776e3487ad61a6ec1390116150f6164643f77c8862265685f6032bfe41f0b73e6050c21b6c9ab769ad6b2fe5d3272d13aea608b33
7
+ data.tar.gz: 6eaa933afbd1a11dd58a2bde616e7fa7f76c7a3ea32958a70e3c55c8cf6167cec0c56b4805f936fbb8d91bd8c8219c3e3a9b27fbdabd9d96f36e0613a40bfdd0
@@ -0,0 +1,15 @@
1
+ .DS_Store
2
+ .rvmrc
3
+ .ruby-version
4
+ coverage
5
+ *.swp
6
+ *.gem
7
+
8
+ gemfiles/*.lock
9
+
10
+ /.bundle
11
+ /.yardoc
12
+ /doc
13
+ /log
14
+ /tmp
15
+ /pkg
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
@@ -0,0 +1,86 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ fluent-plugin-timber (1.0.0)
5
+ fluentd (>= 0.12.0, < 2)
6
+ http (~> 2.0, >= 2.0.3)
7
+
8
+ GEM
9
+ remote: https://rubygems.org/
10
+ specs:
11
+ addressable (2.5.1)
12
+ public_suffix (~> 2.0, >= 2.0.2)
13
+ cool.io (1.5.0)
14
+ crack (0.4.3)
15
+ safe_yaml (~> 1.0.0)
16
+ diff-lcs (1.3)
17
+ domain_name (0.5.20170404)
18
+ unf (>= 0.0.5, < 1.0.0)
19
+ fluentd (0.14.19)
20
+ cool.io (>= 1.4.5, < 2.0.0)
21
+ http_parser.rb (>= 0.5.1, < 0.7.0)
22
+ msgpack (>= 0.7.0, < 2.0.0)
23
+ serverengine (>= 2.0.4, < 3.0.0)
24
+ sigdump (~> 0.2.2)
25
+ strptime (~> 0.1.7)
26
+ tzinfo (~> 1.0)
27
+ tzinfo-data (~> 1.0)
28
+ yajl-ruby (~> 1.0)
29
+ hashdiff (0.3.2)
30
+ http (2.2.2)
31
+ addressable (~> 2.3)
32
+ http-cookie (~> 1.0)
33
+ http-form_data (~> 1.0.1)
34
+ http_parser.rb (~> 0.6.0)
35
+ http-cookie (1.0.3)
36
+ domain_name (~> 0.5)
37
+ http-form_data (1.0.3)
38
+ http_parser.rb (0.6.0)
39
+ msgpack (1.1.0)
40
+ power_assert (0.2.6)
41
+ public_suffix (2.0.5)
42
+ rspec (3.6.0)
43
+ rspec-core (~> 3.6.0)
44
+ rspec-expectations (~> 3.6.0)
45
+ rspec-mocks (~> 3.6.0)
46
+ rspec-core (3.6.0)
47
+ rspec-support (~> 3.6.0)
48
+ rspec-expectations (3.6.0)
49
+ diff-lcs (>= 1.2.0, < 2.0)
50
+ rspec-support (~> 3.6.0)
51
+ rspec-mocks (3.6.0)
52
+ diff-lcs (>= 1.2.0, < 2.0)
53
+ rspec-support (~> 3.6.0)
54
+ rspec-support (3.6.0)
55
+ safe_yaml (1.0.4)
56
+ serverengine (2.0.5)
57
+ sigdump (~> 0.2.2)
58
+ sigdump (0.2.4)
59
+ strptime (0.1.9)
60
+ test-unit (3.1.5)
61
+ power_assert
62
+ thread_safe (0.3.6)
63
+ tzinfo (1.2.3)
64
+ thread_safe (~> 0.1)
65
+ tzinfo-data (1.2017.2)
66
+ tzinfo (>= 1.0.0)
67
+ unf (0.1.4)
68
+ unf_ext
69
+ unf_ext (0.0.7.4)
70
+ webmock (2.3.2)
71
+ addressable (>= 2.3.6)
72
+ crack (>= 0.3.2)
73
+ hashdiff
74
+ yajl-ruby (1.3.0)
75
+
76
+ PLATFORMS
77
+ ruby
78
+
79
+ DEPENDENCIES
80
+ fluent-plugin-timber!
81
+ rspec (~> 3.4)
82
+ test-unit (~> 3.1.0)
83
+ webmock (~> 2.3)
84
+
85
+ BUNDLED WITH
86
+ 1.15.1
@@ -0,0 +1,15 @@
1
+ # License
2
+
3
+ Copyright (c) 2017, Timber Technologies, Inc.
4
+
5
+ Permission to use, copy, modify, and/or distribute this software for any purpose
6
+ with or without fee is hereby granted, provided that the above copyright notice
7
+ and this permission notice appear in all copies.
8
+
9
+ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
10
+ REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
11
+ FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
12
+ INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
13
+ OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
14
+ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
15
+ THIS SOFTWARE.
@@ -0,0 +1,25 @@
1
+ # Fluent::Plugin::Timber, a plugin for [Fluentd](http://fluentd.org)
2
+
3
+ [![Gem Version](https://badge.fury.io/rb/fluent-plugin-timber.png)](http://badge.fury.io/rb/fluent-plugin-timber)
4
+ [![Build Status](https://travis-ci.org/timberio/fluent-plugin-timber.png?branch=master)](https://travis-ci.org/timberio/fluent-plugin-timber)
5
+ [![Issue Stats](http://issuestats.com/github/uken/fluent-plugin-timber/badge/pr)](http://issuestats.com/github/uken/fluent-plugin-timber)
6
+ [![Issue Stats](http://issuestats.com/github/uken/fluent-plugin-timber/badge/issue)](http://issuestats.com/github/uken/fluent-plugin-timber)
7
+
8
+ A highly efficient Fluentd plugin that delivers events to the [Timber.io logging service](https://timber.io). It uses batching, msgpack, and retry logic for highly efficient and reliable delivery of log data.
9
+
10
+ [Timber.io](https://timber.io) is a different kind of logging service with a focus on modern logging best-practices: easy setup, structured data, fast clean usable interface, 6 months of searchable retention, threshold based alerts, simple graphing, and more. Learn more at [https://timber.io](https://timber.io).
11
+
12
+ ```
13
+ <match your_match>
14
+ type timber
15
+ api_key xxxxxxxxxxxxxxxxxxxxxxxxxxx # Your Timber API (required)
16
+ hostname "#{Socket.gethostname}" # Your hostname (required)
17
+ # ip 127.0.0.1 # IP address (optional)
18
+ buffer_chunk_limit 1m # Must be < 5m
19
+ flush_at_shutdown true # Only needed with file buffer
20
+ </match>
21
+ ```
22
+
23
+ For advanced configuration options, please see to the [buffered output parameters documentation.](http://docs.fluentd.org/articles/output-plugin-overview#buffered-output-parameters).
24
+
25
+ Questions? Need help? [support@timber.io](mailto:support@timber.io).
@@ -0,0 +1,28 @@
1
+ # -*- encoding: utf-8 -*-
2
+ require 'date'
3
+
4
+ Gem::Specification.new do |s|
5
+ s.name = 'fluent-plugin-timber'
6
+ s.version = '1.0.0'
7
+ s.date = Date.today.to_s
8
+ s.summary = 'Timber.io plugin for Fluentd'
9
+ s.description = 'Streams Fluentd logs to the Timber.io logging service.'
10
+ s.authors = ['Timber.io']
11
+ s.email = 'hi@timber.io'
12
+ s.homepage = 'https://github.com/timberio/fluent-plugin-timber'
13
+ s.license = 'ISC'
14
+
15
+ s.files = `git ls-files`.split("\n")
16
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
17
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
18
+ s.require_paths = ["lib"]
19
+
20
+ s.required_ruby_version = Gem::Requirement.new(">= 2.0.0".freeze)
21
+
22
+ s.add_runtime_dependency('fluentd', '>= 0.12.0', '< 2')
23
+ s.add_runtime_dependency('http', '~> 2.0', '>= 2.0.3')
24
+
25
+ s.add_development_dependency('rspec', '~> 3.4')
26
+ s.add_development_dependency('test-unit', '~> 3.1.0')
27
+ s.add_development_dependency('webmock', '~> 2.3')
28
+ end
@@ -0,0 +1,86 @@
1
+ require 'base64'
2
+ require 'json'
3
+
4
+ require 'fluent/output'
5
+
6
+ module Fluent
7
+ class TimberOutput < Fluent::BufferedOutput
8
+ Fluent::Plugin.register_output('timber', self)
9
+
10
+ VERSION = "1.0.0".freeze
11
+ CONTENT_TYPE = "application/json".freeze
12
+ HOST = "https://logs.timber.io".freeze
13
+ MAX_ATTEMPTS = 3.freeze
14
+ PATH = "/frames".freeze
15
+ RETRYABLE_CODES = [429, 500, 502, 503, 504].freeze
16
+ USER_AGENT = "Timber Logstash/#{VERSION}".freeze
17
+
18
+ config_param :api_key, :string, secret: true
19
+ config_param :hostname, :string
20
+ config_param :ip, :string, default: nil
21
+
22
+ def configure(conf)
23
+ encoded_api_key = Base64.urlsafe_encode64(conf["api_key"]).chomp
24
+ authorization_value = "Basic #{encoded_api_key}"
25
+ @headers = {
26
+ "Authorization" => authorization_value,
27
+ "Content-Type" => CONTENT_TYPE,
28
+ "User-Agent" => USER_AGENT
29
+ }
30
+ super
31
+ end
32
+
33
+ def start
34
+ super
35
+ require 'http'
36
+ HTTP.default_options = {:keep_alive_timeout => 29}
37
+ @http_client = HTTP.persistent(HOST)
38
+ end
39
+
40
+ def shutdown
41
+ @http_client.close if @http_client
42
+ super
43
+ end
44
+
45
+ def format(tag, time, record)
46
+ dt_iso8601 = Time.at(time).utc.iso8601
47
+ record.merge("dt" => dt_iso8601).to_msgpack
48
+ end
49
+
50
+ def write(chunk)
51
+ deliver(chunk, 1)
52
+ end
53
+
54
+ private
55
+ def deliver(chunk, attempt)
56
+ if attempt > MAX_ATTEMPTS
57
+ log.error("msg=\"Max attempts exceeded dropping chunk\" attempt=#{attempt}")
58
+ return false
59
+ end
60
+
61
+ body = chunk.read
62
+ response = @http_client.headers(@headers).post(PATH, body: body)
63
+ response.flush
64
+ code = response.code
65
+
66
+ if code >= 200 && code <= 299
67
+ true
68
+ elsif RETRYABLE_CODES.include?(code)
69
+ sleep_time = sleep_for_attempt(attempt)
70
+ log.warn("msg=\"Retryable response from the Timber API\" " +
71
+ "code=#{code} attempt=#{attempt} sleep=#{sleep_time}")
72
+ sleep(sleep_time)
73
+ deliver(chunk, attempt + 1)
74
+ else
75
+ log.error("msg=\"Fatal response from the Timber API\" code=#{code} attempt=#{attempt}")
76
+ false
77
+ end
78
+ end
79
+
80
+ def sleep_for_attempt(attempt)
81
+ sleep_for = attempt ** 2
82
+ sleep_for = sleep_for <= 60 ? sleep_for : 60
83
+ (sleep_for / 2) + (rand(0..sleep_for) / 2)
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,64 @@
1
+ require "spec_helper"
2
+ require "fluent/plugin/out_timber"
3
+
4
+ describe Fluent::TimberOutput do
5
+ let(:config) do
6
+ %{
7
+ api_key abcd1234
8
+ hostname my.host.com
9
+ }
10
+ end
11
+
12
+ let(:driver) do
13
+ tag = "test"
14
+ Fluent::Test::BufferedOutputTestDriver.new(Fluent::TimberOutput, tag) {
15
+ # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
16
+ if !defined?(Fluent::Plugin::Output)
17
+ def format(tag, time, record)
18
+ [time, record].to_msgpack
19
+ end
20
+ end
21
+ }.configure(config)
22
+ end
23
+ let(:record) do
24
+ {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
25
+ end
26
+
27
+ before(:each) do
28
+ Fluent::Test.setup
29
+ end
30
+
31
+ describe "#write" do
32
+ it "should send a chunked request to the Timber API" do
33
+ stub = stub_request(:post, "https://logs.timber.io/frames").
34
+ with(
35
+ :body => start_with("\x85\xA3age\x1A\xAArequest_id\xA242\xA9parent_id\xA6parent\xAArouting_id\xA7routing\xA2dt\xB4".force_encoding("ASCII-8BIT")),
36
+ :headers => {'Authorization'=>'Basic YWJjZDEyMzQ=', 'Connection'=>'Keep-Alive', 'Content-Type'=>'application/json', 'User-Agent'=>'Timber Logstash/1.0.0'}
37
+ ).
38
+ to_return(:status => 200, :body => "", :headers => {})
39
+
40
+ driver.emit(record)
41
+ driver.run
42
+
43
+ expect(stub).to have_been_requested.times(1)
44
+ end
45
+
46
+ it "handles 500s" do
47
+ stub = stub_request(:post, "https://logs.timber.io/frames").to_return(:status => 500, :body => "", :headers => {})
48
+
49
+ driver.emit(record)
50
+ driver.run
51
+
52
+ expect(stub).to have_been_requested.times(3)
53
+ end
54
+
55
+ it "handle auth failures" do
56
+ stub = stub_request(:post, "https://logs.timber.io/frames").to_return(:status => 403, :body => "", :headers => {})
57
+
58
+ driver.emit(record)
59
+ driver.run
60
+
61
+ expect(stub).to have_been_requested.times(1)
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,20 @@
1
+ # Base
2
+ require 'rubygems'
3
+ require 'bundler/setup'
4
+
5
+ # Testing
6
+ require 'rspec'
7
+
8
+ # Webmock
9
+ require 'webmock/rspec'
10
+ WebMock.disable_net_connect!
11
+
12
+ # Fluent
13
+ require "fluent/test"
14
+
15
+ # Rspec
16
+ RSpec.configure do |config|
17
+ config.color = true
18
+ config.order = :random
19
+ config.warnings = false
20
+ end
metadata ADDED
@@ -0,0 +1,136 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-timber
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.0
5
+ platform: ruby
6
+ authors:
7
+ - Timber.io
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-07-16 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: 0.12.0
20
+ - - "<"
21
+ - !ruby/object:Gem::Version
22
+ version: '2'
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: 0.12.0
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: '2'
33
+ - !ruby/object:Gem::Dependency
34
+ name: http
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '2.0'
40
+ - - ">="
41
+ - !ruby/object:Gem::Version
42
+ version: 2.0.3
43
+ type: :runtime
44
+ prerelease: false
45
+ version_requirements: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - "~>"
48
+ - !ruby/object:Gem::Version
49
+ version: '2.0'
50
+ - - ">="
51
+ - !ruby/object:Gem::Version
52
+ version: 2.0.3
53
+ - !ruby/object:Gem::Dependency
54
+ name: rspec
55
+ requirement: !ruby/object:Gem::Requirement
56
+ requirements:
57
+ - - "~>"
58
+ - !ruby/object:Gem::Version
59
+ version: '3.4'
60
+ type: :development
61
+ prerelease: false
62
+ version_requirements: !ruby/object:Gem::Requirement
63
+ requirements:
64
+ - - "~>"
65
+ - !ruby/object:Gem::Version
66
+ version: '3.4'
67
+ - !ruby/object:Gem::Dependency
68
+ name: test-unit
69
+ requirement: !ruby/object:Gem::Requirement
70
+ requirements:
71
+ - - "~>"
72
+ - !ruby/object:Gem::Version
73
+ version: 3.1.0
74
+ type: :development
75
+ prerelease: false
76
+ version_requirements: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - "~>"
79
+ - !ruby/object:Gem::Version
80
+ version: 3.1.0
81
+ - !ruby/object:Gem::Dependency
82
+ name: webmock
83
+ requirement: !ruby/object:Gem::Requirement
84
+ requirements:
85
+ - - "~>"
86
+ - !ruby/object:Gem::Version
87
+ version: '2.3'
88
+ type: :development
89
+ prerelease: false
90
+ version_requirements: !ruby/object:Gem::Requirement
91
+ requirements:
92
+ - - "~>"
93
+ - !ruby/object:Gem::Version
94
+ version: '2.3'
95
+ description: Streams Fluentd logs to the Timber.io logging service.
96
+ email: hi@timber.io
97
+ executables: []
98
+ extensions: []
99
+ extra_rdoc_files: []
100
+ files:
101
+ - ".gitignore"
102
+ - Gemfile
103
+ - Gemfile.lock
104
+ - LICENSE.md
105
+ - README.md
106
+ - fluent-plugin-timber.gemspec
107
+ - lib/fluent/plugin/out_timber.rb
108
+ - spec/fluent/plugin/out_timber_spec.rb
109
+ - spec/spec_helper.rb
110
+ homepage: https://github.com/timberio/fluent-plugin-timber
111
+ licenses:
112
+ - ISC
113
+ metadata: {}
114
+ post_install_message:
115
+ rdoc_options: []
116
+ require_paths:
117
+ - lib
118
+ required_ruby_version: !ruby/object:Gem::Requirement
119
+ requirements:
120
+ - - ">="
121
+ - !ruby/object:Gem::Version
122
+ version: 2.0.0
123
+ required_rubygems_version: !ruby/object:Gem::Requirement
124
+ requirements:
125
+ - - ">="
126
+ - !ruby/object:Gem::Version
127
+ version: '0'
128
+ requirements: []
129
+ rubyforge_project:
130
+ rubygems_version: 2.5.2
131
+ signing_key:
132
+ specification_version: 4
133
+ summary: Timber.io plugin for Fluentd
134
+ test_files:
135
+ - spec/fluent/plugin/out_timber_spec.rb
136
+ - spec/spec_helper.rb