fluent-plugin-logtail 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: e0c8596cf46b0faa3a9d7e580d453d0cec5af12bd1b75d42232f72c57a834a7b
4
+ data.tar.gz: 60b7f73e13bedafe63b2cbfb80338ba15644387c292312471cff7247e333df5d
5
+ SHA512:
6
+ metadata.gz: 4103a98491a69624c5354facafd04735c3ba85a2ceb2d8c179435e46f9c82e0ce3376287717e830635c75842ee22aed02822d2cf1afe1f1a931dc820ae2cc063
7
+ data.tar.gz: 83bb7f6e2b67ad1700f1b59b1f52efaa4d33d386e671e98af0dd7b8686df93bb44188f66a7ec42dfad10412df7bbe47323894cf11e39314e8e1fc026f0e98629
@@ -0,0 +1,29 @@
1
+ name: build
2
+
3
+ on: [push, pull_request]
4
+
5
+ jobs:
6
+ test:
7
+
8
+ runs-on: ubuntu-20.04
9
+
10
+ strategy:
11
+ matrix:
12
+ ruby-version:
13
+ - 3.0.0
14
+ - 2.7.2
15
+ - 2.6.6
16
+ - 2.5.8
17
+ - 2.4.10
18
+
19
+ steps:
20
+ - uses: actions/checkout@v2
21
+
22
+ - name: Set up Ruby ${{ matrix.ruby-version }}
23
+ uses: ruby/setup-ruby@v1
24
+ with:
25
+ ruby-version: ${{ matrix.ruby-version }}
26
+ bundler-cache: true
27
+
28
+ - name: Run tests
29
+ run: bundle exec rspec
data/.gitignore ADDED
@@ -0,0 +1,16 @@
1
+ .DS_Store
2
+ .rvmrc
3
+ .ruby-version
4
+ coverage
5
+ *.swp
6
+ *.gem
7
+ Gemfile.lock
8
+
9
+ gemfiles/*.lock
10
+
11
+ /.bundle
12
+ /.yardoc
13
+ /doc
14
+ /log
15
+ /tmp
16
+ /pkg
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
data/LICENSE.md ADDED
@@ -0,0 +1,15 @@
1
+ # License
2
+
3
+ Copyright (c) 2021, Logtail
4
+
5
+ Permission to use, copy, modify, and/or distribute this software for any purpose
6
+ with or without fee is hereby granted, provided that the above copyright notice
7
+ and this permission notice appear in all copies.
8
+
9
+ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
10
+ REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
11
+ FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
12
+ INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
13
+ OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
14
+ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
15
+ THIS SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,31 @@
1
+ # 🪵 Fluent::Plugin::Logtail, a plugin for [Fluentd](http://fluentd.org)
2
+
3
+ [![build](https://github.com/logtail/fluentd-plugin-logtail/actions/workflows/main.yml/badge.svg)](https://github.com/logtail/fluentd-plugin-logtail/actions/workflows/main.yml)
4
+
5
+ A Fluentd plugin that delivers events to the [Logtail.com logging service](https://logtail.com). It uses batching, msgpack, and retry logic for highly efficient and reliable delivery of log data.
6
+
7
+ ## Installation
8
+
9
+ ```
10
+ gem install fluent-plugin-logtail
11
+ ```
12
+
13
+ ## Usage
14
+
15
+ In your Fluentd configuration, use `@type logtail`:
16
+
17
+ ```
18
+ <match your_match>
19
+ @type logtail
20
+ source_token YOUR_SOURCE_TOKEN
21
+ # ip 127.0.0.1
22
+ buffer_chunk_limit 1m # Must be < 5m
23
+ flush_at_shutdown true # Only needed with file buffer
24
+ </match>
25
+ ```
26
+
27
+ ## Configuration
28
+
29
+ * `source_token` - This is your [Logtail source token](https://logtail.com).
30
+
31
+ For advanced configuration options, please see to the [buffered output parameters documentation.](http://docs.fluentd.org/articles/output-plugin-overview#buffered-output-parameters).
@@ -0,0 +1,28 @@
1
+ # -*- encoding: utf-8 -*-
2
+ require 'date'
3
+
4
+ Gem::Specification.new do |s|
5
+ s.name = 'fluent-plugin-logtail'
6
+ s.version = '0.1.0'
7
+ s.date = Date.today.to_s
8
+ s.summary = 'Logtail.com plugin for Fluentd'
9
+ s.description = 'Streams Fluentd logs to the Logtail.com logging service.'
10
+ s.authors = ['Logtail.com']
11
+ s.email = 'hello@logtail.com'
12
+ s.homepage = 'https://github.com/logtail/fluent-plugin-logtail'
13
+ s.license = 'ISC'
14
+
15
+ s.files = `git ls-files`.split("\n")
16
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
17
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
18
+ s.require_paths = ["lib"]
19
+
20
+ s.required_ruby_version = Gem::Requirement.new(">= 2.4.0".freeze)
21
+
22
+ s.add_runtime_dependency('fluentd', '> 1', '< 2')
23
+ s.add_runtime_dependency('http', '~> 2.0', '>= 2.0.3')
24
+
25
+ s.add_development_dependency('rspec', '~> 3.4')
26
+ s.add_development_dependency('test-unit', '~> 3.3.9')
27
+ s.add_development_dependency('webmock', '~> 2.3')
28
+ end
@@ -0,0 +1,80 @@
1
+ require 'fluent/output'
2
+
3
+ module Fluent
4
+ class LogtailOutput < Fluent::BufferedOutput
5
+ Fluent::Plugin.register_output('logtail', self)
6
+
7
+ VERSION = "0.1.0".freeze
8
+ CONTENT_TYPE = "application/msgpack".freeze
9
+ HOST = "https://in.logtail.com".freeze
10
+ PATH = "/".freeze
11
+ MAX_ATTEMPTS = 3.freeze
12
+ RETRYABLE_CODES = [429, 500, 502, 503, 504].freeze
13
+ USER_AGENT = "Logtail Logstash/#{VERSION}".freeze
14
+
15
+ config_param :source_token, :string, secret: true
16
+ config_param :ip, :string, default: nil
17
+
18
+ def configure(conf)
19
+ source_token = conf["source_token"]
20
+ @headers = {
21
+ "Authorization" => "Bearer #{source_token}",
22
+ "Content-Type" => CONTENT_TYPE,
23
+ "User-Agent" => USER_AGENT
24
+ }
25
+ super
26
+ end
27
+
28
+ def start
29
+ super
30
+ require 'http'
31
+ HTTP.default_options = {:keep_alive_timeout => 29}
32
+ @http_client = HTTP.persistent(HOST)
33
+ end
34
+
35
+ def shutdown
36
+ @http_client.close if @http_client
37
+ super
38
+ end
39
+
40
+ def format(tag, time, record)
41
+ record.merge("dt" => Time.at(time).utc.iso8601).to_msgpack
42
+ end
43
+
44
+ def write(chunk)
45
+ deliver(chunk, 1)
46
+ end
47
+
48
+ private
49
+ def deliver(chunk, attempt)
50
+ if attempt > MAX_ATTEMPTS
51
+ log.error("msg=\"Max attempts exceeded dropping chunk\" attempt=#{attempt}")
52
+ return false
53
+ end
54
+
55
+ body = chunk.read
56
+ response = @http_client.headers(@headers).post(PATH, body: body)
57
+ response.flush
58
+ code = response.code
59
+
60
+ if code >= 200 && code <= 299
61
+ true
62
+ elsif RETRYABLE_CODES.include?(code)
63
+ sleep_time = sleep_for_attempt(attempt)
64
+ log.warn("msg=\"Retryable response from the Logtail API\" " +
65
+ "code=#{code} attempt=#{attempt} sleep=#{sleep_time}")
66
+ sleep(sleep_time)
67
+ deliver(chunk, attempt + 1)
68
+ else
69
+ log.error("msg=\"Fatal response from the Logtail API\" code=#{code} attempt=#{attempt}")
70
+ false
71
+ end
72
+ end
73
+
74
+ def sleep_for_attempt(attempt)
75
+ sleep_for = attempt ** 2
76
+ sleep_for = sleep_for <= 60 ? sleep_for : 60
77
+ (sleep_for / 2) + (rand(0..sleep_for) / 2)
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,63 @@
1
+ require "spec_helper"
2
+ require "fluent/plugin/out_logtail"
3
+
4
+ describe Fluent::LogtailOutput do
5
+ let(:config) do
6
+ %{
7
+ source_token abcd1234
8
+ }
9
+ end
10
+
11
+ let(:driver) do
12
+ tag = "test"
13
+ Fluent::Test::BufferedOutputTestDriver.new(Fluent::LogtailOutput, tag) {
14
+ # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
15
+ if !defined?(Fluent::Plugin::Output)
16
+ def format(tag, time, record)
17
+ [time, record].to_msgpack
18
+ end
19
+ end
20
+ }.configure(config)
21
+ end
22
+ let(:record) do
23
+ {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
24
+ end
25
+
26
+ before(:each) do
27
+ Fluent::Test.setup
28
+ end
29
+
30
+ describe "#write" do
31
+ it "should send a chunked request to the Logtail API" do
32
+ stub = stub_request(:post, "https://in.logtail.com/").
33
+ with(
34
+ :body => start_with("\x85\xA3age\x1A\xAArequest_id\xA242\xA9parent_id\xA6parent\xAArouting_id\xA7routing\xA2dt\xB4".force_encoding("ASCII-8BIT")),
35
+ :headers => {'Authorization'=>'Bearer abcd1234', 'Connection'=>'Keep-Alive', 'Content-Type'=>'application/msgpack', 'User-Agent'=>'Logtail Logstash/0.1.0'}
36
+ ).
37
+ to_return(:status => 200, :body => "", :headers => {})
38
+
39
+ driver.emit(record)
40
+ driver.run
41
+
42
+ expect(stub).to have_been_requested.times(1)
43
+ end
44
+
45
+ it "handles 500s" do
46
+ stub = stub_request(:post, "https://in.logtail.com/").to_return(:status => 500, :body => "", :headers => {})
47
+
48
+ driver.emit(record)
49
+ driver.run
50
+
51
+ expect(stub).to have_been_requested.times(3)
52
+ end
53
+
54
+ it "handle auth failures" do
55
+ stub = stub_request(:post, "https://in.logtail.com/").to_return(:status => 403, :body => "", :headers => {})
56
+
57
+ driver.emit(record)
58
+ driver.run
59
+
60
+ expect(stub).to have_been_requested.times(1)
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,20 @@
1
+ # Base
2
+ require 'rubygems'
3
+ require 'bundler/setup'
4
+
5
+ # Testing
6
+ require 'rspec'
7
+
8
+ # Webmock
9
+ require 'webmock/rspec'
10
+ WebMock.disable_net_connect!
11
+
12
+ # Fluent
13
+ require "fluent/test"
14
+
15
+ # Rspec
16
+ RSpec.configure do |config|
17
+ config.color = true
18
+ config.order = :random
19
+ config.warnings = false
20
+ end
metadata ADDED
@@ -0,0 +1,135 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-logtail
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Logtail.com
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2021-03-31 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">"
18
+ - !ruby/object:Gem::Version
19
+ version: '1'
20
+ - - "<"
21
+ - !ruby/object:Gem::Version
22
+ version: '2'
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">"
28
+ - !ruby/object:Gem::Version
29
+ version: '1'
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: '2'
33
+ - !ruby/object:Gem::Dependency
34
+ name: http
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '2.0'
40
+ - - ">="
41
+ - !ruby/object:Gem::Version
42
+ version: 2.0.3
43
+ type: :runtime
44
+ prerelease: false
45
+ version_requirements: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - "~>"
48
+ - !ruby/object:Gem::Version
49
+ version: '2.0'
50
+ - - ">="
51
+ - !ruby/object:Gem::Version
52
+ version: 2.0.3
53
+ - !ruby/object:Gem::Dependency
54
+ name: rspec
55
+ requirement: !ruby/object:Gem::Requirement
56
+ requirements:
57
+ - - "~>"
58
+ - !ruby/object:Gem::Version
59
+ version: '3.4'
60
+ type: :development
61
+ prerelease: false
62
+ version_requirements: !ruby/object:Gem::Requirement
63
+ requirements:
64
+ - - "~>"
65
+ - !ruby/object:Gem::Version
66
+ version: '3.4'
67
+ - !ruby/object:Gem::Dependency
68
+ name: test-unit
69
+ requirement: !ruby/object:Gem::Requirement
70
+ requirements:
71
+ - - "~>"
72
+ - !ruby/object:Gem::Version
73
+ version: 3.3.9
74
+ type: :development
75
+ prerelease: false
76
+ version_requirements: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - "~>"
79
+ - !ruby/object:Gem::Version
80
+ version: 3.3.9
81
+ - !ruby/object:Gem::Dependency
82
+ name: webmock
83
+ requirement: !ruby/object:Gem::Requirement
84
+ requirements:
85
+ - - "~>"
86
+ - !ruby/object:Gem::Version
87
+ version: '2.3'
88
+ type: :development
89
+ prerelease: false
90
+ version_requirements: !ruby/object:Gem::Requirement
91
+ requirements:
92
+ - - "~>"
93
+ - !ruby/object:Gem::Version
94
+ version: '2.3'
95
+ description: Streams Fluentd logs to the Logtail.com logging service.
96
+ email: hello@logtail.com
97
+ executables: []
98
+ extensions: []
99
+ extra_rdoc_files: []
100
+ files:
101
+ - ".github/workflows/main.yml"
102
+ - ".gitignore"
103
+ - Gemfile
104
+ - LICENSE.md
105
+ - README.md
106
+ - fluent-plugin-logtail.gemspec
107
+ - lib/fluent/plugin/out_logtail.rb
108
+ - spec/fluent/plugin/out_logtail_spec.rb
109
+ - spec/spec_helper.rb
110
+ homepage: https://github.com/logtail/fluent-plugin-logtail
111
+ licenses:
112
+ - ISC
113
+ metadata: {}
114
+ post_install_message:
115
+ rdoc_options: []
116
+ require_paths:
117
+ - lib
118
+ required_ruby_version: !ruby/object:Gem::Requirement
119
+ requirements:
120
+ - - ">="
121
+ - !ruby/object:Gem::Version
122
+ version: 2.4.0
123
+ required_rubygems_version: !ruby/object:Gem::Requirement
124
+ requirements:
125
+ - - ">="
126
+ - !ruby/object:Gem::Version
127
+ version: '0'
128
+ requirements: []
129
+ rubygems_version: 3.2.3
130
+ signing_key:
131
+ specification_version: 4
132
+ summary: Logtail.com plugin for Fluentd
133
+ test_files:
134
+ - spec/fluent/plugin/out_logtail_spec.rb
135
+ - spec/spec_helper.rb