fluent-plugin-logfire 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.github/workflows/main.yml +29 -0
- data/.gitignore +16 -0
- data/Gemfile +2 -0
- data/README.md +29 -0
- data/fluent-plugin-logfire.gemspec +27 -0
- data/lib/fluent/plugin/out_logfire.rb +113 -0
- data/spec/fluent/plugin/out_logfire_spec.rb +63 -0
- data/spec/spec_helper.rb +20 -0
- metadata +112 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: d9b6492707112a0f4a156fa9c5bc8a7c57f3502c69baae9580581ad4b9c58a69
|
4
|
+
data.tar.gz: 4b95bd9dfaf66be22c318fd672438214bf3e0c0790a58a17d428038c3de5af6e
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: eb1ac204dcc1fecc38c0890cd908443d414033c56d876d0782fe66a2d63f27cba35f07b215f7d8da9e26ef57012a48ee2d97733f1c60399a1c015935b193e7cc
|
7
|
+
data.tar.gz: 2a6741d11a61b7718a17b0d556d9de2cf9e4b50340cac8dc82f03a2db992eefc819999ee85d7f1b061df581388b43295aff2ba5a3276debe90fe21eefab76140
|
@@ -0,0 +1,29 @@
|
|
1
|
+
name: build
|
2
|
+
|
3
|
+
on: [push, pull_request]
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
test:
|
7
|
+
|
8
|
+
runs-on: ubuntu-22.04
|
9
|
+
|
10
|
+
strategy:
|
11
|
+
matrix:
|
12
|
+
ruby-version:
|
13
|
+
- 3.0.0
|
14
|
+
- 2.7.2
|
15
|
+
- 2.6.6
|
16
|
+
- 2.5.8
|
17
|
+
- 2.4.10
|
18
|
+
|
19
|
+
steps:
|
20
|
+
- uses: actions/checkout@v2
|
21
|
+
|
22
|
+
- name: Set up Ruby ${{ matrix.ruby-version }}
|
23
|
+
uses: ruby/setup-ruby@v1
|
24
|
+
with:
|
25
|
+
ruby-version: ${{ matrix.ruby-version }}
|
26
|
+
bundler-cache: true
|
27
|
+
|
28
|
+
- name: Run tests
|
29
|
+
run: bundle exec rspec
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# 🪵 Fluent::Plugin::logfire, a plugin for [Fluentd](http://fluentd.org)
|
2
|
+
|
3
|
+
A Fluentd plugin that delivers events to the [logfire.sh logging service](https://logfire.sh).
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
```
|
8
|
+
gem install fluent-plugin-logfire
|
9
|
+
```
|
10
|
+
|
11
|
+
## Usage
|
12
|
+
|
13
|
+
In your Fluentd configuration, use `@type logfire`:
|
14
|
+
|
15
|
+
```
|
16
|
+
<match your_match>
|
17
|
+
@type logfire
|
18
|
+
source_token YOUR_SOURCE_TOKEN
|
19
|
+
# ip 127.0.0.1
|
20
|
+
buffer_chunk_limit 1m # Must be < 5m
|
21
|
+
flush_at_shutdown true # Only needed with file buffer
|
22
|
+
</match>
|
23
|
+
```
|
24
|
+
|
25
|
+
## Configuration
|
26
|
+
|
27
|
+
* `source_token` - This is your [logfire source token](https://logfire.sh) whithout the Bearer keyword.
|
28
|
+
|
29
|
+
For advanced configuration options, please see to the [buffered output parameters documentation.](http://docs.fluentd.org/articles/output-plugin-overview#buffered-output-parameters).
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require 'date'
|
3
|
+
|
4
|
+
Gem::Specification.new do |s|
|
5
|
+
s.name = 'fluent-plugin-logfire'
|
6
|
+
s.version = '0.1.0'
|
7
|
+
s.date = Date.today.to_s
|
8
|
+
s.summary = 'logfire.sh plugin for fluentd'
|
9
|
+
s.description = 'Streams fluentd logs to the logfire.sh logging service.'
|
10
|
+
s.authors = ['logfire.sh']
|
11
|
+
s.email = 'hello@logfire.sh'
|
12
|
+
s.homepage = 'https://github.com/logfire-sh/logfire-fluentd-plugin-private'
|
13
|
+
s.license = 'ISC'
|
14
|
+
|
15
|
+
s.files = `git ls-files`.split("\n")
|
16
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
|
+
s.require_paths = ["lib"]
|
19
|
+
|
20
|
+
s.required_ruby_version = Gem::Requirement.new(">= 2.4.0".freeze)
|
21
|
+
|
22
|
+
s.add_runtime_dependency('fluentd', '> 1', '< 2')
|
23
|
+
|
24
|
+
s.add_development_dependency('rspec', '~> 3.4')
|
25
|
+
s.add_development_dependency('test-unit', '~> 3.3.9')
|
26
|
+
s.add_development_dependency('webmock', '~> 2.3')
|
27
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
require 'fluent/output'
|
2
|
+
require 'net/https'
|
3
|
+
|
4
|
+
module Fluent
|
5
|
+
class LogfireOutput < Fluent::BufferedOutput
|
6
|
+
Fluent::Plugin.register_output('logfire', self)
|
7
|
+
|
8
|
+
VERSION = "0.1.0".freeze
|
9
|
+
CONTENT_TYPE = "application/msgpack".freeze
|
10
|
+
HOST = "in.logfire.sh".freeze
|
11
|
+
PORT = 443
|
12
|
+
PATH = "/".freeze
|
13
|
+
MAX_ATTEMPTS = 3.freeze
|
14
|
+
RETRYABLE_CODES = [429, 500, 502, 503, 504].freeze
|
15
|
+
USER_AGENT = "logfire Fluentd/#{VERSION}".freeze
|
16
|
+
|
17
|
+
config_param :source_token, :string, secret: true
|
18
|
+
config_param :ip, :string, default: nil
|
19
|
+
|
20
|
+
def configure(conf)
|
21
|
+
@source_token = conf["source_token"]
|
22
|
+
super
|
23
|
+
end
|
24
|
+
|
25
|
+
def format(tag, time, record)
|
26
|
+
force_utf8_string_values(record.merge("dt" => Time.at(time).utc.iso8601)).to_msgpack
|
27
|
+
end
|
28
|
+
|
29
|
+
def write(chunk)
|
30
|
+
deliver(chunk, 1)
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
def deliver(chunk, attempt)
|
35
|
+
if attempt > MAX_ATTEMPTS
|
36
|
+
log.error("msg=\"Max attempts exceeded dropping chunk\" attempt=#{attempt}")
|
37
|
+
return false
|
38
|
+
end
|
39
|
+
|
40
|
+
http = build_http_client
|
41
|
+
records=0
|
42
|
+
chunk.each do
|
43
|
+
records=records+1
|
44
|
+
end
|
45
|
+
body = [0xdd,records].pack("CN")
|
46
|
+
body << chunk.read
|
47
|
+
|
48
|
+
begin
|
49
|
+
resp = http.start do |conn|
|
50
|
+
req = build_request(body)
|
51
|
+
log.debug("sending #{req.body.length} bytes to logfire")
|
52
|
+
conn.request(req)
|
53
|
+
end
|
54
|
+
ensure
|
55
|
+
http.finish if http.started?
|
56
|
+
end
|
57
|
+
|
58
|
+
code = resp.code.to_i
|
59
|
+
if code >= 200 && code <= 299
|
60
|
+
log.debug "POST request to logfire was responded to with status code #{code}"
|
61
|
+
true
|
62
|
+
elsif RETRYABLE_CODES.include?(code)
|
63
|
+
sleep_time = sleep_for_attempt(attempt)
|
64
|
+
log.warn("msg=\"Retryable response from the logfire API\" " +
|
65
|
+
"code=#{code} attempt=#{attempt} sleep=#{sleep_time}")
|
66
|
+
sleep(sleep_time)
|
67
|
+
deliver(chunk, attempt + 1)
|
68
|
+
else
|
69
|
+
log.error("msg=\"Fatal response from the logfire API\" code=#{code} attempt=#{attempt}")
|
70
|
+
false
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def sleep_for_attempt(attempt)
|
75
|
+
sleep_for = attempt ** 2
|
76
|
+
sleep_for = sleep_for <= 60 ? sleep_for : 60
|
77
|
+
(sleep_for / 2) + (rand(0..sleep_for) / 2)
|
78
|
+
end
|
79
|
+
|
80
|
+
def force_utf8_string_values(data)
|
81
|
+
data.transform_values do |val|
|
82
|
+
if val.is_a?(Hash)
|
83
|
+
force_utf8_string_values(val)
|
84
|
+
elsif val.respond_to?(:force_encoding)
|
85
|
+
val.force_encoding('UTF-8')
|
86
|
+
else
|
87
|
+
val
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def build_http_client
|
93
|
+
http = Net::HTTP.new(HOST, PORT)
|
94
|
+
http.use_ssl = true
|
95
|
+
# Verification on Windows fails despite having a valid certificate.
|
96
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
97
|
+
http.read_timeout = 30
|
98
|
+
http.ssl_timeout = 10
|
99
|
+
http.open_timeout = 10
|
100
|
+
http
|
101
|
+
end
|
102
|
+
|
103
|
+
def build_request(body)
|
104
|
+
path = '/'
|
105
|
+
req = Net::HTTP::Post.new(path)
|
106
|
+
req["Authorization"] = "Bearer #{@source_token}"
|
107
|
+
req["Content-Type"] = CONTENT_TYPE
|
108
|
+
req["User-Agent"] = USER_AGENT
|
109
|
+
req.body = body
|
110
|
+
req
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "fluent/plugin/out_logfire"
|
3
|
+
|
4
|
+
describe Fluent::LogfireOutput do
|
5
|
+
let(:config) do
|
6
|
+
%{
|
7
|
+
source_token abcd1234
|
8
|
+
}
|
9
|
+
end
|
10
|
+
|
11
|
+
let(:driver) do
|
12
|
+
tag = "test"
|
13
|
+
Fluent::Test::BufferedOutputTestDriver.new(Fluent::LogfireOutput, tag) {
|
14
|
+
# v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
|
15
|
+
if !defined?(Fluent::Plugin::Output)
|
16
|
+
def format(tag, time, record)
|
17
|
+
[time, record].to_msgpack
|
18
|
+
end
|
19
|
+
end
|
20
|
+
}.configure(config)
|
21
|
+
end
|
22
|
+
let(:record) do
|
23
|
+
{'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
|
24
|
+
end
|
25
|
+
|
26
|
+
before(:each) do
|
27
|
+
Fluent::Test.setup
|
28
|
+
end
|
29
|
+
|
30
|
+
describe "#write" do
|
31
|
+
it "should send a chunked request to the logfire API" do
|
32
|
+
stub = stub_request(:post, "https://in.logfire.sh/").
|
33
|
+
with(
|
34
|
+
:body => start_with("\xDD\x00\x00\x00\x01\x85\xA3age\x1A\xAArequest_id\xA242\xA9parent_id\xA6parent\xAArouting_id\xA7routing\xA2dt\xB4".force_encoding("ASCII-8BIT")),
|
35
|
+
:headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Authorization'=>'Bearer abcd1234', 'Content-Type'=>'application/msgpack', 'User-Agent'=>'logfire Fluentd/0.1.1'}
|
36
|
+
).
|
37
|
+
to_return(:status => 202, :body => "", :headers => {})
|
38
|
+
|
39
|
+
driver.emit(record)
|
40
|
+
driver.run
|
41
|
+
|
42
|
+
expect(stub).to have_been_requested.times(1)
|
43
|
+
end
|
44
|
+
|
45
|
+
it "handles 500s" do
|
46
|
+
stub = stub_request(:post, "https://in.logfire.sh/").to_return(:status => 500, :body => "", :headers => {})
|
47
|
+
|
48
|
+
driver.emit(record)
|
49
|
+
driver.run
|
50
|
+
|
51
|
+
expect(stub).to have_been_requested.times(3)
|
52
|
+
end
|
53
|
+
|
54
|
+
it "handle auth failures" do
|
55
|
+
stub = stub_request(:post, "https://in.logfire.sh/").to_return(:status => 403, :body => "", :headers => {})
|
56
|
+
|
57
|
+
driver.emit(record)
|
58
|
+
driver.run
|
59
|
+
|
60
|
+
expect(stub).to have_been_requested.times(1)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# Base
|
2
|
+
require 'rubygems'
|
3
|
+
require 'bundler/setup'
|
4
|
+
|
5
|
+
# Testing
|
6
|
+
require 'rspec'
|
7
|
+
|
8
|
+
# Webmock
|
9
|
+
require 'webmock/rspec'
|
10
|
+
WebMock.disable_net_connect!
|
11
|
+
|
12
|
+
# Fluent
|
13
|
+
require "fluent/test"
|
14
|
+
|
15
|
+
# Rspec
|
16
|
+
RSpec.configure do |config|
|
17
|
+
config.color = true
|
18
|
+
config.order = :random
|
19
|
+
config.warnings = false
|
20
|
+
end
|
metadata
ADDED
@@ -0,0 +1,112 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-logfire
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- logfire.sh
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2023-06-15 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1'
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '2'
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">"
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1'
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '2'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: rspec
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - "~>"
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '3.4'
|
40
|
+
type: :development
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - "~>"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '3.4'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: test-unit
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - "~>"
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 3.3.9
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - "~>"
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: 3.3.9
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: webmock
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - "~>"
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '2.3'
|
68
|
+
type: :development
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - "~>"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '2.3'
|
75
|
+
description: Streams fluentd logs to the logfire.sh logging service.
|
76
|
+
email: hello@logfire.sh
|
77
|
+
executables: []
|
78
|
+
extensions: []
|
79
|
+
extra_rdoc_files: []
|
80
|
+
files:
|
81
|
+
- ".github/workflows/main.yml"
|
82
|
+
- ".gitignore"
|
83
|
+
- Gemfile
|
84
|
+
- README.md
|
85
|
+
- fluent-plugin-logfire.gemspec
|
86
|
+
- lib/fluent/plugin/out_logfire.rb
|
87
|
+
- spec/fluent/plugin/out_logfire_spec.rb
|
88
|
+
- spec/spec_helper.rb
|
89
|
+
homepage: https://github.com/logfire-sh/logfire-fluentd-plugin-private
|
90
|
+
licenses:
|
91
|
+
- ISC
|
92
|
+
metadata: {}
|
93
|
+
post_install_message:
|
94
|
+
rdoc_options: []
|
95
|
+
require_paths:
|
96
|
+
- lib
|
97
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
98
|
+
requirements:
|
99
|
+
- - ">="
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: 2.4.0
|
102
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
103
|
+
requirements:
|
104
|
+
- - ">="
|
105
|
+
- !ruby/object:Gem::Version
|
106
|
+
version: '0'
|
107
|
+
requirements: []
|
108
|
+
rubygems_version: 3.1.6
|
109
|
+
signing_key:
|
110
|
+
specification_version: 4
|
111
|
+
summary: logfire.sh plugin for fluentd
|
112
|
+
test_files: []
|