fluent-plugin-logfire 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.github/workflows/main.yml +29 -0
- data/.gitignore +16 -0
- data/Gemfile +2 -0
- data/README.md +29 -0
- data/fluent-plugin-logfire.gemspec +27 -0
- data/lib/fluent/plugin/out_logfire.rb +113 -0
- data/spec/fluent/plugin/out_logfire_spec.rb +63 -0
- data/spec/spec_helper.rb +20 -0
- metadata +112 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: d9b6492707112a0f4a156fa9c5bc8a7c57f3502c69baae9580581ad4b9c58a69
|
4
|
+
data.tar.gz: 4b95bd9dfaf66be22c318fd672438214bf3e0c0790a58a17d428038c3de5af6e
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: eb1ac204dcc1fecc38c0890cd908443d414033c56d876d0782fe66a2d63f27cba35f07b215f7d8da9e26ef57012a48ee2d97733f1c60399a1c015935b193e7cc
|
7
|
+
data.tar.gz: 2a6741d11a61b7718a17b0d556d9de2cf9e4b50340cac8dc82f03a2db992eefc819999ee85d7f1b061df581388b43295aff2ba5a3276debe90fe21eefab76140
|
@@ -0,0 +1,29 @@
|
|
1
|
+
name: build
|
2
|
+
|
3
|
+
on: [push, pull_request]
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
test:
|
7
|
+
|
8
|
+
runs-on: ubuntu-22.04
|
9
|
+
|
10
|
+
strategy:
|
11
|
+
matrix:
|
12
|
+
ruby-version:
|
13
|
+
- 3.0.0
|
14
|
+
- 2.7.2
|
15
|
+
- 2.6.6
|
16
|
+
- 2.5.8
|
17
|
+
- 2.4.10
|
18
|
+
|
19
|
+
steps:
|
20
|
+
- uses: actions/checkout@v2
|
21
|
+
|
22
|
+
- name: Set up Ruby ${{ matrix.ruby-version }}
|
23
|
+
uses: ruby/setup-ruby@v1
|
24
|
+
with:
|
25
|
+
ruby-version: ${{ matrix.ruby-version }}
|
26
|
+
bundler-cache: true
|
27
|
+
|
28
|
+
- name: Run tests
|
29
|
+
run: bundle exec rspec
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# 🪵 Fluent::Plugin::logfire, a plugin for [Fluentd](http://fluentd.org)
|
2
|
+
|
3
|
+
A Fluentd plugin that delivers events to the [logfire.sh logging service](https://logfire.sh).
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
```
|
8
|
+
gem install fluent-plugin-logfire
|
9
|
+
```
|
10
|
+
|
11
|
+
## Usage
|
12
|
+
|
13
|
+
In your Fluentd configuration, use `@type logfire`:
|
14
|
+
|
15
|
+
```
|
16
|
+
<match your_match>
|
17
|
+
@type logfire
|
18
|
+
source_token YOUR_SOURCE_TOKEN
|
19
|
+
# ip 127.0.0.1
|
20
|
+
buffer_chunk_limit 1m # Must be < 5m
|
21
|
+
flush_at_shutdown true # Only needed with file buffer
|
22
|
+
</match>
|
23
|
+
```
|
24
|
+
|
25
|
+
## Configuration
|
26
|
+
|
27
|
+
* `source_token` - This is your [logfire source token](https://logfire.sh) whithout the Bearer keyword.
|
28
|
+
|
29
|
+
For advanced configuration options, please see to the [buffered output parameters documentation.](http://docs.fluentd.org/articles/output-plugin-overview#buffered-output-parameters).
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require 'date'
|
3
|
+
|
4
|
+
Gem::Specification.new do |s|
|
5
|
+
s.name = 'fluent-plugin-logfire'
|
6
|
+
s.version = '0.1.0'
|
7
|
+
s.date = Date.today.to_s
|
8
|
+
s.summary = 'logfire.sh plugin for fluentd'
|
9
|
+
s.description = 'Streams fluentd logs to the logfire.sh logging service.'
|
10
|
+
s.authors = ['logfire.sh']
|
11
|
+
s.email = 'hello@logfire.sh'
|
12
|
+
s.homepage = 'https://github.com/logfire-sh/logfire-fluentd-plugin-private'
|
13
|
+
s.license = 'ISC'
|
14
|
+
|
15
|
+
s.files = `git ls-files`.split("\n")
|
16
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
|
+
s.require_paths = ["lib"]
|
19
|
+
|
20
|
+
s.required_ruby_version = Gem::Requirement.new(">= 2.4.0".freeze)
|
21
|
+
|
22
|
+
s.add_runtime_dependency('fluentd', '> 1', '< 2')
|
23
|
+
|
24
|
+
s.add_development_dependency('rspec', '~> 3.4')
|
25
|
+
s.add_development_dependency('test-unit', '~> 3.3.9')
|
26
|
+
s.add_development_dependency('webmock', '~> 2.3')
|
27
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
require 'fluent/output'
|
2
|
+
require 'net/https'
|
3
|
+
|
4
|
+
module Fluent
|
5
|
+
class LogfireOutput < Fluent::BufferedOutput
|
6
|
+
Fluent::Plugin.register_output('logfire', self)
|
7
|
+
|
8
|
+
VERSION = "0.1.0".freeze
|
9
|
+
CONTENT_TYPE = "application/msgpack".freeze
|
10
|
+
HOST = "in.logfire.sh".freeze
|
11
|
+
PORT = 443
|
12
|
+
PATH = "/".freeze
|
13
|
+
MAX_ATTEMPTS = 3.freeze
|
14
|
+
RETRYABLE_CODES = [429, 500, 502, 503, 504].freeze
|
15
|
+
USER_AGENT = "logfire Fluentd/#{VERSION}".freeze
|
16
|
+
|
17
|
+
config_param :source_token, :string, secret: true
|
18
|
+
config_param :ip, :string, default: nil
|
19
|
+
|
20
|
+
def configure(conf)
|
21
|
+
@source_token = conf["source_token"]
|
22
|
+
super
|
23
|
+
end
|
24
|
+
|
25
|
+
def format(tag, time, record)
|
26
|
+
force_utf8_string_values(record.merge("dt" => Time.at(time).utc.iso8601)).to_msgpack
|
27
|
+
end
|
28
|
+
|
29
|
+
def write(chunk)
|
30
|
+
deliver(chunk, 1)
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
def deliver(chunk, attempt)
|
35
|
+
if attempt > MAX_ATTEMPTS
|
36
|
+
log.error("msg=\"Max attempts exceeded dropping chunk\" attempt=#{attempt}")
|
37
|
+
return false
|
38
|
+
end
|
39
|
+
|
40
|
+
http = build_http_client
|
41
|
+
records=0
|
42
|
+
chunk.each do
|
43
|
+
records=records+1
|
44
|
+
end
|
45
|
+
body = [0xdd,records].pack("CN")
|
46
|
+
body << chunk.read
|
47
|
+
|
48
|
+
begin
|
49
|
+
resp = http.start do |conn|
|
50
|
+
req = build_request(body)
|
51
|
+
log.debug("sending #{req.body.length} bytes to logfire")
|
52
|
+
conn.request(req)
|
53
|
+
end
|
54
|
+
ensure
|
55
|
+
http.finish if http.started?
|
56
|
+
end
|
57
|
+
|
58
|
+
code = resp.code.to_i
|
59
|
+
if code >= 200 && code <= 299
|
60
|
+
log.debug "POST request to logfire was responded to with status code #{code}"
|
61
|
+
true
|
62
|
+
elsif RETRYABLE_CODES.include?(code)
|
63
|
+
sleep_time = sleep_for_attempt(attempt)
|
64
|
+
log.warn("msg=\"Retryable response from the logfire API\" " +
|
65
|
+
"code=#{code} attempt=#{attempt} sleep=#{sleep_time}")
|
66
|
+
sleep(sleep_time)
|
67
|
+
deliver(chunk, attempt + 1)
|
68
|
+
else
|
69
|
+
log.error("msg=\"Fatal response from the logfire API\" code=#{code} attempt=#{attempt}")
|
70
|
+
false
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def sleep_for_attempt(attempt)
|
75
|
+
sleep_for = attempt ** 2
|
76
|
+
sleep_for = sleep_for <= 60 ? sleep_for : 60
|
77
|
+
(sleep_for / 2) + (rand(0..sleep_for) / 2)
|
78
|
+
end
|
79
|
+
|
80
|
+
def force_utf8_string_values(data)
|
81
|
+
data.transform_values do |val|
|
82
|
+
if val.is_a?(Hash)
|
83
|
+
force_utf8_string_values(val)
|
84
|
+
elsif val.respond_to?(:force_encoding)
|
85
|
+
val.force_encoding('UTF-8')
|
86
|
+
else
|
87
|
+
val
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def build_http_client
|
93
|
+
http = Net::HTTP.new(HOST, PORT)
|
94
|
+
http.use_ssl = true
|
95
|
+
# Verification on Windows fails despite having a valid certificate.
|
96
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
97
|
+
http.read_timeout = 30
|
98
|
+
http.ssl_timeout = 10
|
99
|
+
http.open_timeout = 10
|
100
|
+
http
|
101
|
+
end
|
102
|
+
|
103
|
+
def build_request(body)
|
104
|
+
path = '/'
|
105
|
+
req = Net::HTTP::Post.new(path)
|
106
|
+
req["Authorization"] = "Bearer #{@source_token}"
|
107
|
+
req["Content-Type"] = CONTENT_TYPE
|
108
|
+
req["User-Agent"] = USER_AGENT
|
109
|
+
req.body = body
|
110
|
+
req
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "fluent/plugin/out_logfire"
|
3
|
+
|
4
|
+
describe Fluent::LogfireOutput do
|
5
|
+
let(:config) do
|
6
|
+
%{
|
7
|
+
source_token abcd1234
|
8
|
+
}
|
9
|
+
end
|
10
|
+
|
11
|
+
let(:driver) do
|
12
|
+
tag = "test"
|
13
|
+
Fluent::Test::BufferedOutputTestDriver.new(Fluent::LogfireOutput, tag) {
|
14
|
+
# v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
|
15
|
+
if !defined?(Fluent::Plugin::Output)
|
16
|
+
def format(tag, time, record)
|
17
|
+
[time, record].to_msgpack
|
18
|
+
end
|
19
|
+
end
|
20
|
+
}.configure(config)
|
21
|
+
end
|
22
|
+
let(:record) do
|
23
|
+
{'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
|
24
|
+
end
|
25
|
+
|
26
|
+
before(:each) do
|
27
|
+
Fluent::Test.setup
|
28
|
+
end
|
29
|
+
|
30
|
+
describe "#write" do
|
31
|
+
it "should send a chunked request to the logfire API" do
|
32
|
+
stub = stub_request(:post, "https://in.logfire.sh/").
|
33
|
+
with(
|
34
|
+
:body => start_with("\xDD\x00\x00\x00\x01\x85\xA3age\x1A\xAArequest_id\xA242\xA9parent_id\xA6parent\xAArouting_id\xA7routing\xA2dt\xB4".force_encoding("ASCII-8BIT")),
|
35
|
+
:headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Authorization'=>'Bearer abcd1234', 'Content-Type'=>'application/msgpack', 'User-Agent'=>'logfire Fluentd/0.1.1'}
|
36
|
+
).
|
37
|
+
to_return(:status => 202, :body => "", :headers => {})
|
38
|
+
|
39
|
+
driver.emit(record)
|
40
|
+
driver.run
|
41
|
+
|
42
|
+
expect(stub).to have_been_requested.times(1)
|
43
|
+
end
|
44
|
+
|
45
|
+
it "handles 500s" do
|
46
|
+
stub = stub_request(:post, "https://in.logfire.sh/").to_return(:status => 500, :body => "", :headers => {})
|
47
|
+
|
48
|
+
driver.emit(record)
|
49
|
+
driver.run
|
50
|
+
|
51
|
+
expect(stub).to have_been_requested.times(3)
|
52
|
+
end
|
53
|
+
|
54
|
+
it "handle auth failures" do
|
55
|
+
stub = stub_request(:post, "https://in.logfire.sh/").to_return(:status => 403, :body => "", :headers => {})
|
56
|
+
|
57
|
+
driver.emit(record)
|
58
|
+
driver.run
|
59
|
+
|
60
|
+
expect(stub).to have_been_requested.times(1)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# Base
|
2
|
+
require 'rubygems'
|
3
|
+
require 'bundler/setup'
|
4
|
+
|
5
|
+
# Testing
|
6
|
+
require 'rspec'
|
7
|
+
|
8
|
+
# Webmock
|
9
|
+
require 'webmock/rspec'
|
10
|
+
WebMock.disable_net_connect!
|
11
|
+
|
12
|
+
# Fluent
|
13
|
+
require "fluent/test"
|
14
|
+
|
15
|
+
# Rspec
|
16
|
+
RSpec.configure do |config|
|
17
|
+
config.color = true
|
18
|
+
config.order = :random
|
19
|
+
config.warnings = false
|
20
|
+
end
|
metadata
ADDED
@@ -0,0 +1,112 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-logfire
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- logfire.sh
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2023-06-15 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1'
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '2'
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">"
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1'
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '2'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: rspec
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - "~>"
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '3.4'
|
40
|
+
type: :development
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - "~>"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '3.4'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: test-unit
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - "~>"
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 3.3.9
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - "~>"
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: 3.3.9
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: webmock
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - "~>"
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '2.3'
|
68
|
+
type: :development
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - "~>"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '2.3'
|
75
|
+
description: Streams fluentd logs to the logfire.sh logging service.
|
76
|
+
email: hello@logfire.sh
|
77
|
+
executables: []
|
78
|
+
extensions: []
|
79
|
+
extra_rdoc_files: []
|
80
|
+
files:
|
81
|
+
- ".github/workflows/main.yml"
|
82
|
+
- ".gitignore"
|
83
|
+
- Gemfile
|
84
|
+
- README.md
|
85
|
+
- fluent-plugin-logfire.gemspec
|
86
|
+
- lib/fluent/plugin/out_logfire.rb
|
87
|
+
- spec/fluent/plugin/out_logfire_spec.rb
|
88
|
+
- spec/spec_helper.rb
|
89
|
+
homepage: https://github.com/logfire-sh/logfire-fluentd-plugin-private
|
90
|
+
licenses:
|
91
|
+
- ISC
|
92
|
+
metadata: {}
|
93
|
+
post_install_message:
|
94
|
+
rdoc_options: []
|
95
|
+
require_paths:
|
96
|
+
- lib
|
97
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
98
|
+
requirements:
|
99
|
+
- - ">="
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: 2.4.0
|
102
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
103
|
+
requirements:
|
104
|
+
- - ">="
|
105
|
+
- !ruby/object:Gem::Version
|
106
|
+
version: '0'
|
107
|
+
requirements: []
|
108
|
+
rubygems_version: 3.1.6
|
109
|
+
signing_key:
|
110
|
+
specification_version: 4
|
111
|
+
summary: logfire.sh plugin for fluentd
|
112
|
+
test_files: []
|