fluent-plugin-logfire 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: d9b6492707112a0f4a156fa9c5bc8a7c57f3502c69baae9580581ad4b9c58a69
4
+ data.tar.gz: 4b95bd9dfaf66be22c318fd672438214bf3e0c0790a58a17d428038c3de5af6e
5
+ SHA512:
6
+ metadata.gz: eb1ac204dcc1fecc38c0890cd908443d414033c56d876d0782fe66a2d63f27cba35f07b215f7d8da9e26ef57012a48ee2d97733f1c60399a1c015935b193e7cc
7
+ data.tar.gz: 2a6741d11a61b7718a17b0d556d9de2cf9e4b50340cac8dc82f03a2db992eefc819999ee85d7f1b061df581388b43295aff2ba5a3276debe90fe21eefab76140
@@ -0,0 +1,29 @@
1
+ name: build
2
+
3
+ on: [push, pull_request]
4
+
5
+ jobs:
6
+ test:
7
+
8
+ runs-on: ubuntu-22.04
9
+
10
+ strategy:
11
+ matrix:
12
+ ruby-version:
13
+ - 3.0.0
14
+ - 2.7.2
15
+ - 2.6.6
16
+ - 2.5.8
17
+ - 2.4.10
18
+
19
+ steps:
20
+ - uses: actions/checkout@v2
21
+
22
+ - name: Set up Ruby ${{ matrix.ruby-version }}
23
+ uses: ruby/setup-ruby@v1
24
+ with:
25
+ ruby-version: ${{ matrix.ruby-version }}
26
+ bundler-cache: true
27
+
28
+ - name: Run tests
29
+ run: bundle exec rspec
data/.gitignore ADDED
@@ -0,0 +1,16 @@
1
+ .DS_Store
2
+ .rvmrc
3
+ .ruby-version
4
+ coverage
5
+ *.swp
6
+ *.gem
7
+ Gemfile.lock
8
+
9
+ gemfiles/*.lock
10
+
11
+ /.bundle
12
+ /.yardoc
13
+ /doc
14
+ /log
15
+ /tmp
16
+ /pkg
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
data/README.md ADDED
@@ -0,0 +1,29 @@
1
+ # 🪵 Fluent::Plugin::logfire, a plugin for [Fluentd](http://fluentd.org)
2
+
3
+ A Fluentd plugin that delivers events to the [logfire.sh logging service](https://logfire.sh).
4
+
5
+ ## Installation
6
+
7
+ ```
8
+ gem install fluent-plugin-logfire
9
+ ```
10
+
11
+ ## Usage
12
+
13
+ In your Fluentd configuration, use `@type logfire`:
14
+
15
+ ```
16
+ <match your_match>
17
+ @type logfire
18
+ source_token YOUR_SOURCE_TOKEN
19
+ # ip 127.0.0.1
20
+ buffer_chunk_limit 1m # Must be < 5m
21
+ flush_at_shutdown true # Only needed with file buffer
22
+ </match>
23
+ ```
24
+
25
+ ## Configuration
26
+
27
+ * `source_token` - This is your [logfire source token](https://logfire.sh) whithout the Bearer keyword.
28
+
29
+ For advanced configuration options, please see to the [buffered output parameters documentation.](http://docs.fluentd.org/articles/output-plugin-overview#buffered-output-parameters).
@@ -0,0 +1,27 @@
1
+ # -*- encoding: utf-8 -*-
2
+ require 'date'
3
+
4
+ Gem::Specification.new do |s|
5
+ s.name = 'fluent-plugin-logfire'
6
+ s.version = '0.1.0'
7
+ s.date = Date.today.to_s
8
+ s.summary = 'logfire.sh plugin for fluentd'
9
+ s.description = 'Streams fluentd logs to the logfire.sh logging service.'
10
+ s.authors = ['logfire.sh']
11
+ s.email = 'hello@logfire.sh'
12
+ s.homepage = 'https://github.com/logfire-sh/logfire-fluentd-plugin-private'
13
+ s.license = 'ISC'
14
+
15
+ s.files = `git ls-files`.split("\n")
16
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
17
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
18
+ s.require_paths = ["lib"]
19
+
20
+ s.required_ruby_version = Gem::Requirement.new(">= 2.4.0".freeze)
21
+
22
+ s.add_runtime_dependency('fluentd', '> 1', '< 2')
23
+
24
+ s.add_development_dependency('rspec', '~> 3.4')
25
+ s.add_development_dependency('test-unit', '~> 3.3.9')
26
+ s.add_development_dependency('webmock', '~> 2.3')
27
+ end
@@ -0,0 +1,113 @@
1
+ require 'fluent/output'
2
+ require 'net/https'
3
+
4
+ module Fluent
5
+ class LogfireOutput < Fluent::BufferedOutput
6
+ Fluent::Plugin.register_output('logfire', self)
7
+
8
+ VERSION = "0.1.0".freeze
9
+ CONTENT_TYPE = "application/msgpack".freeze
10
+ HOST = "in.logfire.sh".freeze
11
+ PORT = 443
12
+ PATH = "/".freeze
13
+ MAX_ATTEMPTS = 3.freeze
14
+ RETRYABLE_CODES = [429, 500, 502, 503, 504].freeze
15
+ USER_AGENT = "logfire Fluentd/#{VERSION}".freeze
16
+
17
+ config_param :source_token, :string, secret: true
18
+ config_param :ip, :string, default: nil
19
+
20
+ def configure(conf)
21
+ @source_token = conf["source_token"]
22
+ super
23
+ end
24
+
25
+ def format(tag, time, record)
26
+ force_utf8_string_values(record.merge("dt" => Time.at(time).utc.iso8601)).to_msgpack
27
+ end
28
+
29
+ def write(chunk)
30
+ deliver(chunk, 1)
31
+ end
32
+
33
+ private
34
+ def deliver(chunk, attempt)
35
+ if attempt > MAX_ATTEMPTS
36
+ log.error("msg=\"Max attempts exceeded dropping chunk\" attempt=#{attempt}")
37
+ return false
38
+ end
39
+
40
+ http = build_http_client
41
+ records=0
42
+ chunk.each do
43
+ records=records+1
44
+ end
45
+ body = [0xdd,records].pack("CN")
46
+ body << chunk.read
47
+
48
+ begin
49
+ resp = http.start do |conn|
50
+ req = build_request(body)
51
+ log.debug("sending #{req.body.length} bytes to logfire")
52
+ conn.request(req)
53
+ end
54
+ ensure
55
+ http.finish if http.started?
56
+ end
57
+
58
+ code = resp.code.to_i
59
+ if code >= 200 && code <= 299
60
+ log.debug "POST request to logfire was responded to with status code #{code}"
61
+ true
62
+ elsif RETRYABLE_CODES.include?(code)
63
+ sleep_time = sleep_for_attempt(attempt)
64
+ log.warn("msg=\"Retryable response from the logfire API\" " +
65
+ "code=#{code} attempt=#{attempt} sleep=#{sleep_time}")
66
+ sleep(sleep_time)
67
+ deliver(chunk, attempt + 1)
68
+ else
69
+ log.error("msg=\"Fatal response from the logfire API\" code=#{code} attempt=#{attempt}")
70
+ false
71
+ end
72
+ end
73
+
74
+ def sleep_for_attempt(attempt)
75
+ sleep_for = attempt ** 2
76
+ sleep_for = sleep_for <= 60 ? sleep_for : 60
77
+ (sleep_for / 2) + (rand(0..sleep_for) / 2)
78
+ end
79
+
80
+ def force_utf8_string_values(data)
81
+ data.transform_values do |val|
82
+ if val.is_a?(Hash)
83
+ force_utf8_string_values(val)
84
+ elsif val.respond_to?(:force_encoding)
85
+ val.force_encoding('UTF-8')
86
+ else
87
+ val
88
+ end
89
+ end
90
+ end
91
+
92
+ def build_http_client
93
+ http = Net::HTTP.new(HOST, PORT)
94
+ http.use_ssl = true
95
+ # Verification on Windows fails despite having a valid certificate.
96
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
97
+ http.read_timeout = 30
98
+ http.ssl_timeout = 10
99
+ http.open_timeout = 10
100
+ http
101
+ end
102
+
103
+ def build_request(body)
104
+ path = '/'
105
+ req = Net::HTTP::Post.new(path)
106
+ req["Authorization"] = "Bearer #{@source_token}"
107
+ req["Content-Type"] = CONTENT_TYPE
108
+ req["User-Agent"] = USER_AGENT
109
+ req.body = body
110
+ req
111
+ end
112
+ end
113
+ end
@@ -0,0 +1,63 @@
1
+ require "spec_helper"
2
+ require "fluent/plugin/out_logfire"
3
+
4
+ describe Fluent::LogfireOutput do
5
+ let(:config) do
6
+ %{
7
+ source_token abcd1234
8
+ }
9
+ end
10
+
11
+ let(:driver) do
12
+ tag = "test"
13
+ Fluent::Test::BufferedOutputTestDriver.new(Fluent::LogfireOutput, tag) {
14
+ # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
15
+ if !defined?(Fluent::Plugin::Output)
16
+ def format(tag, time, record)
17
+ [time, record].to_msgpack
18
+ end
19
+ end
20
+ }.configure(config)
21
+ end
22
+ let(:record) do
23
+ {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
24
+ end
25
+
26
+ before(:each) do
27
+ Fluent::Test.setup
28
+ end
29
+
30
+ describe "#write" do
31
+ it "should send a chunked request to the logfire API" do
32
+ stub = stub_request(:post, "https://in.logfire.sh/").
33
+ with(
34
+ :body => start_with("\xDD\x00\x00\x00\x01\x85\xA3age\x1A\xAArequest_id\xA242\xA9parent_id\xA6parent\xAArouting_id\xA7routing\xA2dt\xB4".force_encoding("ASCII-8BIT")),
35
+ :headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Authorization'=>'Bearer abcd1234', 'Content-Type'=>'application/msgpack', 'User-Agent'=>'logfire Fluentd/0.1.1'}
36
+ ).
37
+ to_return(:status => 202, :body => "", :headers => {})
38
+
39
+ driver.emit(record)
40
+ driver.run
41
+
42
+ expect(stub).to have_been_requested.times(1)
43
+ end
44
+
45
+ it "handles 500s" do
46
+ stub = stub_request(:post, "https://in.logfire.sh/").to_return(:status => 500, :body => "", :headers => {})
47
+
48
+ driver.emit(record)
49
+ driver.run
50
+
51
+ expect(stub).to have_been_requested.times(3)
52
+ end
53
+
54
+ it "handle auth failures" do
55
+ stub = stub_request(:post, "https://in.logfire.sh/").to_return(:status => 403, :body => "", :headers => {})
56
+
57
+ driver.emit(record)
58
+ driver.run
59
+
60
+ expect(stub).to have_been_requested.times(1)
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,20 @@
1
+ # Base
2
+ require 'rubygems'
3
+ require 'bundler/setup'
4
+
5
+ # Testing
6
+ require 'rspec'
7
+
8
+ # Webmock
9
+ require 'webmock/rspec'
10
+ WebMock.disable_net_connect!
11
+
12
+ # Fluent
13
+ require "fluent/test"
14
+
15
+ # Rspec
16
+ RSpec.configure do |config|
17
+ config.color = true
18
+ config.order = :random
19
+ config.warnings = false
20
+ end
metadata ADDED
@@ -0,0 +1,112 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-logfire
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - logfire.sh
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2023-06-15 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">"
18
+ - !ruby/object:Gem::Version
19
+ version: '1'
20
+ - - "<"
21
+ - !ruby/object:Gem::Version
22
+ version: '2'
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">"
28
+ - !ruby/object:Gem::Version
29
+ version: '1'
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: '2'
33
+ - !ruby/object:Gem::Dependency
34
+ name: rspec
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '3.4'
40
+ type: :development
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - "~>"
45
+ - !ruby/object:Gem::Version
46
+ version: '3.4'
47
+ - !ruby/object:Gem::Dependency
48
+ name: test-unit
49
+ requirement: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - "~>"
52
+ - !ruby/object:Gem::Version
53
+ version: 3.3.9
54
+ type: :development
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: 3.3.9
61
+ - !ruby/object:Gem::Dependency
62
+ name: webmock
63
+ requirement: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - "~>"
66
+ - !ruby/object:Gem::Version
67
+ version: '2.3'
68
+ type: :development
69
+ prerelease: false
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - "~>"
73
+ - !ruby/object:Gem::Version
74
+ version: '2.3'
75
+ description: Streams fluentd logs to the logfire.sh logging service.
76
+ email: hello@logfire.sh
77
+ executables: []
78
+ extensions: []
79
+ extra_rdoc_files: []
80
+ files:
81
+ - ".github/workflows/main.yml"
82
+ - ".gitignore"
83
+ - Gemfile
84
+ - README.md
85
+ - fluent-plugin-logfire.gemspec
86
+ - lib/fluent/plugin/out_logfire.rb
87
+ - spec/fluent/plugin/out_logfire_spec.rb
88
+ - spec/spec_helper.rb
89
+ homepage: https://github.com/logfire-sh/logfire-fluentd-plugin-private
90
+ licenses:
91
+ - ISC
92
+ metadata: {}
93
+ post_install_message:
94
+ rdoc_options: []
95
+ require_paths:
96
+ - lib
97
+ required_ruby_version: !ruby/object:Gem::Requirement
98
+ requirements:
99
+ - - ">="
100
+ - !ruby/object:Gem::Version
101
+ version: 2.4.0
102
+ required_rubygems_version: !ruby/object:Gem::Requirement
103
+ requirements:
104
+ - - ">="
105
+ - !ruby/object:Gem::Version
106
+ version: '0'
107
+ requirements: []
108
+ rubygems_version: 3.1.6
109
+ signing_key:
110
+ specification_version: 4
111
+ summary: logfire.sh plugin for fluentd
112
+ test_files: []