fluent-plugin-logfire 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d9b6492707112a0f4a156fa9c5bc8a7c57f3502c69baae9580581ad4b9c58a69
+  data.tar.gz: 4b95bd9dfaf66be22c318fd672438214bf3e0c0790a58a17d428038c3de5af6e
+SHA512:
+  metadata.gz: eb1ac204dcc1fecc38c0890cd908443d414033c56d876d0782fe66a2d63f27cba35f07b215f7d8da9e26ef57012a48ee2d97733f1c60399a1c015935b193e7cc
+  data.tar.gz: 2a6741d11a61b7718a17b0d556d9de2cf9e4b50340cac8dc82f03a2db992eefc819999ee85d7f1b061df581388b43295aff2ba5a3276debe90fe21eefab76140

data/.github/workflows/main.yml

@@ -0,0 +1,29 @@
+name: build
+
+on: [push, pull_request]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-22.04
+
+    strategy:
+      matrix:
+        ruby-version:
+          - 3.0.0
+          - 2.7.2
+          - 2.6.6
+          - 2.5.8
+          - 2.4.10
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Ruby ${{ matrix.ruby-version }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true
+
+      - name: Run tests
+        run: bundle exec rspec

data/.gitignore

@@ -0,0 +1,16 @@
+.DS_Store
+.rvmrc
+.ruby-version
+coverage
+*.swp
+*.gem
+Gemfile.lock
+
+gemfiles/*.lock
+
+/.bundle
+/.yardoc
+/doc
+/log
+/tmp
+/pkg

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/README.md

@@ -0,0 +1,29 @@
+# 🪵 Fluent::Plugin::logfire, a plugin for [Fluentd](http://fluentd.org)
+
+A Fluentd plugin that delivers events to the [logfire.sh logging service](https://logfire.sh).
+
+## Installation
+
+```
+gem install fluent-plugin-logfire
+```
+
+## Usage
+
+In your Fluentd configuration, use `@type logfire`:
+
+```
+<match your_match>
+  @type logfire
+  source_token YOUR_SOURCE_TOKEN
+  # ip 127.0.0.1
+  buffer_chunk_limit 1m                      # Must be < 5m
+  flush_at_shutdown true                     # Only needed with file buffer
+</match>
+```
+
+## Configuration
+
+* `source_token` - This is your [logfire source token](https://logfire.sh) whithout the Bearer keyword.
+
+For advanced configuration options, please see to the [buffered output parameters documentation.](http://docs.fluentd.org/articles/output-plugin-overview#buffered-output-parameters).

data/fluent-plugin-logfire.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+require 'date'
+
+Gem::Specification.new do |s|
+  s.name        = 'fluent-plugin-logfire'
+  s.version     = '0.1.0'
+  s.date        = Date.today.to_s
+  s.summary     = 'logfire.sh plugin for fluentd'
+  s.description = 'Streams fluentd logs to the logfire.sh logging service.'
+  s.authors     = ['logfire.sh']
+  s.email       = 'hello@logfire.sh'
+  s.homepage    = 'https://github.com/logfire-sh/logfire-fluentd-plugin-private'
+  s.license     = 'ISC'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.required_ruby_version = Gem::Requirement.new(">= 2.4.0".freeze)
+
+  s.add_runtime_dependency('fluentd', '> 1', '< 2')
+
+  s.add_development_dependency('rspec', '~> 3.4')
+  s.add_development_dependency('test-unit', '~> 3.3.9')
+  s.add_development_dependency('webmock', '~> 2.3')
+end

data/lib/fluent/plugin/out_logfire.rb

@@ -0,0 +1,113 @@
+require 'fluent/output'
+require 'net/https'
+
+module Fluent
+  class LogfireOutput < Fluent::BufferedOutput
+    Fluent::Plugin.register_output('logfire', self)
+
+    VERSION = "0.1.0".freeze
+    CONTENT_TYPE = "application/msgpack".freeze
+    HOST = "in.logfire.sh".freeze
+    PORT = 443
+    PATH = "/".freeze
+    MAX_ATTEMPTS = 3.freeze
+    RETRYABLE_CODES = [429, 500, 502, 503, 504].freeze
+    USER_AGENT = "logfire Fluentd/#{VERSION}".freeze
+
+    config_param :source_token, :string, secret: true
+    config_param :ip, :string, default: nil
+
+    def configure(conf)
+      @source_token = conf["source_token"]
+      super
+    end
+
+    def format(tag, time, record)
+      force_utf8_string_values(record.merge("dt" => Time.at(time).utc.iso8601)).to_msgpack
+    end
+
+    def write(chunk)
+      deliver(chunk, 1)
+    end
+
+    private
+      def deliver(chunk, attempt)
+        if attempt > MAX_ATTEMPTS
+          log.error("msg=\"Max attempts exceeded dropping chunk\" attempt=#{attempt}")
+          return false
+        end
+
+        http = build_http_client
+        records=0
+        chunk.each do
+          records=records+1
+        end
+        body = [0xdd,records].pack("CN")
+        body << chunk.read
+
+        begin
+          resp = http.start do |conn|
+            req = build_request(body)
+            log.debug("sending #{req.body.length} bytes to logfire")
+            conn.request(req)
+          end
+        ensure
+          http.finish if http.started?
+        end
+
+        code = resp.code.to_i
+        if code >= 200 && code <= 299
+          log.debug "POST request to logfire was responded to with status code #{code}"
+          true
+        elsif RETRYABLE_CODES.include?(code)
+          sleep_time = sleep_for_attempt(attempt)
+          log.warn("msg=\"Retryable response from the logfire API\" " +
+            "code=#{code} attempt=#{attempt} sleep=#{sleep_time}")
+          sleep(sleep_time)
+          deliver(chunk, attempt + 1)
+        else
+          log.error("msg=\"Fatal response from the logfire API\" code=#{code} attempt=#{attempt}")
+          false
+        end
+      end
+
+      def sleep_for_attempt(attempt)
+        sleep_for = attempt ** 2
+        sleep_for = sleep_for <= 60 ? sleep_for : 60
+        (sleep_for / 2) + (rand(0..sleep_for) / 2)
+      end
+
+      def force_utf8_string_values(data)
+        data.transform_values do |val|
+          if val.is_a?(Hash)
+            force_utf8_string_values(val)
+          elsif val.respond_to?(:force_encoding)
+            val.force_encoding('UTF-8')
+          else
+            val
+          end
+        end
+      end
+
+      def build_http_client
+        http = Net::HTTP.new(HOST, PORT)
+        http.use_ssl = true
+        # Verification on Windows fails despite having a valid certificate.
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        http.read_timeout = 30
+        http.ssl_timeout = 10
+        http.open_timeout = 10
+        http
+      end
+
+      def build_request(body)
+        path = '/'
+        req = Net::HTTP::Post.new(path)
+        req["Authorization"] = "Bearer #{@source_token}"
+        req["Content-Type"] = CONTENT_TYPE
+        req["User-Agent"] = USER_AGENT
+        req.body = body
+        req
+      end
+  end
+end

data/spec/fluent/plugin/out_logfire_spec.rb

@@ -0,0 +1,63 @@
+require "spec_helper"
+require "fluent/plugin/out_logfire"
+
+describe Fluent::LogfireOutput do
+  let(:config) do
+    %{
+      source_token abcd1234
+    }
+  end
+
+  let(:driver) do
+    tag = "test"
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::LogfireOutput, tag) {
+      # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
+      if !defined?(Fluent::Plugin::Output)
+        def format(tag, time, record)
+          [time, record].to_msgpack
+        end
+      end
+    }.configure(config)
+  end
+  let(:record) do
+    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
+  end
+
+  before(:each) do
+    Fluent::Test.setup
+  end
+
+  describe "#write" do
+    it "should send a chunked request to the logfire API" do
+      stub = stub_request(:post, "https://in.logfire.sh/").
+        with(
+          :body => start_with("\xDD\x00\x00\x00\x01\x85\xA3age\x1A\xAArequest_id\xA242\xA9parent_id\xA6parent\xAArouting_id\xA7routing\xA2dt\xB4".force_encoding("ASCII-8BIT")),
+          :headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Authorization'=>'Bearer abcd1234', 'Content-Type'=>'application/msgpack', 'User-Agent'=>'logfire Fluentd/0.1.1'}
+        ).
+        to_return(:status => 202, :body => "", :headers => {})
+
+      driver.emit(record)
+      driver.run
+
+      expect(stub).to have_been_requested.times(1)
+    end
+
+    it "handles 500s" do
+      stub = stub_request(:post, "https://in.logfire.sh/").to_return(:status => 500, :body => "", :headers => {})
+
+      driver.emit(record)
+      driver.run
+
+      expect(stub).to have_been_requested.times(3)
+    end
+
+    it "handle auth failures" do
+      stub = stub_request(:post, "https://in.logfire.sh/").to_return(:status => 403, :body => "", :headers => {})
+
+      driver.emit(record)
+      driver.run
+
+      expect(stub).to have_been_requested.times(1)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,20 @@
+# Base
+require 'rubygems'
+require 'bundler/setup'
+
+# Testing
+require 'rspec'
+
+# Webmock
+require 'webmock/rspec'
+WebMock.disable_net_connect!
+
+# Fluent
+require "fluent/test"
+
+# Rspec
+RSpec.configure do |config|
+  config.color = true
+  config.order = :random
+  config.warnings = false
+end

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-logfire
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- logfire.sh
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-06-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.3.9
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.3.9
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+description: Streams fluentd logs to the logfire.sh logging service.
+email: hello@logfire.sh
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/main.yml"
+- ".gitignore"
+- Gemfile
+- README.md
+- fluent-plugin-logfire.gemspec
+- lib/fluent/plugin/out_logfire.rb
+- spec/fluent/plugin/out_logfire_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/logfire-sh/logfire-fluentd-plugin-private
+licenses:
+- ISC
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key:
+specification_version: 4
+summary: logfire.sh plugin for fluentd
+test_files: []