fluent-plugin-timber 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 496cd7f7702fb5c4753cd01e1f5757956110b873
+  data.tar.gz: dcb578e403619c54d9bbd4b631f432d532300d70
+SHA512:
+  metadata.gz: 94c65c0c0f837a5f1c0fbbe776e3487ad61a6ec1390116150f6164643f77c8862265685f6032bfe41f0b73e6050c21b6c9ab769ad6b2fe5d3272d13aea608b33
+  data.tar.gz: 6eaa933afbd1a11dd58a2bde616e7fa7f76c7a3ea32958a70e3c55c8cf6167cec0c56b4805f936fbb8d91bd8c8219c3e3a9b27fbdabd9d96f36e0613a40bfdd0

data/.gitignore

@@ -0,0 +1,15 @@
+.DS_Store
+.rvmrc
+.ruby-version
+coverage
+*.swp
+*.gem
+
+gemfiles/*.lock
+
+/.bundle
+/.yardoc
+/doc
+/log
+/tmp
+/pkg

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,86 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-timber (1.0.0)
+      fluentd (>= 0.12.0, < 2)
+      http (~> 2.0, >= 2.0.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.5.1)
+      public_suffix (~> 2.0, >= 2.0.2)
+    cool.io (1.5.0)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.3)
+    domain_name (0.5.20170404)
+      unf (>= 0.0.5, < 1.0.0)
+    fluentd (0.14.19)
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 0.7.0, < 2.0.0)
+      serverengine (>= 2.0.4, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (~> 0.1.7)
+      tzinfo (~> 1.0)
+      tzinfo-data (~> 1.0)
+      yajl-ruby (~> 1.0)
+    hashdiff (0.3.2)
+    http (2.2.2)
+      addressable (~> 2.3)
+      http-cookie (~> 1.0)
+      http-form_data (~> 1.0.1)
+      http_parser.rb (~> 0.6.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    http-form_data (1.0.3)
+    http_parser.rb (0.6.0)
+    msgpack (1.1.0)
+    power_assert (0.2.6)
+    public_suffix (2.0.5)
+    rspec (3.6.0)
+      rspec-core (~> 3.6.0)
+      rspec-expectations (~> 3.6.0)
+      rspec-mocks (~> 3.6.0)
+    rspec-core (3.6.0)
+      rspec-support (~> 3.6.0)
+    rspec-expectations (3.6.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.6.0)
+    rspec-mocks (3.6.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.6.0)
+    rspec-support (3.6.0)
+    safe_yaml (1.0.4)
+    serverengine (2.0.5)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.1.9)
+    test-unit (3.1.5)
+      power_assert
+    thread_safe (0.3.6)
+    tzinfo (1.2.3)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2017.2)
+      tzinfo (>= 1.0.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.4)
+    webmock (2.3.2)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+    yajl-ruby (1.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fluent-plugin-timber!
+  rspec (~> 3.4)
+  test-unit (~> 3.1.0)
+  webmock (~> 2.3)
+
+BUNDLED WITH
+   1.15.1

data/LICENSE.md

@@ -0,0 +1,15 @@
+# License
+
+Copyright (c) 2017, Timber Technologies, Inc.
+
+Permission to use, copy, modify, and/or distribute this software for any purpose
+with or without fee is hereby granted, provided that the above copyright notice
+and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
+THIS SOFTWARE.

data/README.md

@@ -0,0 +1,25 @@
+# Fluent::Plugin::Timber, a plugin for [Fluentd](http://fluentd.org)
+
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-timber.png)](http://badge.fury.io/rb/fluent-plugin-timber)
+[![Build Status](https://travis-ci.org/timberio/fluent-plugin-timber.png?branch=master)](https://travis-ci.org/timberio/fluent-plugin-timber)
+[![Issue Stats](http://issuestats.com/github/uken/fluent-plugin-timber/badge/pr)](http://issuestats.com/github/uken/fluent-plugin-timber)
+[![Issue Stats](http://issuestats.com/github/uken/fluent-plugin-timber/badge/issue)](http://issuestats.com/github/uken/fluent-plugin-timber)
+
+A highly efficient Fluentd plugin that delivers events to the [Timber.io logging service](https://timber.io). It uses batching, msgpack, and retry logic for highly efficient and reliable delivery of log data.
+
+[Timber.io](https://timber.io) is a different kind of logging service with a focus on modern logging best-practices: easy setup, structured data, fast clean usable interface, 6 months of searchable retention, threshold based alerts, simple graphing, and more. Learn more at [https://timber.io](https://timber.io).
+
+```
+<match your_match>
+  type timber
+  api_key xxxxxxxxxxxxxxxxxxxxxxxxxxx        # Your Timber API (required)
+  hostname "#{Socket.gethostname}"           # Your hostname (required)
+  # ip 127.0.0.1                             # IP address (optional)
+  buffer_chunk_limit 1m                      # Must be < 5m
+  flush_at_shutdown true                     # Only needed with file buffer
+</match>
+```
+
+For advanced configuration options, please see to the [buffered output parameters documentation.](http://docs.fluentd.org/articles/output-plugin-overview#buffered-output-parameters).
+
+Questions? Need help? [support@timber.io](mailto:support@timber.io).

data/fluent-plugin-timber.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+require 'date'
+
+Gem::Specification.new do |s|
+  s.name        = 'fluent-plugin-timber'
+  s.version     = '1.0.0'
+  s.date        = Date.today.to_s
+  s.summary     = 'Timber.io plugin for Fluentd'
+  s.description = 'Streams Fluentd logs to the Timber.io logging service.'
+  s.authors     = ['Timber.io']
+  s.email       = 'hi@timber.io'
+  s.homepage    = 'https://github.com/timberio/fluent-plugin-timber'
+  s.license     = 'ISC'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.required_ruby_version = Gem::Requirement.new(">= 2.0.0".freeze)
+
+  s.add_runtime_dependency('fluentd', '>= 0.12.0', '< 2')
+  s.add_runtime_dependency('http', '~> 2.0', '>= 2.0.3')
+
+  s.add_development_dependency('rspec', '~> 3.4')
+  s.add_development_dependency('test-unit', '~> 3.1.0')
+  s.add_development_dependency('webmock', '~> 2.3')
+end

data/lib/fluent/plugin/out_timber.rb

@@ -0,0 +1,86 @@
+require 'base64'
+require 'json'
+
+require 'fluent/output'
+
+module Fluent
+  class TimberOutput < Fluent::BufferedOutput
+    Fluent::Plugin.register_output('timber', self)
+
+    VERSION = "1.0.0".freeze
+    CONTENT_TYPE = "application/json".freeze
+    HOST = "https://logs.timber.io".freeze
+    MAX_ATTEMPTS = 3.freeze
+    PATH = "/frames".freeze
+    RETRYABLE_CODES = [429, 500, 502, 503, 504].freeze
+    USER_AGENT = "Timber Logstash/#{VERSION}".freeze
+
+    config_param :api_key, :string, secret: true
+    config_param :hostname, :string
+    config_param :ip, :string, default: nil
+
+    def configure(conf)
+      encoded_api_key = Base64.urlsafe_encode64(conf["api_key"]).chomp
+      authorization_value = "Basic #{encoded_api_key}"
+      @headers = {
+        "Authorization" => authorization_value,
+        "Content-Type" => CONTENT_TYPE,
+        "User-Agent" => USER_AGENT
+      }
+      super
+    end
+
+    def start
+      super
+      require 'http'
+      HTTP.default_options = {:keep_alive_timeout => 29}
+      @http_client = HTTP.persistent(HOST)
+    end
+
+    def shutdown
+      @http_client.close if @http_client
+      super
+    end
+
+    def format(tag, time, record)
+      dt_iso8601 = Time.at(time).utc.iso8601
+      record.merge("dt" => dt_iso8601).to_msgpack
+    end
+
+    def write(chunk)
+      deliver(chunk, 1)
+    end
+
+    private
+      def deliver(chunk, attempt)
+        if attempt > MAX_ATTEMPTS
+          log.error("msg=\"Max attempts exceeded dropping chunk\" attempt=#{attempt}")
+          return false
+        end
+
+        body = chunk.read
+        response = @http_client.headers(@headers).post(PATH, body: body)
+        response.flush
+        code = response.code
+
+        if code >= 200 && code <= 299
+          true
+        elsif RETRYABLE_CODES.include?(code)
+          sleep_time = sleep_for_attempt(attempt)
+          log.warn("msg=\"Retryable response from the Timber API\" " +
+            "code=#{code} attempt=#{attempt} sleep=#{sleep_time}")
+          sleep(sleep_time)
+          deliver(chunk, attempt + 1)
+        else
+          log.error("msg=\"Fatal response from the Timber API\" code=#{code} attempt=#{attempt}")
+          false
+        end
+      end
+
+      def sleep_for_attempt(attempt)
+        sleep_for = attempt ** 2
+        sleep_for = sleep_for <= 60 ? sleep_for : 60
+        (sleep_for / 2) + (rand(0..sleep_for) / 2)
+      end
+  end
+end

data/spec/fluent/plugin/out_timber_spec.rb

@@ -0,0 +1,64 @@
+require "spec_helper"
+require "fluent/plugin/out_timber"
+
+describe Fluent::TimberOutput do
+  let(:config) do
+    %{
+      api_key  abcd1234
+      hostname my.host.com
+    }
+  end
+
+  let(:driver) do
+    tag = "test"
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::TimberOutput, tag) {
+      # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
+      if !defined?(Fluent::Plugin::Output)
+        def format(tag, time, record)
+          [time, record].to_msgpack
+        end
+      end
+    }.configure(config)
+  end
+  let(:record) do
+    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
+  end
+
+  before(:each) do
+    Fluent::Test.setup
+  end
+
+  describe "#write" do
+    it "should send a chunked request to the Timber API" do
+      stub = stub_request(:post, "https://logs.timber.io/frames").
+        with(
+          :body => start_with("\x85\xA3age\x1A\xAArequest_id\xA242\xA9parent_id\xA6parent\xAArouting_id\xA7routing\xA2dt\xB4".force_encoding("ASCII-8BIT")),
+          :headers => {'Authorization'=>'Basic YWJjZDEyMzQ=', 'Connection'=>'Keep-Alive', 'Content-Type'=>'application/json', 'User-Agent'=>'Timber Logstash/1.0.0'}
+        ).
+        to_return(:status => 200, :body => "", :headers => {})
+
+      driver.emit(record)
+      driver.run
+
+      expect(stub).to have_been_requested.times(1)
+    end
+
+    it "handles 500s" do
+      stub = stub_request(:post, "https://logs.timber.io/frames").to_return(:status => 500, :body => "", :headers => {})
+
+      driver.emit(record)
+      driver.run
+
+      expect(stub).to have_been_requested.times(3)
+    end
+
+    it "handle auth failures" do
+      stub = stub_request(:post, "https://logs.timber.io/frames").to_return(:status => 403, :body => "", :headers => {})
+
+      driver.emit(record)
+      driver.run
+
+      expect(stub).to have_been_requested.times(1)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,20 @@
+# Base
+require 'rubygems'
+require 'bundler/setup'
+
+# Testing
+require 'rspec'
+
+# Webmock
+require 'webmock/rspec'
+WebMock.disable_net_connect!
+
+# Fluent
+require "fluent/test"
+
+# Rspec
+RSpec.configure do |config|
+  config.color = true
+  config.order = :random
+  config.warnings = false
+end

metadata

@@ -0,0 +1,136 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-timber
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Timber.io
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-07-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.3
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+description: Streams Fluentd logs to the Timber.io logging service.
+email: hi@timber.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE.md
+- README.md
+- fluent-plugin-timber.gemspec
+- lib/fluent/plugin/out_timber.rb
+- spec/fluent/plugin/out_timber_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/timberio/fluent-plugin-timber
+licenses:
+- ISC
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Timber.io plugin for Fluentd
+test_files:
+- spec/fluent/plugin/out_timber_spec.rb
+- spec/spec_helper.rb