fluent-plugin-logdna 0.1.5 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 14b55b11f2a8e65f520848e562af9c9b35c5fcc7
-  data.tar.gz: 2d37a86e2329cfd1b24850f362cd90b9c30f07d0
+SHA256:
+  metadata.gz: 7dc5c3a579927d59c9a9705bd5ceec17d5a6dc3d8964d1639dd51afb23eaca80
+  data.tar.gz: 2519afb42481d5840bd11b5d579637e4ea55008dc89996421013185d76439374
 SHA512:
-  metadata.gz: 4af3c7d17e7aef3a6b62f1ca890bb32ebf6f6ce9a768aa2bc6dc6169d2d8ae0a057aab8a0592cb9470a38cc9b0755c4d58ed751c9dd17bcf21f334547209a382
-  data.tar.gz: c37bc197d3799956743d4596f1c8b37ea7a13f10c4053c9ffd62a54a6366115aa9c149e949a46f6ab19079b2ccb4e1a555e402eff813ea00b7c495234dfbadcc
+  metadata.gz: 511d77f656a7584723954052a7f4c0e70b39f8115f88cbfff2369b1d9e7388fac35242ff9b52a5e0b6b2292e73c757a3efb1025833bfd6b9a96dd9ef1437fad5
+  data.tar.gz: c1058ffb07ab69ad383c3b1d8015aaf912ebe3d0632c612c6e63d2ba41b75c6b930fa03f8f4a415c066501bbf4ab38f900f8728b9d630cf41bee002f5b8f6b2c

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem "rake", "~> 12.3"
+gem "webmock"

data/Gemfile.lock

@@ -0,0 +1,70 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-logdna (0.3.1)
+      fluentd (>= 0.12.0, < 2)
+      http (~> 2.0, >= 2.0.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    concurrent-ruby (1.1.6)
+    cool.io (1.6.0)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    fluentd (1.11.1)
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 1.3.1, < 2.0.0)
+      serverengine (>= 2.0.4, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.2, < 1.0.0)
+      tzinfo (>= 1.0, < 3.0)
+      tzinfo-data (~> 1.0)
+      yajl-ruby (~> 1.0)
+    hashdiff (1.0.1)
+    http (2.2.2)
+      addressable (~> 2.3)
+      http-cookie (~> 1.0)
+      http-form_data (~> 1.0.1)
+      http_parser.rb (~> 0.6.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    http-form_data (1.0.3)
+    http_parser.rb (0.6.0)
+    msgpack (1.3.3)
+    public_suffix (4.0.3)
+    rake (12.3.3)
+    safe_yaml (1.0.5)
+    serverengine (2.2.1)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.2.4)
+    tzinfo (2.0.2)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2020.1)
+      tzinfo (>= 1.0.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+    webmock (3.8.3)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  fluent-plugin-logdna!
+  rake (~> 12.3)
+  webmock
+
+BUNDLED WITH
+   1.17.2

data/Makefile

@@ -0,0 +1,62 @@
+# Generate a debian package for fluent-plugin-logdna
+#
+# This assumes that you have the td-agent package already installed
+# https://docs.fluentd.org/v1.0/articles/install-by-deb
+#
+# The package is created with fpm (it does not necessarily need to be
+# installed into the td-agent embedded ruby). To install fpm run:
+# $ sudo gem install --no-ri --no-rdoc fpm
+#
+# The td-agent package installs an embedded ruby distribution in
+# /opt/td-agent/embedded which has its own list of gems. Our built package
+# cannot conflict with those gems so we will elect to bundle a subset of gems
+# with our package. This may conflict with other packages and this package is
+# not intended to be a general purpose package.
+
+# Paths to td-agent specific versions of bundle and gem
+BUNDLE=/opt/td-agent/embedded/bin/bundle
+GEM=/opt/td-agent/embedded/bin/gem
+RUBY=/opt/td-agent/embedded/bin/ruby
+RAKE=/opt/td-agent/embedded/bin/rake
+FPM=fpm
+
+# Bundle will unpack gems into the bundle directory
+BUNDLE_DIR=./vendor/bundle
+# Bundle will save gems in the cache directory
+CACHE_DIR=./vendor/cache
+PKG_DIR=pkg
+# Bundle will store settings and locks in the dot_bundle directory
+DOT_BUNDLE=./.bundle
+
+GEMSPEC := ${wildcard *.gemspec}
+NAME    := ${shell $(RUBY) -e 'puts Gem::Specification::load("$(GEMSPEC)").name'}
+VERSION := ${shell $(RUBY) -e 'puts Gem::Specification::load("$(GEMSPEC)").version.to_s'}
+GEMFILE := ${shell $(RUBY) -e 'puts Gem::Specification::load("$(GEMSPEC)").file_name'}
+DEBFILE := $(NAME)_$(VERSION)_all.deb
+
+all: $(DEBFILE)
+
+$(BUNDLE_DIR): Gemfile.lock
+	$(BUNDLE) install --frozen --path $(BUNDLE_DIR)
+
+$(CACHE_DIR): $(BUNDLE_DIR) Gemfile.lock
+	$(BUNDLE) package --no-install
+
+$(GEMFILE): $(BUNDLE_DIR)
+	$(BUNDLE) exec $(GEM) build -V $(GEMSPEC)
+
+$(DEBFILE): $(GEMFILE) $(CACHE_DIR)
+	$(BUNDLE) exec $(RAKE) gemdep:minimal
+	$(FPM) --input-type gem --output-type deb \
+	  --no-auto-depends \
+	  --no-gem-fix-name \
+	  --gem-gem $(GEM) \
+	  --depends 'td-agent > 2' \
+	  --deb-build-depends 'td-agent > 2' \
+	  pkg/*.gem $(GEMFILE)
+
+clean:
+	-$(BUNDLE) exec $(RAKE) clobber
+	rm -rf $(BUNDLE_DIR) $(CACHE_DIR) $(DOT_BUNDLE) $(GEMFILE) $(DEBFILE)
+
+.PHONY: all clean

data/README.md

@@ -7,22 +7,74 @@ Using fluent-plugin-logdna, you can send the logs you collect with Fluentd to Lo
 ## Instructions
 
 * Install [Fluentd](http://www.fluentd.org/download)
-* `gem install fluent-plugin-logdna` or `td-agent-gem install fluent-plugin-logdna` if you are using td-agent.
-* Make sure you have a LogDNA account.
-* Configure Fluentd like the following:
+* Alternative install if using fluentd package manager (td-agent): `td-agent-gem install fluent-plugin-logdna`
+* Add the contents below to `/etc/fluent/fluent.conf`. For td-agent, use `/etc/td-agent/td-agent.conf`:
+* Alternative install without td-agent is: `gem install fluent-plugin-logdna`
 
-~~~~~
-<match your_match>
+~~~~~configuration
+<match **>
   @type logdna
-  api_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx # paste your api key here (required)
-  hostname my_host                         # replace with your hostname (required)
-  mac C0:FF:EE:C0:FF:EE                    # replace with host mac address
-  ip 127.0.0.1                             # replace with host ip address
-  app my_app                               # replace with your app name
+  api_key xxxxxxxxxxxxxxxxxxxxxxxxxxx        # paste your api key here (required)
+  hostname "#{Socket.gethostname}"           # your hostname (required)
+  app my_app                                 # replace with your app name
+  #mac C0:FF:EE:C0:FF:EE                     # optional mac address
+  #ip 127.0.0.1                              # optional ip address
+  #tags web,dev                              # optional tags
+  request_timeout 30000 ms                   # optional timeout for upload request, supports seconds (s, default) and milliseconds (ms) suffixes, default 30 seconds
+  buffer_chunk_limit 1m                      # do not increase past 8m (8MB) or your logs will be rejected by our server.
+  flush_at_shutdown true                     # only needed with file buffer
 </match>
 ~~~~~
+
 * Restart fluentd to pick up the configuration changes.
+* `sudo /etc/init.d/td-agent stop`
+* `sudo /etc/init.d/td-agent start`
+
+### Recommended Configuration Parameters
+
+* buffer_type
+  * We recommend setting this to memory for development and file for production (file setting requires a buffer_path).
+* buffer_queue_limit, buffer_chunk_limit
+  * We do not recommend increasing buffer_chunk_limit past 8MB.
+* flush_interval
+  * Default is 60s. We recommend keeping this well above 5s.
+* retry_wait, max_retry_wait, retry_limit, disable_retry_limit
+  * We recommend increasing these values if you are encountering problems.
+
+### Options
+
+* App name and log level can also be provided on a line-by-line basis over JSON:
+* `_app` and `level` will override the config
+
+If you don't have a LogDNA account, you can create one on [https://logdna.com](https://logdna.com) or if you're on macOS w/[Homebrew](https://brew.sh) installed:
+
+~~~~~bash
+brew cask install logdna-cli
+logdna register <email>
+# now paste the api key above
+~~~~~
+
+### LogDNA Pay-per-gig Pricing
+
+Our [paid plans](https://logdna.com/#pricing) start at $1.25/GB per month, pay for what you use / no fixed data buckets / all paid plans include all features.
+
+## Building a debian package for td-agent
+
+If you use td-agent you can build a debian package instead of installing via
+td-agent-gem. This requires that td-agent is already installed and that you've
+installed [fpm](http://fpm.readthedocs.io/en/latest/index.html). Then just run
+`make` in your git directory.
+
+~~~~~bash
+gem install --no-ri --no-rdoc fpm
+git clone https://github.com/logdna/fluent-plugin-logdna
+cd fluent-plugin-logdna
+make
+sudo dpkg -i fluent-plugin-logdna*.deb
+~~~~~
+
+## Additional Options
 
-For advanced configuration options, refer to the [buffered output parameters documentation.](http://docs.fluentd.org/articles/output-plugin-overview#buffered-output-parameters)
+For advanced configuration options, please refer to the [buffered output parameters documentation.](http://docs.fluentd.org/articles/output-plugin-overview#buffered-output-parameters)
 
 Questions or concerns? Contact [support@logdna.com](mailto:support@logdna.com).

data/Rakefile

@@ -0,0 +1,48 @@
+require "set"
+
+# Directory to place gems to bundle with our main gem
+PACKAGE_DIR = "pkg"
+require "rake/clean"
+CLOBBER.include PACKAGE_DIR
+
+desc "Copies gem dependencies to the #{PACKAGE_DIR} folder"
+task "gemdep" do
+  gem_copy gen_gem_list
+end
+
+desc "Copies gem dependencies which don't overlap with td-agent to the #{PACKAGE_DIR} folder"
+task "gemdep:minimal" do
+  MINIMAL_GEMS = %W[
+    unf_ext
+    unf
+    domain_name
+    http-cookie
+    http-form_data
+    http
+  ]
+  gem_copy gen_gem_list MINIMAL_GEMS
+end
+
+def gen_gem_list(subset = [])
+  myself = Bundler.load_gemspec(Dir[File.join(Bundler::SharedHelpers.pwd, "{,*}.gemspec")].first)
+  to_pull = Set[]
+  Bundler.locked_gems.specs.each do |spec|
+    next if spec.name == myself.name
+    if subset.empty?
+      to_pull.add(spec)
+    else
+      if subset.include? spec.name
+        to_pull.add(spec)
+      end
+    end
+  end
+  to_pull
+end
+
+# This assumes that bundle install has already been run (which populates
+# Bundler.app_cache with the gems of dependencies).
+def gem_copy(specs)
+  Bundler.mkdir_p PACKAGE_DIR
+  sources = specs.collect { |s| Bundler.app_cache.join "#{s.full_name}.gem" }
+  FileUtils.cp sources, PACKAGE_DIR, :verbose => true
+end

data/fluent-plugin-logdna.gemspec

@@ -3,7 +3,7 @@ require 'date'
 
 Gem::Specification.new do |s|
   s.name        = 'fluent-plugin-logdna'
-  s.version     = '0.1.5'
+  s.version     = '0.3.1'
   s.date        = Date.today.to_s
   s.summary     = 'LogDNA plugin for Fluentd'
   s.description = 'Fluentd plugin for supplying output to LogDNA.'
@@ -17,6 +17,7 @@ Gem::Specification.new do |s|
   s.files         = `git ls-files -z`.split("\x0")
   s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
   s.required_ruby_version = Gem::Requirement.new('>= 2.0.0')
+  s.add_development_dependency "bundler", "~> 1.16"
   s.add_runtime_dependency 'fluentd', '>= 0.12.0', '< 2'
   s.add_runtime_dependency 'http', '~> 2.0', '>= 2.0.3'
 end

data/lib/fluent/plugin/out_logdna.rb

@@ -4,17 +4,36 @@ module Fluent
   class LogDNAOutput < Fluent::BufferedOutput
     Fluent::Plugin.register_output('logdna', self)
 
-    INGESTER_DOMAIN = 'https://logs.logdna.com'.freeze
+    MAX_RETRIES = 5
 
     config_param :api_key, :string, secret: true
     config_param :hostname, :string
     config_param :mac, :string, default: nil
+    config_param :tags, :string, default: nil
     config_param :ip, :string, default: nil
     config_param :app, :string, default: nil
+    config_param :file, :string, default: nil
+    config_param :ingester_domain, :string, default: 'https://logs.logdna.com'
+    config_param :request_timeout, :string, default: '30'
 
     def configure(conf)
       super
       @host = conf['hostname']
+
+      # make these two variables globals
+      timeout_unit_map = { s: 1.0, ms: 0.001  }
+      timeout_regex = Regexp.new("^([0-9]+)\s*(#{timeout_unit_map.keys.join("|")})$")
+
+      # this section goes into this part of the code
+      num_component = 30.0
+      unit_component = 's'
+
+      timeout_regex.match(@request_timeout) do |match|
+        num_component = match[1].to_f
+        unit_component = match[2]
+      end
+
+      @request_timeout = num_component * timeout_unit_map[unit_component.to_sym]
     end
 
     def start
@@ -22,7 +41,9 @@ module Fluent
       require 'json'
       require 'base64'
       require 'http'
-      @ingester = HTTP.persistent INGESTER_DOMAIN
+      HTTP.default_options = { :keep_alive_timeout => 60 }
+      @ingester = HTTP.persistent @ingester_domain
+      @requests = Queue.new
     end
 
     def shutdown
@@ -37,7 +58,8 @@ module Fluent
     def write(chunk)
       body = chunk_to_body(chunk)
       response = send_request(body)
-      handle(response)
+      raise 'Encountered server error' if response.code >= 400
+      response.flush
     end
 
     private
@@ -47,7 +69,7 @@ module Fluent
 
       chunk.msgpack_each do |(tag, time, record)|
         line = gather_line_data(tag, time, record)
-        data << line
+        data << line unless line[:line].empty?
       end
 
       { lines: data }
@@ -57,31 +79,29 @@ module Fluent
       line = {
         level: record['level'] || record['severity'] || tag.split('.').last,
         timestamp: time,
-        line: record['message'] || record.to_json
+        line: record.to_json
       }
+      # At least one of "file" or "app" is required.
+      line[:file] = record['file']
+      line[:file] ||= @file if @file
+      line.delete(:file) if line[:file].nil?
       line[:app] = record['_app'] || record['app']
       line[:app] ||= @app if @app
       line.delete(:app) if line[:app].nil?
+      line[:env] = record['env']
+      line.delete(:env) if line[:env].nil?
+      line[:meta] = record['meta']
+      line.delete(:meta) if line[:meta].nil?
       line
     end
 
-    def handle(response)
-      if response.code >= 400
-        print "Error connecting to LogDNA ingester. \n"
-        print "Details: #{response}"
-      else
-        print "Success! #{response}"
-      end
-
-      response.flush
-    end
-
     def send_request(body)
       now = Time.now.to_i
-      url = "/logs/ingest?hostname=#{@host}&mac=#{@mac}&ip=#{@ip}&now=#{now}"
+      url = "/logs/ingest?hostname=#{@host}&mac=#{@mac}&ip=#{@ip}&now=#{now}&tags=#{@tags}"
       @ingester.headers('apikey' => @api_key,
                         'content-type' => 'application/json')
-               .post(url, json: body)
+                .timeout(connect: @request_timeout, write: @request_timeout, read: @request_timeout)
+                .post(url, json: body)
     end
   end
 end

data/test/helper.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+require "test-unit"
+require "fluent/test"
+require "fluent/test/driver/output"
+require "fluent/test/helpers"
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_out_logdna.rb

@@ -0,0 +1,110 @@
+# https://docs.fluentd.org/plugin-development/api-plugin-output#how-to-write-tests
+
+require_relative '../helper'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_stdout'
+require 'fluent/plugin/input'
+require 'webmock/test_unit'
+include WebMock::API
+
+require 'lib/fluent/plugin/out_logdna.rb'
+
+class LogdnaOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+
+    stub_request(:post, /logs.logdna.com\/logs\/ingest/)
+  end
+
+  # default configuration for tests
+  CONFIG = %[
+    api_key this-is-my-key
+    hostname "localhost"
+    app my_app
+    mac C0:FF:EE:C0:FF:EE
+    ip 127.0.0.1
+    tags  "my-tag"
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::LogDNAOutput).configure(conf)
+  end
+
+  sub_test_case 'configuration' do
+    test 'instantiate the plugin and check default config values' do
+        d = create_driver
+        time = event_time
+
+        # check defaults
+        assert_equal 'https://logs.logdna.com', d.instance.ingester_domain
+        assert_equal 30, d.instance.request_timeout
+    end
+
+    test 'instantiate the plugin and check setting config values' do
+        conf = %[
+            api_key this-is-my-key
+            hostname "localhost"
+            app my_app
+            mac C0:FF:EE:C0:FF:EE
+            ip 127.0.0.1
+            tags  "my-tag"
+            request_timeout 17s
+        ]
+
+        d = create_driver(conf)
+
+        # check set config values
+        assert_equal 'my-tag', d.instance.tags
+        assert_equal 17, d.instance.request_timeout
+    end
+
+    test 'instantiate the plugin with ms request_timeout value' do
+
+        conf = %[
+            api_key this-is-my-key
+            hostname "localhost"
+            app my_app
+            mac C0:FF:EE:C0:FF:EE
+            ip 127.0.0.1
+            tags  "my-tag"
+            request_timeout 17000 ms
+        ]
+
+        d = create_driver(conf)
+
+        # check set config values
+        assert_equal 17, d.instance.request_timeout
+    end
+
+    test 'instantiate the plugin with nonesense request_timeout value' do
+
+        conf = %[
+            api_key this-is-my-key
+            hostname "localhost"
+            app my_app
+            mac C0:FF:EE:C0:FF:EE
+            ip 127.0.0.1
+            tags  "my-tag"
+            request_timeout "asdf ms"
+        ]
+
+        d = create_driver(conf)
+
+        # check set config values
+        assert_equal 30, d.instance.request_timeout
+    end
+  end
+
+  test 'simple #write' do
+    d = create_driver
+    time = event_time
+
+    d.run do
+        d.feed("output.test", time, {'foo' => 'bar', 'message' => 'myLine'})
+        d.feed("output.test", time, {'foo' => 'bar', 'message' => 'myLine'})
+    end
+
+    assert_equal(2, d.formatted.size)
+  end
+
+end

metadata

@@ -1,53 +1,67 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-logdna
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.3.1
 platform: ruby
 authors:
 - Edwin Lai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-16 00:00:00.000000000 Z
+date: 2020-07-21 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.12.0
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.12.0
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: '2'
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.3
 description: Fluentd plugin for supplying output to LogDNA.
@@ -56,12 +70,18 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .ruby-version
+- ".gitignore"
+- ".ruby-version"
+- Gemfile
+- Gemfile.lock
 - LICENSE
+- Makefile
 - README.md
+- Rakefile
 - fluent-plugin-logdna.gemspec
 - lib/fluent/plugin/out_logdna.rb
+- test/helper.rb
+- test/plugin/test_out_logdna.rb
 homepage: https://github.com/logdna/fluent-plugin-logdna
 licenses:
 - MIT
@@ -72,17 +92,16 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.0.14.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: LogDNA plugin for Fluentd