fluent-plugin-heroku-syslog-http 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 232b9309f8c101e5ff0845ef5a8330661d98816d3c25068914a7e2dfbdf5566a
+  data.tar.gz: d5f49511ab61357336c8ef9d99936781709a044994c0af8f0cf0c5e22d537623
+SHA512:
+  metadata.gz: 14d6c03abb519a8f07fbf2ba0e0f894aee8a9b20aa7c3ab2699b6c44317feff51f54b2ce4bd20d0f5e197cdd6090c8994406cb1651bd1e9f8071c731b3197fc2
+  data.tar.gz: 13bf91845d3591a7df8206ab1730c5db0d4350b92678ec393f60567956788da03e127dca649a423024db467929a7f43e6d3fdd35dd46f8f30f58088dfa61e9af

data/.circleci/config.yml

@@ -0,0 +1,35 @@
+aliases:
+  - &step_cache_restore_deps
+    restore_cache:
+      keys:
+        - drivy-rails-fluent-plugin-heroku-syslog-http-{{ .Branch }}-{{ .Revision }}
+        - drivy-rails-fluent-plugin-heroku-syslog-http-{{ .Branch }}
+        - drivy-rails-fluent-plugin-heroku-syslog-http-
+  - &step_cache_save_deps
+    save_cache:
+      key: drivy-rails-fluent-plugin-heroku-syslog-http-{{ .Branch }}-{{ .Revision }}
+      paths: [ vendor/bundle ]
+  - &step_bundler_install_deps
+    run: bundle install --path vendor/bundle --jobs=2 --without development:deployment
+
+version: 2
+jobs:
+  test:
+    docker:
+      - image: circleci/ruby:2.4
+    steps:
+      - checkout
+      - *step_cache_restore_deps
+      # Bundler install
+      - *step_bundler_install_deps
+      - run: bundle clean
+      - *step_cache_save_deps
+      # Run tests
+      - run: bundle exec rake
+
+workflows:
+  version: 2
+
+  test-and-deploy:
+    jobs:
+      - checkout

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+test.conf

data/.rubocop.yml

@@ -0,0 +1,17 @@
+Style/Documentation:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/LineLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/PerceivedComplexity:
+  Enabled: false

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+ruby '2.4.4'
+
+# Specify your gem's dependencies in fluent-plugin-heroku-syslog.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,14 @@
+Copyright (c) 2014-2018 Kazuyuki Honda
+Copyright (c) 2108- Drivy
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,57 @@
+# fluent-plugin-heroku-syslog-http
+
+Plugins to accept and parse syslog input from [heroku http(s) drains](https://devcenter.heroku.com/articles/log-drains#http-s-drains), based on fluentd'd [http input](https://docs.fluentd.org/v1.0/articles/in_http) and [regexp parser](https://docs.fluentd.org/v1.0/articles/parser_regexp)
+
+## Installation
+
+Install with gem or fluent-gem command as:
+
+```
+# for fluentd
+$ gem install fluent-plugin-heroku-syslog-http
+
+# for td-agent
+$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-heroku-syslog-http
+```
+
+## Usage
+
+### Configure heroku_syslog_http input
+
+```
+<source>
+  type heroku_syslog_http
+  port 9880
+  bind 0.0.0.0
+  tag  heroku
+  drain_ids ["YOUR-HEROKU-DRAIN-ID"] # optional
+</source>
+```
+
+### Example
+
+Heroku's http syslog format:
+`00 <13>1 2014-01-01T01:23:45.123456+00:00 host app web.1 - foo`
+
+Will parse the following key/values:
+```
+{
+  'syslog.pri' => '13',
+  'syslog.facility' => 'user',
+  'syslog.severity' => 'notice',
+  'syslog.hostname' => 'host',
+  'syslog.appname' => 'app',
+  'syslog.procid' => 'web.1',
+  'syslog.timestamp' => '2014-01-29T06:25:52.589365+00:00',
+  'message' => 'foo'
+}
+```
+
+
+## Copyright
+
+- Copyright
+  - Copytight(C) 2018- Drivy
+  - Copyright(C) 2014-2018 Kazuyuki Honda (hakobera)
+- License
+  - Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task default: :test

data/fluent-plugin-heroku-syslog.gemspec

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |gem|
+  gem.name          = 'fluent-plugin-heroku-syslog-http'
+  gem.version       = '0.2.0'
+  gem.authors       = ['Drivy', 'Kazuyuki Honda']
+  gem.email         = ['sre@drivy.com']
+  gem.description   = 'fluent plugin to drain heroku syslog'
+  gem.summary       = 'fluent plugin to drain heroku syslog'
+  gem.homepage      = 'https://github.com/drivy/fluent-plugin-heroku-syslog-http'
+  gem.license       = 'APLv2'
+
+  gem.files         = `git ls-files`.split($ORS)
+  gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  gem.add_runtime_dependency 'fluentd', '>= 1.0.0'
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency('test-unit', ['~> 3.1.0'])
+end

data/lib/fluent/plugin/in_heroku_syslog_http.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'fluent/plugin/in_http'
+
+module Fluent
+  module Plugin
+    class HerokuSyslogHttpInput < HttpInput
+      Fluent::Plugin.register_input('heroku_syslog_http', self)
+
+      config_section :parse do
+        config_set_default :@type, 'heroku_syslog_http'
+      end
+      config_param :drain_ids, :array, default: nil
+
+      private
+
+      def parse_params_with_parser(params)
+        content = params[EVENT_RECORD_PARAMETER]
+        raise "'#{EVENT_RECORD_PARAMETER}' parameter is required" unless content
+
+        records = []
+        messages = content.split("\n")
+        messages.each do |msg|
+          @parser.parse(msg) do |time, record|
+            raise "Could not parse event: #{content}" if record.nil?
+
+            record['syslog.timestamp'] ||= Time.at(time).utc.strftime('%Y-%m-%dT%H:%M:%S%z')
+            record['logplex.drain_id'] = params['HTTP_LOGPLEX_DRAIN_TOKEN']
+            record['logplex.frame_id'] = params['HTTP_LOGPLEX_FRAME_ID']
+            unless @drain_ids.nil? || @drain_ids.include?(record['logplex.drain_id'])
+              log.warn "drain_id not match: #{msg.inspect}"
+              next
+            end
+            records << record
+          end
+        end
+        [nil, records]
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_heroku_syslog_http.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'fluent/plugin/in_http'
+require 'fluent/plugin/parser_regexp'
+
+module Fluent
+  module Plugin
+    class HerokuSyslogHttpParser < RegexpParser
+      Fluent::Plugin.register_parser('heroku_syslog_http', self)
+
+      SYSLOG_HTTP_REGEXP = %r{^([0-9]+) +\<(?<syslog.pri>[0-9]+)\>([0-9]+) (?<syslog.timestamp>[^ ]+) (?<syslog.hostname>[^ ]+) (?<syslog.appname>[^ ]+) (?<syslog.procid>[^ ]+) - *(?<message>.*)$}
+
+      FACILITY_MAP = {
+        0   => 'kern',
+        1   => 'user',
+        2   => 'mail',
+        3   => 'daemon',
+        4   => 'auth',
+        5   => 'syslog',
+        6   => 'lpr',
+        7   => 'news',
+        8   => 'uucp',
+        9   => 'cron',
+        10  => 'authpriv',
+        11  => 'ftp',
+        12  => 'ntp',
+        13  => 'audit',
+        14  => 'alert',
+        15  => 'at',
+        16  => 'local0',
+        17  => 'local1',
+        18  => 'local2',
+        19  => 'local3',
+        20  => 'local4',
+        21  => 'local5',
+        22  => 'local6',
+        23  => 'local7'
+      }.freeze
+
+      SEVERITY_MAP = {
+        0  => 'emerg',
+        1  => 'alert',
+        2  => 'crit',
+        3  => 'err',
+        4  => 'warn',
+        5  => 'notice',
+        6  => 'info',
+        7  => 'debug'
+      }.freeze
+
+      config_set_default :expression, SYSLOG_HTTP_REGEXP
+      config_set_default :time_key, 'syslog.timestamp'
+      config_set_default :keep_time_key, true
+
+      def parse_prival(record)
+        if record && record['syslog.pri']
+          pri = record['syslog.pri'].to_i
+          record['syslog.facility'] = FACILITY_MAP[pri >> 3]
+          record['syslog.severity'] = SEVERITY_MAP[pri & 0b111]
+        end
+        record
+      end
+
+      def parse(text)
+        super(text) do |time, record|
+          yield time, parse_prival(record)
+        end
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'rubygems'
+require 'bundler'
+
+require 'fluent/log'
+require 'fluent/test'
+
+def unused_port
+  s = TCPServer.open(0)
+  port = s.addr[1]
+  s.close
+  port
+end

data/test/plugin/test_in_heroku_syslog_http.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require 'helper'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_heroku_syslog_http'
+
+require 'net/http'
+
+class HerokuSyslogHttpInputTest < Test::Unit::TestCase
+  class << self
+    def startup
+      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+      @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
+      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+    end
+
+    def shutdown
+      @server.close
+    end
+  end
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  PORT = unused_port
+  CONFIG = %(
+    @type heroku_syslog_http
+    port #{PORT}
+    bind 127.0.0.1
+    body_size_limit 10m
+    keepalive_timeout 5
+  )
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::HerokuSyslogHttpInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal PORT, d.instance.port
+    assert_equal '127.0.0.1', d.instance.bind
+    assert_equal 10 * 1024 * 1024, d.instance.body_size_limit
+    assert_equal 5, d.instance.keepalive_timeout
+  end
+
+  def test_time_format
+    messages = [
+      '59 <13>1 2014-01-29T06:25:52.589365+00:00 host app web.1 - foo',
+      '59 <13>1 2014-01-30T07:35:00.123456+09:00 host app web.1 - bar'
+    ]
+    event_times = [
+      Time.strptime('2014-01-29T06:25:52+00:00', '%Y-%m-%dT%H:%M:%S%z').to_i,
+      Time.strptime('2014-01-30T07:35:00+09:00', '%Y-%m-%dT%H:%M:%S%z').to_i
+    ]
+
+    d = create_driver
+    d.run(expect_records: 2, timeout: 5) do
+      res = post(messages)
+      assert_equal '', res.body
+      assert_equal '200', res.code
+    end
+
+    assert_equal event_times[0], d.events[0][1]
+    assert_equal event_times[1], d.events[1][1]
+  end
+
+  def test_msg_size
+    messages = [
+      '00 <13>1 2014-01-01T01:23:45.123456+00:00 host app web.1 - ' + 'x' * 100,
+      '00 <13>1 2014-01-01T01:23:45.123456+00:00 host app web.1 - ' + 'x' * 1024
+    ]
+
+    d = create_driver
+    d.run(expect_records: 2, timeout: 5) do
+      res = post(messages)
+      assert_equal '200', res.code
+    end
+
+    assert_equal 'x' * 100, d.events[0][2]['message']
+    assert_equal 'x' * 1024, d.events[1][2]['message']
+  end
+
+  def test_accept_matched_drain_id_multiple
+    messages = [
+      '00 <13>1 2014-01-01T01:23:45.123456+00:00 host app web.1 - foo',
+      '00 <13>1 2014-01-01T01:23:45.123456+00:00 host app web.1 - bar'
+    ]
+    d = create_driver(CONFIG + %(
+      drain_ids ["abc", "d.fc6b856b-3332-4546-93de-7d0ee272c3bd"]
+    ))
+    d.run(expect_records: 2, timeout: 5) do
+      res = post(messages)
+      assert_equal '200', res.code
+    end
+
+    assert_equal 2, d.events.length
+  end
+
+  def test_ignore_unmatched_drain_id
+    messages = [
+      '00 <13>1 2014-01-01T01:23:45.123456+00:00 host app web.1 - foo',
+      '00 <13>1 2014-01-01T01:23:45.123456+00:00 host app web.1 - bar'
+    ]
+    d = create_driver(CONFIG + %(
+      drain_ids ["abc"]
+    ))
+    d.run(expect_records: 0, timeout: 5) do
+      res = post(messages)
+      assert_equal '200', res.code
+    end
+
+    assert_equal 0, d.events.length
+  end
+
+  def test_logplex_metas
+    messages = [
+      '00 <13>1 2014-01-01T01:23:45.123456+00:00 host app web.1 - ' + 'x' * 100
+    ]
+    d = create_driver
+    d.run(expect_records: 1, timeout: 5) do
+      res = post(messages)
+      assert_equal '200', res.code
+    end
+
+    assert_equal '09C557EAFCFB6CF2740EE62F62971098', d.events[0][2]['logplex.frame_id']
+    assert_equal 'd.fc6b856b-3332-4546-93de-7d0ee272c3bd', d.events[0][2]['logplex.drain_id']
+  end
+
+  def post(messages)
+    # https://github.com/heroku/logplex/blob/master/doc/README.http_drains.md
+    http = Net::HTTP.new('127.0.0.1', PORT)
+    headers = {
+      'Content-Type' => 'application/logplex-1',
+      'Logplex-Msg-Count' => messages.length.to_s,
+      'Logplex-Frame-Id' => '09C557EAFCFB6CF2740EE62F62971098',
+      'Logplex-Drain-Token' => 'd.fc6b856b-3332-4546-93de-7d0ee272c3bd',
+      'User-Agent' => 'Logplex/v49'
+    }
+    req = Net::HTTP::Post.new('/heroku', headers)
+    req.body = messages.join("\n")
+    http.request(req)
+  end
+end

data/test/plugin/test_parser_heroku_syslog_http.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'helper'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/in_heroku_syslog_http'
+
+require 'net/http'
+
+# Stolen from fluentd/test/helper.rb
+class Hash
+  def corresponding_proxies
+    @corresponding_proxies ||= []
+  end
+
+  def to_masked_element
+    self
+  end
+end
+
+class HerokuSyslogHttpParseTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf = {})
+    d = Struct.new(:instance).new
+    d.instance = Fluent::Plugin::HerokuSyslogHttpParser.new
+    d.instance.configure(conf)
+    d
+  end
+
+  def test_parsing_with_default_conf
+    text = '59 <13>1 2014-01-29T06:25:52.589365+00:00 host app web.1 - foo'
+    expected_time = Time.strptime('2014-01-29T07:25:52+01:00', '%Y-%m-%dT%H:%M:%S%z').to_i
+    event = {
+      'syslog.pri' => '13',
+      'syslog.facility' => 'user',
+      'syslog.severity' => 'notice',
+      'syslog.hostname' => 'host',
+      'syslog.appname' => 'app',
+      'syslog.procid' => 'web.1',
+      'syslog.timestamp' => '2014-01-29T06:25:52.589365+00:00',
+      'message' => 'foo'
+    }
+    d = create_driver
+    d.instance.parse(text) do |time, record|
+      assert_equal expected_time, time
+      assert_equal event, record
+    end
+  end
+
+  def test_parsing_pri_conf
+    d = create_driver
+
+    d.instance.parse('00 <13>1 2014-01-29T06:25:52.589365+00:00 host app web.1 - foo') do |_, record|
+      assert_equal 'notice', record['syslog.severity']
+      assert_equal 'user', record['syslog.facility']
+    end
+    d.instance.parse('00 <42>1 2014-01-29T06:25:52.589365+00:00 host app web.1 - foo') do |_, record|
+      assert_equal 'crit', record['syslog.severity']
+      assert_equal 'syslog', record['syslog.facility']
+    end
+    d.instance.parse('00 <27>1 2014-01-29T06:25:52.589365+00:00 host app web.1 - foo') do |_, record|
+      assert_equal 'err', record['syslog.severity']
+      assert_equal 'daemon', record['syslog.facility']
+    end
+  end
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-heroku-syslog-http
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Drivy
+- Kazuyuki Honda
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-10-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+description: fluent plugin to drain heroku syslog
+email:
+- sre@drivy.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".gitignore"
+- ".rubocop.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-heroku-syslog.gemspec
+- lib/fluent/plugin/in_heroku_syslog_http.rb
+- lib/fluent/plugin/parser_heroku_syslog_http.rb
+- test/helper.rb
+- test/plugin/test_in_heroku_syslog_http.rb
+- test/plugin/test_parser_heroku_syslog_http.rb
+homepage: https://github.com/drivy/fluent-plugin-heroku-syslog-http
+licenses:
+- APLv2
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: fluent plugin to drain heroku syslog
+test_files:
+- test/helper.rb
+- test/plugin/test_in_heroku_syslog_http.rb
+- test/plugin/test_parser_heroku_syslog_http.rb