fluent-plugin-docker-journald-concat 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7698d8dc7b9b2976c8963c23d5ff7ee1f9793554
+  data.tar.gz: f18bf9669f0f5988157bced5b64ec191dd125cd3
+SHA512:
+  metadata.gz: 49b5f3dd8bf8edc1b246f7059e3509118bead0956401e18a0108032825289e5fc36d5e6510f85b1cec0f4500cdf27c6f064abe8145775c5d50c26a07c38a5c9b
+  data.tar.gz: 35c410ae81440d38777aedb814afc7713748356b82c763bf49f5dfff78c7fb639df3d570349dacd0017ec7fe7c2b98faa281b0abd9d6656a8d8f9242d89bb6ba

data/.gitignore

@@ -0,0 +1,50 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.rubocop.yml

@@ -0,0 +1,55 @@
+AllCops:
+  TargetRubyVersion: 2.3
+  DisplayCopNames: true
+  Exclude:
+    - "vendor/**/*"
+
+Lint/UnderscorePrefixedVariableName:
+  Enabled: false
+Lint/UnusedMethodArgument:
+  Enabled: false
+
+Metrics/LineLength:
+  Enabled: false
+Metrics/ClassLength:
+  Exclude:
+    - "test/**/*.rb"
+Metrics/ParameterLists:
+  Max: 7
+Metrics/MethodLength:
+  Max: 30
+
+Style/AsciiComments:
+  Enabled: false
+Style/HashSyntax:
+  Exclude:
+    - "**/*.rake"
+    - "Rakefile"
+Style/FrozenStringLiteralComment:
+  Enabled: false
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+Style/TrailingCommaInLiteral:
+  Enabled: false
+Style/TrailingCommaInArguments:
+  Enabled: false
+Style/FileName:
+  Enabled: false
+Style/Documentation:
+  Enabled: false
+Style/WordArray:
+  Enabled: false
+Style/BarePercentLiterals:
+  EnforcedStyle: percent_q
+Style/SpaceInsideBlockBraces:
+  EnforcedStyle: space
+  SpaceBeforeBlockParameters: false
+Style/BracesAroundHashParameters:
+  Enabled: false
+Style/MultilineMethodCallIndentation:
+  EnforcedStyle: indented
+Style/DoubleNegation:
+  Enabled: false
+Style/EmptyCaseCondition:
+  Enabled: false
+

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in fluent-plugin-concat.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,55 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-docker-journald-concat (0.1.0)
+      fluentd (~> 0.14)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    cool.io (1.5.1)
+    fluentd (0.14.21)
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 0.7.0, < 2.0.0)
+      ruby_dig (~> 0.0.2)
+      serverengine (>= 2.0.4, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (~> 0.1.7)
+      tzinfo (~> 1.0)
+      tzinfo-data (~> 1.0)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.6.0)
+    msgpack (1.1.0)
+    power_assert (1.1.0)
+    rake (12.1.0)
+    rr (1.2.1)
+    ruby_dig (0.0.2)
+    serverengine (2.0.5)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.1.9)
+    test-unit (3.2.6)
+      power_assert
+    test-unit-rr (1.0.5)
+      rr (>= 1.1.1)
+      test-unit (>= 2.5.2)
+    thread_safe (0.3.6)
+    tzinfo (1.2.3)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2017.2)
+      tzinfo (>= 1.0.0)
+    yajl-ruby (1.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.11)
+  fluent-plugin-docker-journald-concat!
+  rake (~> 12.0)
+  test-unit (~> 3.1)
+  test-unit-rr (~> 1.0)
+
+BUNDLED WITH
+   1.14.3

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Oleksandr Kushchenko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,4 @@
+# fluent-plugin-docker-journald-concat
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-docker-journald-concat.svg)](https://badge.fury.io/rb/fluent-plugin-docker-journald-concat)
+
+Fluentd Filter plugin to concatenate partial log messages generated by Docker daemon with Journald logging driver

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+task :default => :test
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.test_files = Dir["test/**/test_*.rb"].sort
+  t.verbose = false
+  t.warning = true
+end

data/fluent-plugin-docker-journald-concat.gemspec

@@ -0,0 +1,23 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-docker-journald-concat"
+  spec.version       = "0.1.1"
+  spec.authors       = ["Oleksandr Kushchenko"]
+  spec.email         = ["gearok@gmail.com"]
+
+  spec.summary       = "Fluentd Filter plugin to concatenate partial log messages generated by Docker daemon with Journald logging driver"
+  spec.description   = "Fluentd Filter plugin to concatenate partial log messages generated by Docker daemon with Journald logging driver"
+  spec.homepage      = "https://github.com/okushchenko/fluent-plugin-docker-journald-concat"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject {|f| f.match(%r{^(test|spec|features)/}) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "fluentd", "~> 0.14"
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "test-unit", "~> 3.1"
+  spec.add_development_dependency "test-unit-rr", "~> 1.0"
+end

data/lib/fluent/plugin/filter_docker_journal_concat.rb

@@ -0,0 +1,160 @@
+require "fluent/plugin/filter"
+
+module Fluent::Plugin
+  class DockerJournalConcatFilter < Filter
+    Fluent::Plugin.register_filter("docker_journal_concat", self)
+
+    helpers :timer, :event_emitter
+
+    desc "The key to merge records on"
+    config_param :key, :string, default: "message"
+    desc "The key to test if record is multipart against"
+    config_param :match_key, :string, default: "container_partial_message"
+    desc "The key to determine which stream an event belongs to"
+    config_param :stream_identity_key, :string, default: "container_id"
+    desc "The interval between data flushes, 0 means disable timeout"
+    config_param :flush_interval, :time, default: 60
+    desc "The label name to handle timeout"
+    config_param :timeout_label, :string, default: nil
+
+    class TimeoutError < StandardError
+    end
+
+    def initialize
+      super
+
+      @buffer = Hash.new {|h, k| h[k] = [] }
+      @timeout_map = Hash.new {|h, k| h[k] = Fluent::Engine.now }
+    end
+
+    def configure(conf)
+      super
+
+      @separator = ""
+    end
+
+    def start
+      super
+
+      @finished = false
+      timer_execute(:filter_concat_timer, 1, &method(:on_timer))
+    end
+
+    def shutdown
+      super
+
+      @finished = true
+      flush_shutdown_buffer
+    end
+
+    def filter_stream(tag, es)
+      new_es = Fluent::MultiEventStream.new
+      es.each do |time, record|
+        if /\Afluent\.(?:trace|debug|info|warn|error|fatal)\z/ =~ tag
+          new_es.add(time, record)
+          next
+        end
+        begin
+          flushed_es = process(tag, time, record)
+          unless flushed_es.empty?
+            flushed_es.each do |_time, new_record|
+              new_es.add(time, record.merge(new_record))
+            end
+          end
+        rescue => e
+          router.emit_error_event(tag, time, record, e)
+        end
+      end
+      new_es
+    end
+
+    private
+
+    def on_timer
+      return if @flush_interval <= 0
+      return if @finished
+      flush_timeout_buffer
+    end
+
+    def process(tag, time, record)
+      stream_identity = "#{tag}:#{record[@stream_identity_key]}"
+      @timeout_map[stream_identity] = Fluent::Engine.now
+      new_es = Fluent::MultiEventStream.new
+      if partial?(record[@match_key])
+        # partial record
+        @buffer[stream_identity] << [tag, time, record]
+        return new_es
+      elsif !@buffer[stream_identity].empty?
+        # last partial record
+        @buffer[stream_identity] << [tag, time, record]
+        _, new_record = flush_buffer(stream_identity)
+        new_es.add(time, new_record)
+        return new_es
+      end
+      # regular record
+      new_es.add(time, record)
+      new_es
+    end
+
+    def partial?(text)
+      /^true$/.match(text)
+    end
+
+    def sanitize_record(record)
+      record.delete(@match_key)
+    end
+
+    def flush_buffer(stream_identity)
+      lines = @buffer[stream_identity].map {|_tag, _time, record| record[@key] }
+      _tag, time, first_record = @buffer[stream_identity].first
+      new_record = {
+        @key => lines.join(@separator)
+      }
+      @buffer[stream_identity].clear
+      merged_record = first_record.merge(new_record)
+      sanitize_record(merged_record)
+      [time, merged_record]
+    end
+
+    def flush_timeout_buffer
+      # now = Fluent::Engine.now
+      now = Fluent::EventTime.now
+      timeout_stream_identities = []
+      @timeout_map.each do |stream_identity, previous_timestamp|
+        next if @flush_interval > (now - previous_timestamp)
+        next if @buffer[stream_identity].empty?
+        time, record = flush_buffer(stream_identity)
+        sanitize_record(record)
+        timeout_stream_identities << stream_identity
+        tag = stream_identity.split(":").first
+        message = "Timeout flush: #{stream_identity}"
+        handle_timeout_error(tag, time, record, message)
+        log.info(message)
+      end
+      @timeout_map.reject! do |stream_identity, _|
+        timeout_stream_identities.include?(stream_identity)
+      end
+    end
+
+    def flush_shutdown_buffer
+      @buffer.each do |stream_identity, elements|
+        next if elements.empty?
+        tag = stream_identity.split(":").first
+        time, record = flush_buffer(stream_identity)
+        router.emit(tag, time, record)
+        log.info("Shutdown flush: #{stream_identity}")
+      end
+      @buffer.clear
+      log.info("Filter docker_journal_concat shutdown finished")
+    end
+
+    def handle_timeout_error(tag, time, record, message)
+      if @timeout_label
+        event_router = event_emitter_router(@timeout_label)
+        event_router.emit(tag, time, record)
+      else
+        router.emit_error_event(tag, time, record, TimeoutError.new(message))
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,125 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-docker-journald-concat
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Oleksandr Kushchenko
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-10-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: Fluentd Filter plugin to concatenate partial log messages generated by
+  Docker daemon with Journald logging driver
+email:
+- gearok@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-docker-journald-concat.gemspec
+- lib/fluent/plugin/filter_docker_journal_concat.rb
+homepage: https://github.com/okushchenko/fluent-plugin-docker-journald-concat
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Fluentd Filter plugin to concatenate partial log messages generated by Docker
+  daemon with Journald logging driver
+test_files: []