fluent-plugin-tail_path 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2b4639f29d0249ebd21ba0e5691498bf4b78e301
+  data.tar.gz: 2e6c03cb4576e8454348da3785b76edd5425f908
+SHA512:
+  metadata.gz: 4e6b5b7ac271210134c713f97ca1eca279fccf46e278db730e8d34d841aa7f5b6ac40f18c480acebff9d53231c28ede20c87333cc1ee223ce8ef8b0aa431869d
+  data.tar.gz: 1d02771648360cddea2c64a2d2d0d36eb9b409ecd687ce9da2eebd39c8bbed28ee06ec08ee133121c375e508c1a9db1f973dcaca8898598da11b415905325e75

data/.gitignore

@@ -0,0 +1,14 @@
+/*.gem
+~*
+#*
+*~
+.bundle
+Gemfile.lock
+.rbenv-version
+vendor
+doc/*
+tmp/*
+.yardoc
+.ruby-version
+pkg/*
+test/tmp/*

data/.travis.yml

@@ -0,0 +1,6 @@
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.*
+gemfile:
+  - Gemfile

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.0.1
+
+First version

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1 @@
+Same with Fluentd

data/README.md

@@ -0,0 +1,33 @@
+# fluent-plugin-tail_path
+
+## About
+
+This is an extension of fluentd in\_tail plugin to add `path` field which tells the log path being tailed
+
+## Parameters
+
+Basically same with in\_tail plugin. See http://docs.fluentd.org/articles/in_tail
+
+Following parameters are additionally available: 
+
+- path_key
+
+    Add `path` field which tells the log path being tailed. Specify the field name. 
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new [Pull Request](../../pull/new/master)
+
+## ChangeLog
+
+See [CHANGELOG.md](CHANGELOG.md) for details.
+
+## Copyright
+
+* Copyright (c) 2014- @szhem
+* Copyright (c) 2014- Naotoshi Seo
+* See [LICENSE](LICENSE) for details.

data/Rakefile

@@ -0,0 +1,16 @@
+# encoding: utf-8
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*.rb'
+  test.verbose = true
+end
+task :default => :test
+
+desc 'Open an irb session preloaded with the gem library'
+task :console do
+    sh 'irb -rubygems -I lib'
+end
+task :c => :console

data/fluent-plugin-tail_path.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "fluent-plugin-tail_path"
+  s.version     = "0.0.1"
+  s.authors     = ["szhem", "Naotoshi Seo"]
+  s.email       = ["sonots@gmail.com"]
+  s.homepage    = "https://github.com/sonots/fluent-plugin-tail_path"
+  s.summary     = "Fluentd in_tail extension to add `path` field"
+  s.description = s.summary
+
+  s.rubyforge_project = "fluent-plugin-tail_path"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency "fluentd", ">= 0.10.45"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "pry"
+  s.add_development_dependency "pry-nav"
+end

data/lib/fluent/plugin/in_tail_path.rb

@@ -0,0 +1,59 @@
+require 'fluent/plugin/in_tail'
+
+class Fluent::NewTailPathInput < Fluent::NewTailInput
+  Fluent::Plugin.register_output('in_taiL_path', self)
+
+  config_param :path_key, :string, :default => nil
+
+  def configure(conf)
+    super
+  end
+
+  # Override to add path field
+  def convert_line_to_event(line, es, tail_watcher)
+    begin
+      line.chomp!  # remove \n
+      time, record = parse_line(line)
+      if time && record
+        record[@path_key] ||= tail_watcher.path unless @path_key.nil?
+        es.add(time, record)
+      else
+        log.warn "pattern not match: #{line.inspect}"
+      end
+    rescue => e
+      log.warn line.dump, :error => e.to_s
+      log.debug_backtrace(e)
+    end
+  end
+
+  # Override to pass tail_watcher to convert_line_to_event
+  def parse_singleline(lines, tail_watcher)
+    es = ::Fluent::MultiEventStream.new
+    lines.each { |line|
+      convert_line_to_event(line, es, tail_watcher)
+    }
+    es
+  end
+
+  # Override to pass tail_watcher to convert_line_to_event
+  def parse_multilines(lines, tail_watcher)
+    lb = tail_watcher.line_buffer
+    es = ::Fluent::MultiEventStream.new
+    lines.each { |line|
+      if @parser.parser.firstline?(line)
+        if lb
+          convert_line_to_event(lb, es, tail_watcher)
+        end
+        lb = line
+      else
+        if lb.nil?
+          log.warn "got incomplete line before first line from #{tail_watcher.path}: #{lb.inspect}"
+        else
+          lb << line
+        end
+      end
+    }
+    tail_watcher.line_buffer = lb
+    es
+  end
+end

data/test/helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+require 'fluent/plugin/in_tail_path'
+
+class Test::Unit::TestCase
+end

data/test/plugin/in_tail_path.rb

@@ -0,0 +1,52 @@
+require_relative '../helper'
+
+class TailPathInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
+  end
+
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
+
+  COMMON_CONFIG = %[
+    path #{TMP_DIR}/tail.txt
+    tag t1
+    rotate_wait 2s
+    pos_file #{TMP_DIR}/tail.pos
+  ]
+  SINGLE_LINE_CONFIG = %[
+    format /(?<message>.*)/
+  ]
+
+  def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true)
+    config = use_common_conf ? COMMON_CONFIG + conf : conf
+    Fluent::Test::InputTestDriver.new(Fluent::NewTailPathInput).configure(config)
+  end
+
+  def test_path_key
+    File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
+
+    d = create_driver(%[
+       path #{TMP_DIR}/tail.txt
+       tag t1
+       format /(?<message>.*)/
+       path_key foobar
+    ], false)
+
+    d.run do
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal(true, emits.length > 0)
+    assert_equal({"message"=>"test1", "foobar"=>"#{TMP_DIR}/tail.txt"}, emits[0][2])
+    assert_equal({"message"=>"test2", "foobar"=>"#{TMP_DIR}/tail.txt"}, emits[1][2])
+  end
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-tail_path
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- szhem
+- Naotoshi Seo
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.45
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.45
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd in_tail extension to add `path` field
+email:
+- sonots@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-tail_path.gemspec
+- lib/fluent/plugin/in_tail_path.rb
+- test/helper.rb
+- test/plugin/in_tail_path.rb
+homepage: https://github.com/sonots/fluent-plugin-tail_path
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: fluent-plugin-tail_path
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Fluentd in_tail extension to add `path` field
+test_files:
+- test/helper.rb
+- test/plugin/in_tail_path.rb