fluent-plugin-tailpath 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ebda3e465349195927fef2fab4ca4b0926b960ae
+  data.tar.gz: 7bce2f30a26b4c2d5085f38ccdfba842f134d006
+SHA512:
+  metadata.gz: c9c77945298c37e6f2da4ee3de05c9f39da1b14c048a2d651e41d3bf011bd4098a691e714b8345a081bab52f889006685c9ba1e31bc045ac31e3619d02ed4cb1
+  data.tar.gz: ee91142c0cb273a69a5d84da297df1b73b31992cc122ee58cf4c8f2575a160ea07ac3b0069560c1a55e555c67a6545464bdac2fcce0ae0031b537581323c965a

data/.gitignore

@@ -0,0 +1,14 @@
+/*.gem
+~*
+#*
+*~
+.bundle
+Gemfile.lock
+.rbenv-version
+vendor
+doc/*
+tmp/*
+coverage
+.yardoc
+pkg/
+.ruby-version

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,38 @@
+Copyright (c) 2014 Jacob Wirth
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Apache 2.0 License
+
+Copyright (C) 2011 FURUHASHI Sadayuki
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,27 @@
+# fluent-plugin-tailpath
+
+Fluentd plugin to tail files and add the file path to the message
+
+## Installation
+
+Use RubyGems:
+
+    gem install fluent-plugin-tailpath
+
+## Configuration
+
+For more info, the configuration is identical to [im_file](http://docs.fluentd.org/articles/in_tail)
+
+Example:
+
+     <source>
+       type tailpath
+       path /var/log/httpd-access.log
+       pos_file /var/log/td-agent/httpd-access.log.pos
+       tag apache.access
+       format apache2
+     </source>
+
+## Copyright
+
+Copyright (c) 2014 Jacob Wirth. See [LICENSE](LICENSE) for details.

data/fluent-plugin-tailpath.gemspec

@@ -0,0 +1,25 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-tailpath"
+  gem.version     = "0.0.1"
+  gem.authors     = ["Jacob Wirth"]
+  gem.email       = "xthexder@live.com"
+  gem.homepage    = "https://github.com/xthexder/fluent-plugin-tailpath"
+  gem.description = "Fluentd plugin to tail files and add the file path to the message"
+  gem.summary     = gem.description
+  gem.licenses    = ["MIT"]
+  gem.has_rdoc    = false
+
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.17"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "pry"
+  gem.add_development_dependency "pry-nav"
+end

data/lib/fluent/plugin/in_tailpath.rb

@@ -0,0 +1,497 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+  class TailPathInput < Input
+    Plugin.register_input('tailpath', self)
+
+    def initialize
+      super
+      @paths = []
+    end
+
+    config_param :path, :string
+    config_param :tag, :string
+    config_param :rotate_wait, :time, :default => 5
+    config_param :pos_file, :string, :default => nil
+
+    attr_reader :paths
+
+    def configure(conf)
+      super
+
+      @paths = @path.split(',').map {|path| path.strip }
+      if @paths.empty?
+        raise ConfigError, "tailpath: 'path' parameter is required on tail input"
+      end
+
+      unless @pos_file
+        $log.warn "'pos_file PATH' parameter is not set to a 'tail' source."
+        $log.warn "this parameter is highly recommended to save the position to resume tailing."
+      end
+
+      configure_parser(conf)
+    end
+
+    def configure_parser(conf)
+      @parser = TextParser.new
+      @parser.configure(conf)
+    end
+
+    def start
+      if @pos_file
+        @pf_file = File.open(@pos_file, File::RDWR|File::CREAT, DEFAULT_FILE_PERMISSION)
+        @pf_file.sync = true
+        @pf = PositionFile.parse(@pf_file)
+      end
+
+      @loop = Coolio::Loop.new
+      @tails = @paths.map {|path|
+        pe = @pf ? @pf[path] : MemoryPositionEntry.new
+        TailWatcher.new(path, @rotate_wait, pe, &method(:receive_lines))
+      }
+      @tails.each {|tail|
+        tail.attach(@loop)
+      }
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      @tails.each {|tail|
+        tail.close
+      }
+      @loop.stop
+      @thread.join
+      @pf_file.close if @pf_file
+    end
+
+    def run
+      @loop.run
+    rescue
+      $log.error "unexpected error", :error=>$!.to_s
+      $log.error_backtrace
+    end
+
+    def receive_lines(path, lines)
+      es = MultiEventStream.new
+      lines.each {|line|
+        begin
+          line.chomp!  # remove \n
+          time, record = parse_line(line)
+          if time && record
+            record['path'] = path
+            es.add(time, record)
+          end
+        rescue
+          $log.warn line.dump, :error=>$!.to_s
+          $log.debug_backtrace
+        end
+      }
+
+      unless es.empty?
+        begin
+          Engine.emit_stream(@tag, es)
+        rescue
+          # ignore errors. Engine shows logs and backtraces.
+        end
+      end
+    end
+
+    def parse_line(line)
+      return @parser.parse(line)
+    end
+
+    class TailWatcher
+      def initialize(path, rotate_wait, pe, &receive_lines)
+        @path = path
+        @rotate_wait = rotate_wait
+        @pe = pe || MemoryPositionEntry.new
+        @receive_lines = receive_lines
+
+        @rotate_queue = []
+
+        @timer_trigger = TimerWatcher.new(1, true, &method(:on_notify))
+        @stat_trigger = StatWatcher.new(path, &method(:on_notify))
+
+        @rotate_handler = RotateHandler.new(path, &method(:on_rotate))
+        @io_handler = nil
+      end
+
+      def attach(loop)
+        @timer_trigger.attach(loop)
+        @stat_trigger.attach(loop)
+        on_notify
+      end
+
+      def detach
+        @timer_trigger.detach if @timer_trigger.attached?
+        @stat_trigger.detach if @stat_trigger.attached?
+      end
+
+      def close
+        @rotate_queue.reject! {|req|
+          req.io.close
+          true
+        }
+        detach
+      end
+
+      def on_notify
+        @rotate_handler.on_notify
+        return unless @io_handler
+        @io_handler.on_notify
+
+        # proceeds rotate queue
+        return if @rotate_queue.empty?
+        @rotate_queue.first.tick
+
+        while @rotate_queue.first.ready?
+          if io = @rotate_queue.first.io
+            stat = io.stat
+            inode = stat.ino
+            if inode == @pe.read_inode
+              # rotated file has the same inode number with the last file.
+              # assuming following situation:
+              #   a) file was once renamed and backed, or
+              #   b) symlink or hardlink to the same file is recreated
+              # in either case, seek to the saved position
+              pos = @pe.read_pos
+            else
+              pos = io.pos
+            end
+            @pe.update(inode, pos)
+            io_handler = IOHandler.new(io, @pe, &@receive_lines)
+          else
+            io_handler = NullIOHandler.new
+          end
+          @io_handler.close
+          @io_handler = io_handler
+          @rotate_queue.shift
+          break if @rotate_queue.empty?
+        end
+      end
+
+      def on_rotate(io)
+        if @io_handler == nil
+          if io
+            # first time
+            stat = io.stat
+            fsize = stat.size
+            inode = stat.ino
+
+            last_inode = @pe.read_inode
+            if inode == last_inode
+              # seek to the saved position
+              pos = @pe.read_pos
+            elsif last_inode != 0
+              # this is FilePositionEntry and fluentd once started.
+              # read data from the head of the rotated file.
+              # logs never duplicate because this file is a rotated new file.
+              pos = 0
+              @pe.update(inode, pos)
+            else
+              # this is MemoryPositionEntry or this is the first time fluentd started.
+              # seek to the end of the any files.
+              # logs may duplicate without this seek because it's not sure the file is
+              # existent file or rotated new file.
+              pos = fsize
+              @pe.update(inode, pos)
+            end
+            io.seek(pos)
+
+            @io_handler = IOHandler.new(io, @pe, &@receive_lines)
+          else
+            @io_handler = NullIOHandler.new
+          end
+
+        else
+          if io && @rotate_queue.find {|req| req.io == io }
+            return
+          end
+          last_io = @rotate_queue.empty? ? @io_handler.io : @rotate_queue.last.io
+          if last_io == nil
+            $log.info "detected rotation of #{@path}"
+            # rotate imeediately if previous file is nil
+            wait = 0
+          else
+            $log.info "detected rotation of #{@path}; waiting #{@rotate_wait} seconds"
+            wait = @rotate_wait
+            wait -= @rotate_queue.first.wait unless @rotate_queue.empty?
+          end
+          @rotate_queue << RotationRequest.new(io, wait)
+        end
+      end
+
+      class TimerWatcher < Coolio::TimerWatcher
+        def initialize(interval, repeat, &callback)
+          @callback = callback
+          super(interval, repeat)
+        end
+
+        def on_timer
+          @callback.call
+        rescue
+          # TODO log?
+          $log.error $!.to_s
+          $log.error_backtrace
+        end
+      end
+
+      class StatWatcher < Coolio::StatWatcher
+        def initialize(path, &callback)
+          @callback = callback
+          super(path)
+        end
+
+        def on_change(prev, cur)
+          @callback.call
+        rescue
+          # TODO log?
+          $log.error $!.to_s
+          $log.error_backtrace
+        end
+      end
+
+      class RotationRequest
+        def initialize(io, wait)
+          @io = io
+          @wait = wait
+        end
+
+        attr_reader :io, :wait
+
+        def tick
+          @wait -= 1
+        end
+
+        def ready?
+          @wait <= 0
+        end
+      end
+
+      MAX_LINES_AT_ONCE = 1000
+
+      class IOHandler
+        def initialize(io, pe, &receive_lines)
+          $log.info "following tail of #{io.path}"
+          @io = io
+          @pe = pe
+          @receive_lines = receive_lines
+          @buffer = ''.force_encoding('ASCII-8BIT')
+          @iobuf = ''.force_encoding('ASCII-8BIT')
+        end
+
+        attr_reader :io
+
+        def on_notify
+          begin
+            lines = []
+            read_more = false
+
+            begin
+              while true
+                if @buffer.empty?
+                  @io.read_nonblock(2048, @buffer)
+                else
+                  @buffer << @io.read_nonblock(2048, @iobuf)
+                end
+                while line = @buffer.slice!(/.*?\n/m)
+                  lines << line
+                end
+                if lines.size >= MAX_LINES_AT_ONCE
+                  # not to use too much memory in case the file is very large
+                  read_more = true
+                  break
+                end
+              end
+            rescue EOFError
+            end
+
+            unless lines.empty?
+              @receive_lines.call(@io.path, lines)
+              @pe.update_pos(@io.pos - @buffer.bytesize)
+            end
+
+          end while read_more
+
+        rescue
+          $log.error $!.to_s
+          $log.error_backtrace
+          close
+        end
+
+        def close
+          @io.close unless @io.closed?
+        end
+      end
+
+      class NullIOHandler
+        def initialize
+        end
+
+        def io
+        end
+
+        def on_notify
+        end
+
+        def close
+        end
+      end
+
+      class RotateHandler
+        def initialize(path, &on_rotate)
+          @path = path
+          @inode = nil
+          @fsize = -1  # first
+          @on_rotate = on_rotate
+        end
+
+        def on_notify
+          begin
+            io = File.open(@path)
+            stat = io.stat
+            inode = stat.ino
+            fsize = stat.size
+          rescue Errno::ENOENT
+            # moved or deleted
+            inode = nil
+            fsize = 0
+          end
+
+          begin
+            if @inode != inode || fsize < @fsize
+              # rotated or truncated
+              @on_rotate.call(io)
+              io = nil
+            end
+
+            @inode = inode
+            @fsize = fsize
+          ensure
+            io.close if io
+          end
+
+        rescue
+          $log.error $!.to_s
+          $log.error_backtrace
+        end
+      end
+    end
+
+
+    class PositionFile
+      def initialize(file, map, last_pos)
+        @file = file
+        @map = map
+        @last_pos = last_pos
+      end
+
+      def [](path)
+        if m = @map[path]
+          return m
+        end
+
+        @file.pos = @last_pos
+        @file.write path
+        @file.write "\t"
+        seek = @file.pos
+        @file.write "0000000000000000\t00000000\n"
+        @last_pos = @file.pos
+
+        @map[path] = FilePositionEntry.new(@file, seek)
+      end
+
+      def self.parse(file)
+        map = {}
+        file.pos = 0
+        file.each_line {|line|
+          m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
+          next unless m
+          path = m[1]
+          pos = m[2].to_i(16)
+          ino = m[3].to_i(16)
+          seek = file.pos - line.bytesize + path.bytesize + 1
+          map[path] = FilePositionEntry.new(file, seek)
+        }
+        new(file, map, file.pos)
+      end
+    end
+
+    # pos               inode
+    # ffffffffffffffff\tffffffff\n
+    class FilePositionEntry
+      POS_SIZE = 16
+      INO_OFFSET = 17
+      INO_SIZE = 8
+      LN_OFFSET = 25
+      SIZE = 26
+
+      def initialize(file, seek)
+        @file = file
+        @seek = seek
+      end
+
+      def update(ino, pos)
+        @file.pos = @seek
+        @file.write "%016x\t%08x" % [pos, ino]
+        @inode = ino
+      end
+
+      def update_pos(pos)
+        @file.pos = @seek
+        @file.write "%016x" % pos
+      end
+
+      def read_inode
+        @file.pos = @seek + INO_OFFSET
+        raw = @file.read(8)
+        raw ? raw.to_i(16) : 0
+      end
+
+      def read_pos
+        @file.pos = @seek
+        raw = @file.read(16)
+        raw ? raw.to_i(16) : 0
+      end
+    end
+
+    class MemoryPositionEntry
+      def initialize
+        @pos = 0
+        @inode = 0
+      end
+
+      def update(ino, pos)
+        @inode = ino
+        @pos = pos
+      end
+
+      def update_pos(pos)
+        @pos = pos
+      end
+
+      def read_pos
+        @pos
+      end
+
+      def read_inode
+        @inode
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-tailpath
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jacob Wirth
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.17
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.17
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin to tail files and add the file path to the message
+email: xthexder@live.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- fluent-plugin-tailpath.gemspec
+- lib/fluent/plugin/in_tailpath.rb
+homepage: https://github.com/xthexder/fluent-plugin-tailpath
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin to tail files and add the file path to the message
+test_files: []