logstash-input-file_progress 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8ce74ef18daca1f4dee56f79adf5c680423dac47
+  data.tar.gz: 639c3ae4c64b2d7224668ffbf2a6bb219f9499ed
+SHA512:
+  metadata.gz: 1f2a51b2364f190771284c31baaa9723f613957ce70fd5918707533ed3b6ce815c930fa3fc41a30d87a5b00612ccb14c074f2f1014718f6dc66f18bb5c076613
+  data.tar.gz: 8e02ebb1782de1c4c5151f7d5f0c4471e4e03394e0cc7566044f4414ae81332030205045e613ad428ec478953bbd9d4713c199455608e08b6c7853ec6418a6b7

data/.gitignore

@@ -0,0 +1,3 @@
+Gemfile.lock
+.ruby-version
+*.gem

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -0,0 +1,7 @@
+@files=[]
+
+task :default do
+  system("rake -T")
+end
+
+require "logstash/devutils/rake"

data/lib/logstash/inputs/file_progress.rb

@@ -0,0 +1,149 @@
+require "logstash/inputs/base"
+require "logstash/namespace"
+
+require "pathname"
+require "socket" # for Socket.gethostname
+
+# Stream events from files.
+#
+# By default, each event is assumed to be one line. If you
+# want to join lines, you'll want to use the multiline filter.
+#
+# Files are followed in a manner similar to "tail -0F". File rotation
+# is detected and handled by this input.
+#
+# In addition to 'normal' file input we add events with sincedb-data to the pipe.
+# This can be later used to serve progress date with e.g. faye-output
+class LogStash::Inputs::FileProgress < LogStash::Inputs::Base
+  config_name "file_progress"
+
+  # TODO(sissel): This should switch to use the 'line' codec by default
+  # once file following
+  default :codec, "line"
+
+  # The path to the file to use as an input.
+  # You can use globs here, such as `/var/log/*.log`
+  # Paths must be absolute and cannot be relative.
+  config :path, :validate => :array, :required => true
+
+  # Exclusions (matched against the filename, not full path). Globs
+  # are valid here, too. For example, if you have
+  #
+  #     path => "/var/log/*"
+  #
+  # you might want to exclude gzipped files:
+  #
+  #     exclude => "*.gz"
+  config :exclude, :validate => :array
+
+  # How often we stat files to see if they have been modified. Increasing
+  # this interval will decrease the number of system calls we make, but
+  # increase the time to detect new log lines.
+  config :stat_interval, :validate => :number, :default => 1
+
+  # How often we expand globs to discover new files to watch.
+  config :discover_interval, :validate => :number, :default => 15
+
+  # Where to write the since database (keeps track of the current
+  # position of monitored log files). The default will write
+  # sincedb files to some path matching "$HOME/.sincedb*"
+  config :sincedb_path, :validate => :string, :required => true
+
+  # How often to write a since database with the current position of
+  # monitored log files.
+  config :sincedb_write_interval, :validate => :number, :default => 15
+
+  # Choose where logstash starts initially reading files - at the beginning or
+  # at the end. The default behavior treats files like live streams and thus
+  # starts at the end. If you have old data you want to import, set this
+  # to 'beginning'
+  #
+  # This option only modifieds "first contact" situations where a file is new
+  # and not seen before. If a file has already been seen before, this option
+  # has no effect.
+  config :start_position, :validate => [ "beginning", "end"], :default => "beginning"
+
+  # Should the progressdb events be send to the pipeline
+  config :progressdb, :validate => :boolean, :default => false
+
+  # Should the processdb entry be deleted after file-deletion
+  config :progressdb_del, :validate => :boolean, :default => false
+
+  # Close the file when end is reached
+  # This make sense when reading a file once from the beginning and want to e.g.
+  # proceed renaming or deleting the parent folder
+  config :eof_close, :validate => :boolean, :default => false
+
+  public
+  def register
+    require "addressable/uri"
+    require "filewatch/ext/filetail"
+    require "digest/md5"
+    @logger.info("Registering file input", :path => @path)
+
+    @tail_config = {
+      :exclude => @exclude,
+      :stat_interval => @stat_interval,
+      :discover_interval => @discover_interval,
+      :sincedb_write_interval => @sincedb_write_interval,
+      :logger => @logger,
+      :progressdb => @progressdb,
+      :progressdb_del => @progressdb_del,
+      :eof_close => @eof_close,
+    }
+
+    @path.each do |path|
+      if Pathname.new(path).relative?
+        raise ArgumentError.new("File paths must be absolute, relative path specified: #{path}")
+      end
+    end
+
+    @tail_config[:sincedb_path] = @sincedb_path
+
+    if @start_position == "beginning"
+      @tail_config[:start_new_files_at] = :beginning
+    end
+
+    @codec_plain = LogStash::Codecs::Plain.new
+  end # def register
+
+  public
+  def run(queue)
+    @tail = FileWatch::Ext::FileTail.new(@tail_config)
+    @tail.logger = @logger
+    @path.each { |path| @tail.tail(path) }
+    hostname = Socket.gethostname
+
+    @tail.subscribe do |path, data, type|
+      @logger.debug("Received line", :path => path, :data => data) if logger.debug?
+
+      if type == :log
+        @codec.decode(data) do |event|
+
+          decorate(event)
+          event["host"] = hostname
+          event["path"] = path
+
+          queue << event
+        end
+      elsif type == :progressdb
+        @codec_plain.decode(data) do |event|
+
+          decorate(event)
+          event["host"] = hostname
+          event["path"] = path
+          event["type"] = "progressdb";
+
+          queue << event
+        end        
+      end # if
+    end # subscribe
+    finished
+  end # def run
+
+  public
+  def teardown
+    @tail.sincedb_write
+    @tail.quit
+  end # def teardown
+end # class LogStash::Inputs::TriggeredPackage

data/logstash-input-file_progress.gemspec

@@ -0,0 +1,32 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-input-file_progress'
+  s.version         = '0.1.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Stream events from files with progress-events."
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Signify"]
+  s.email           = 'dietmar@signifydata.com'
+  s.homepage        = "http://www.signifydata.com"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core', '>= 1.4.0', '< 2.0.0'
+
+  s.add_runtime_dependency 'logstash-codec-line'
+  s.add_runtime_dependency 'addressable'
+  s.add_runtime_dependency "filewatch-ext", ["~> 0.3.0"]
+
+
+  s.add_development_dependency 'logstash-devutils'
+end
+

data/spec/inputs/file_progress_spec.rb

@@ -0,0 +1,141 @@
+# encoding: utf-8
+
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/file_progress"
+require "tempfile"
+
+describe "inputs/file_progress" do
+
+
+  describe "starts at the end of an existing file" do
+    tmp_file = Tempfile.new('logstash-spec-input-file')
+
+    config = %q[
+	    input {
+	      file_progress {
+	        type => "blah"
+	        path => "#{tmp_file.path}"
+	        sincedb_path => "/dev/null"
+	      }
+	    }
+	]
+
+    #input do |pipeline, queue|
+      File.open(tmp_file, "w") do |fd|
+        fd.puts("ignore me 1")
+        fd.puts("ignore me 2")
+      end
+
+      pipeline = LogStash::Pipeline.new(config)
+      queue = Queue.new
+      pipeline.instance_eval do
+        @output_func = lambda { |event| queue << event }
+      end
+      pipeline_thread = Thread.new { pipeline.run }
+      event = queue.pop
+
+      # at this point even if pipeline.ready? == true the plugins
+      # threads might still be initializing so we cannot know when the
+      # file plugin will have seen the original file, it could see it
+      # after the first(s) hello world appends below, hence the
+      # retry logic.
+
+      retries = 0
+      loop do
+        insist { retries } < 20 # 2 secs should be plenty?
+
+        File.open(tmp_file, "a") do |fd|
+          fd.puts("hello")
+          fd.puts("world")
+        end
+
+        if queue.size >= 2
+          events = 2.times.collect { queue.pop }
+          insist { events[0]["message"] } == "hello"
+          insist { events[1]["message"] } == "world"
+          break
+        end
+
+        sleep(0.1)
+        retries += 1
+      end
+
+      pipeline_thread.join
+
+    #end
+  end
+
+  #describe "can start at the beginning of an existing file" do
+  #    tmp_file = Tempfile.new('logstash-spec-input-file')#
+
+  #    config <<-CONFIG
+  #    input {
+  #      file_progress {
+  #        type => "blah"
+  #        path => "#{tmp_file.path}"
+  #        start_position => "beginning"
+  #        sincedb_path => "/dev/null"
+  #      }
+  #    }
+  #    CONFIG#
+
+  #    input do |pipeline, queue|
+  #      File.open(tmp_file, "a") do |fd|
+  #        fd.puts("hello")
+  #        fd.puts("world")
+  #      end#
+
+  #      Thread.new { pipeline.run }
+  #      sleep 0.1 while !pipeline.ready?#
+
+  #      events = 2.times.collect { queue.pop }
+  #      insist { events[0]["message"] } == "hello"
+  #      insist { events[1]["message"] } == "world"
+  #    end
+  #  end#
+
+  #  describe "restarts at the sincedb value" do
+  #    tmp_file = Tempfile.new('logstash-spec-input-file')
+  #    tmp_sincedb = Tempfile.new('logstash-spec-input-file-sincedb')#
+
+  #    config <<-CONFIG
+  #    input {
+  #      file_progress {
+  #        type => "blah"
+  #        path => "#{tmp_file.path}"
+  #        start_position => "beginning"
+  #        sincedb_path => "#{tmp_sincedb.path}"
+  #      }
+  #    }
+  #    CONFIG#
+
+  #    input do |pipeline, queue|
+  #      File.open(tmp_file, "w") do |fd|
+  #        fd.puts("hello")
+  #        fd.puts("world")
+  #      end#
+
+  #      t = Thread.new { pipeline.run }
+  #      sleep 0.1 while !pipeline.ready?#
+
+  #      events = 2.times.collect { queue.pop }
+  #      pipeline.shutdown
+  #      t.join#
+
+  #      File.open(tmp_file, "a") do |fd|
+  #        fd.puts("foo")
+  #        fd.puts("bar")
+  #        fd.puts("baz")
+  #      end#
+
+  #      Thread.new { pipeline.run }
+  #      sleep 0.1 while !pipeline.ready?#
+
+  #      events = 3.times.collect { queue.pop }#
+
+  #      insist { events[0]["message"] } == "foo"
+  #      insist { events[1]["message"] } == "bar"
+  #      insist { events[2]["message"] } == "baz"
+  #    end
+  #  end
+end

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-file_progress
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Signify
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-08-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-line
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: addressable
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  name: filewatch-ext
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: dietmar@signifydata.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- Rakefile
+- lib/logstash/inputs/file_progress.rb
+- logstash-input-file_progress.gemspec
+- spec/inputs/file_progress_spec.rb
+homepage: http://www.signifydata.com
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Stream events from files with progress-events.
+test_files:
+- spec/inputs/file_progress_spec.rb