logstash-input-file 4.0.0 → 4.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 00f5e922badbdd1b31300eb066d5d9b3917ed967
-  data.tar.gz: 06bd5932092a2b1989f7478c469d587d4d240928
+SHA256:
+  metadata.gz: 4bfa2200c86a94ed11e3691e5769dabafed7b0794f6fbba92cd1b8d68af828a9
+  data.tar.gz: f5c9d5dc49814dd1040cc6af8e7fca6c9c1ef5d149cea96a0152a05fcb4e11de
 SHA512:
-  metadata.gz: e7b898485e484698a4774dadd163402ba0492d8bcda39559fa5b4be14ff8d9ededc7fbe2f7831de16092770b9ce688131d0e22be4c0568c8ed354ce62e82e676
-  data.tar.gz: bd425a072f2c1b262d522db2a36f53ec2ac88e47f6f406f69bc2c3dec8370bc8f7d70562065a87b108e1ebb09aae2d4f26765a09aa46e168c4a2feeede64c516
+  metadata.gz: 44d10ae77862a89089f48088420c67e5430623b46b2bc9d3d65f9ef5c5d90abdfd759112a3c763d4db6425f0c61ad8b6031f5e466de690b1b378f7080b1f3141
+  data.tar.gz: c85ca0e8e74cefa9a89dc33cc6d3163b1ab866bd59c9aca5fb1b44cbdc523e880a7c01a18d9de70fc02307a88d07b8bfad7fd19c74dd028d41bcdb854fdd6b21

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.0.1
+  - Docs: Fix the description with the logstash documentation generator
+  - Fix an issue with the rspec suite not finding log4j
+
 ## 4.0.0
   - Breaking: `ignore_older` settings is disabled by default. Previously if the file was older than 
     24 hours (the default for ignore_older), it would be ignored. This confused new users a lot, specially 

data/Gemfile

@@ -1,4 +1,11 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in logstash-mass_effect.gemspec
 gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/docs/index.asciidoc

@@ -0,0 +1,256 @@
+:plugin: file
+:type: input
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== File input plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+Stream events from files, normally by tailing them in a manner
+similar to `tail -0F` but optionally reading them from the
+beginning.
+
+By default, each event is assumed to be one line and a line is
+taken to be the text before a newline character.
+Normally, logging will add a newline to the end of each line written.
+If you would like to join multiple log lines into one event,
+you'll want to use the multiline codec or filter.
+
+The plugin aims to track changing files and emit new content as it's
+appended to each file. It's not well-suited for reading a file from
+beginning to end and storing all of it in a single event (not even
+with the multiline codec or filter).
+
+==== Reading from remote network volumes
+
+The file input is not tested on remote filesystems such as NFS, Samba, s3fs-fuse, etc. These 
+remote filesystems typically have behaviors that are very different from local filesystems and 
+are therefore unlikely to work correctly when used with the file input.
+
+==== Tracking of current position in watched files
+
+The plugin keeps track of the current position in each file by
+recording it in a separate file named sincedb. This makes it
+possible to stop and restart Logstash and have it pick up where it
+left off without missing the lines that were added to the file while
+Logstash was stopped.
+
+By default, the sincedb file is placed in the home directory of the
+user running Logstash with a filename based on the filename patterns
+being watched (i.e. the `path` option). Thus, changing the filename
+patterns will result in a new sincedb file being used and any
+existing current position state will be lost. If you change your
+patterns with any frequency it might make sense to explicitly choose
+a sincedb path with the `sincedb_path` option.
+
+A different `sincedb_path` must be used for each input. Using the same
+path will cause issues. The read checkpoints for each input must be
+stored in a different path so the information does not override.
+
+Sincedb files are text files with four columns:
+
+. The inode number (or equivalent).
+. The major device number of the file system (or equivalent).
+. The minor device number of the file system (or equivalent).
+. The current byte offset within the file.
+
+On non-Windows systems you can obtain the inode number of a file
+with e.g. `ls -li`.
+
+==== File rotation
+
+File rotation is detected and handled by this input, regardless of
+whether the file is rotated via a rename or a copy operation. To
+support programs that write to the rotated file for some time after
+the rotation has taken place, include both the original filename and
+the rotated filename (e.g. /var/log/syslog and /var/log/syslog.1) in
+the filename patterns to watch (the `path` option). Note that the
+rotated filename will be treated as a new file so if
+`start_position` is set to 'beginning' the rotated file will be
+reprocessed.
+
+With the default value of `start_position` ('end') any messages
+written to the end of the file between the last read operation prior
+to the rotation and its reopening under the new name (an interval
+determined by the `stat_interval` and `discover_interval` options)
+will not get picked up.
+
+[id="plugins-{type}s-{plugin}-options"]
+==== File Input Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-close_older>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-delimiter>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-discover_interval>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-exclude>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-ignore_older>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-max_open_files>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-path>> |<<array,array>>|Yes
+| <<plugins-{type}s-{plugin}-sincedb_path>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-sincedb_write_interval>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-start_position>> |<<string,string>>, one of `["beginning", "end"]`|No
+| <<plugins-{type}s-{plugin}-stat_interval>> |<<number,number>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+input plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-close_older"]
+===== `close_older` 
+
+  * Value type is <<number,number>>
+  * Default value is `3600`
+
+The file input closes any files that were last read the specified
+timespan in seconds ago.
+This has different implications depending on if a file is being tailed or
+read. If tailing, and there is a large time gap in incoming data the file
+can be closed (allowing other files to be opened) but will be queued for
+reopening when new data is detected. If reading, the file will be closed
+after closed_older seconds from when the last bytes were read.
+The default is 1 hour
+
+[id="plugins-{type}s-{plugin}-delimiter"]
+===== `delimiter` 
+
+  * Value type is <<string,string>>
+  * Default value is `"\n"`
+
+set the new line delimiter, defaults to "\n"
+
+[id="plugins-{type}s-{plugin}-discover_interval"]
+===== `discover_interval` 
+
+  * Value type is <<number,number>>
+  * Default value is `15`
+
+How often (in seconds) we expand the filename patterns in the
+`path` option to discover new files to watch.
+
+[id="plugins-{type}s-{plugin}-exclude"]
+===== `exclude` 
+
+  * Value type is <<array,array>>
+  * There is no default value for this setting.
+
+Exclusions (matched against the filename, not full path). Filename
+patterns are valid here, too. For example, if you have
+[source,ruby]
+    path => "/var/log/*"
+
+You might want to exclude gzipped files:
+[source,ruby]
+    exclude => "*.gz"
+
+[id="plugins-{type}s-{plugin}-ignore_older"]
+===== `ignore_older` 
+
+  * Value type is <<number,number>>
+  * There is no default value for this setting.
+
+When the file input discovers a file that was last modified
+before the specified timespan in seconds, the file is ignored.
+After it's discovery, if an ignored file is modified it is no
+longer ignored and any new data is read. By default, this option is
+disabled. Note this unit is in seconds.
+
+[id="plugins-{type}s-{plugin}-max_open_files"]
+===== `max_open_files` 
+
+  * Value type is <<number,number>>
+  * There is no default value for this setting.
+
+What is the maximum number of file_handles that this input consumes
+at any one time. Use close_older to close some files if you need to
+process more files than this number. This should not be set to the
+maximum the OS can do because file handles are needed for other
+LS plugins and OS processes.
+The default of 4095 is set in filewatch.
+
+[id="plugins-{type}s-{plugin}-path"]
+===== `path` 
+
+  * This is a required setting.
+  * Value type is <<array,array>>
+  * There is no default value for this setting.
+
+The path(s) to the file(s) to use as an input.
+You can use filename patterns here, such as `/var/log/*.log`.
+If you use a pattern like `/var/log/**/*.log`, a recursive search
+of `/var/log` will be done for all `*.log` files.
+Paths must be absolute and cannot be relative.
+
+You may also configure multiple paths. See an example
+on the <<array,Logstash configuration page>>.
+
+[id="plugins-{type}s-{plugin}-sincedb_path"]
+===== `sincedb_path` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Path of the sincedb database file (keeps track of the current
+position of monitored log files) that will be written to disk.
+The default will write sincedb files to `<path.data>/plugins/inputs/file`
+NOTE: it must be a file path and not a directory path
+
+[id="plugins-{type}s-{plugin}-sincedb_write_interval"]
+===== `sincedb_write_interval` 
+
+  * Value type is <<number,number>>
+  * Default value is `15`
+
+How often (in seconds) to write a since database with the current position of
+monitored log files.
+
+[id="plugins-{type}s-{plugin}-start_position"]
+===== `start_position` 
+
+  * Value can be any of: `beginning`, `end`
+  * Default value is `"end"`
+
+Choose where Logstash starts initially reading files: at the beginning or
+at the end. The default behavior treats files like live streams and thus
+starts at the end. If you have old data you want to import, set this
+to 'beginning'.
+
+This option only modifies "first contact" situations where a file
+is new and not seen before, i.e. files that don't have a current
+position recorded in a sincedb file read by Logstash. If a file
+has already been seen before, this option has no effect and the
+position recorded in the sincedb file will be used.
+
+[id="plugins-{type}s-{plugin}-stat_interval"]
+===== `stat_interval` 
+
+  * Value type is <<number,number>>
+  * Default value is `1`
+
+How often (in seconds) we stat files to see if they have been modified.
+Increasing this interval will decrease the number of system calls we make,
+but increase the time to detect new log lines.
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/inputs/file.rb

@@ -6,6 +6,7 @@ require "logstash/codecs/identity_map_codec"
 require "pathname"
 require "socket" # for Socket.gethostname
 require "fileutils"
+require_relative "file/patch"
 
 # Stream events from files, normally by tailing them in a manner
 # similar to `tail -0F` but optionally reading them from the
@@ -23,6 +24,12 @@ require "fileutils"
 # beginning to end and storing all of it in a single event (not even
 # with the multiline codec or filter).
 #
+# ==== Reading from remote network volumes
+#
+# The file input is not tested on remote filesystems such as NFS, Samba, s3fs-fuse, etc. These 
+# remote filesystems typically have behaviors that are very different from local filesystems and 
+# are therefore unlikely to work correctly when used with the file input.
+#
 # ==== Tracking of current position in watched files
 #
 # The plugin keeps track of the current position in each file by
@@ -71,21 +78,6 @@ require "fileutils"
 # determined by the `stat_interval` and `discover_interval` options)
 # will not get picked up.
 
-class LogStash::Codecs::Base
-  # TODO - move this to core
-  if !method_defined?(:accept)
-    def accept(listener)
-      decode(listener.data) do |event|
-        listener.process_event(event)
-      end
-    end
-  end
-  if !method_defined?(:auto_flush)
-    def auto_flush(*)
-    end
-  end
-end
-
 class LogStash::Inputs::File < LogStash::Inputs::Base
   config_name "file"
 
@@ -120,7 +112,7 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
 
   # Path of the sincedb database file (keeps track of the current
   # position of monitored log files) that will be written to disk.
-  # The default will write sincedb files to some path matching `$HOME/.sincedb*`
+  # The default will write sincedb files to `<path.data>/plugins/inputs/file`
   # NOTE: it must be a file path and not a directory path
   config :sincedb_path, :validate => :string
 

data/lib/logstash/inputs/file/patch.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+class LogStash::Codecs::Base
+  # TODO - move this to core
+  if !method_defined?(:accept)
+    def accept(listener)
+      decode(listener.data) do |event|
+        listener.process_event(event)
+      end
+    end
+  end
+  if !method_defined?(:auto_flush)
+    def auto_flush(*)
+    end
+  end
+end
+

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '4.0.0'
+  s.version         = '4.0.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Stream events from files."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

data/spec/inputs/file_spec.rb

@@ -1,7 +1,6 @@
 # encoding: utf-8
-
-require "logstash/inputs/file"
 require_relative "../spec_helper"
+require "logstash/inputs/file"
 require "tempfile"
 require "stud/temporary"
 require "logstash/codecs/multiline"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 4.0.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-17 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -148,7 +148,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -160,7 +162,9 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- docs/index.asciidoc
 - lib/logstash/inputs/file.rb
+- lib/logstash/inputs/file/patch.rb
 - logstash-input-file.gemspec
 - spec/inputs/file_spec.rb
 - spec/spec_helper.rb
@@ -186,7 +190,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.11
 signing_key:
 specification_version: 4
 summary: Stream events from files.