logstash-output-file 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 34e5dcb5b4862273a9cc92f1c4ab73235db7df4a
-  data.tar.gz: 802613ca119351bcb1970d3ec96efa52c18b6a38
+  metadata.gz: aa80999ed0b695c5970cccfc4ef9a84c6c688d60
+  data.tar.gz: c4f1584eff0dc5eae5bc4eb109b6102bba60f79a
 SHA512:
-  metadata.gz: ab71908733217054a01c5688cd3a31a6257350440440ba4885d7ac15a8404207b7fd2b19b067790541fd6db40d5e99e3b7cfd9173fc82f3b38c16daa4d4b4f76
-  data.tar.gz: 798c54d4b133f2aed3dba737254dc2807ad3d6f0253a2ae6e06e8b51bee5e7b691415532cab396868b60959a6c92c84ba1eb4424786fccf7a4b978914c338a36
+  metadata.gz: ef3b7ea580bb5f0278fda8cf7c0da9f2ddffc6b1dcec67469e4e1f70d57fbc0cf4eefe1177147833a8afd214d905719f41f7bf25bda3ef3b1d538282497035db
+  data.tar.gz: 07c59b12f42771a38d93b4021ed2d5992433e8ad6bda53bf951ad594a42845d93d502701678df20f977add39168cfdac06e0fbc59d13a3d1031f429b3dc695e4

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.2.0
+ - Add support for codec, using **json_lines** as default codec to keep default behavior.
+   Ref: https://github.com/logstash-plugins/logstash-output-file/pull/9
+
 ## 2.1.0
  - Add create_if_deleted option to create a destination file in case it
    was deleted by another agent in the machine. In case of being false

data/lib/logstash/outputs/file.rb

@@ -4,8 +4,18 @@ require "logstash/outputs/base"
 require "logstash/errors"
 require "zlib"
 
-# This output will write events to files on disk. You can use fields
+# This output writes events to files on disk. You can use fields
 # from the event as parts of the filename and/or path.
+#
+# By default, this output writes one event per line in **json** format.
+# You can customise the line format using the `line` codec like
+# [source,ruby]
+# output {
+#  file {
+#    path => ...
+#    codec => { line { format => "custom format: %{message}"}}
+#  }
+# }
 class LogStash::Outputs::File < LogStash::Outputs::Base
   FIELD_REF = /%\{[^}]+\}/
 
@@ -31,7 +41,7 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
   #
   # If this setting is omitted, the full json representation of the
   # event will be written as a single line.
-  config :message_format, :validate => :string
+  config :message_format, :validate => :string, :deprecated => "You can achieve the same behavior with the 'line' codec"
 
   # Flush interval (in seconds) for flushing writes to log files.
   # 0 will flush on every message.
@@ -47,7 +57,8 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
   # If the a file is deleted, but an event is comming with the need to be stored
   # in such a file, the plugin will created a gain this file. Default => true
   config :create_if_deleted, :validate => :boolean, :default => true
- 
+
+  default :codec, "json_lines"
 
   public
   def register
@@ -73,6 +84,13 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
     @last_stale_cleanup_cycle = now
     @flush_interval = @flush_interval.to_i
     @stale_cleanup_interval = 10
+
+    if @message_format
+     @codec = LogStash::Plugin.lookup("codec", "line").new
+     @codec.format = @message_format
+    end
+
+    @codec.on_event(&method(:write_event))
   end # def register
 
   private
@@ -96,18 +114,8 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
 
   public
   def receive(event)
-
-    file_output_path = generate_filepath(event)
-
-    if path_with_field_ref? && !inside_file_root?(file_output_path)
-      @logger.warn("File: the event tried to write outside the files root, writing the event to the failure file",  :event => event, :filename => @failure_path)
-      file_output_path = @failure_path
-    elsif !@create_if_deleted && deleted?(file_output_path)
-      file_output_path = @failure_path
-    end
-
-    output = format_message(event)
-    write_event(file_output_path, output)
+    @codec.encode(event)
+    close_stale_files
   end # def receive
 
   public
@@ -130,16 +138,19 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
   end
 
   private
-  def write_event(log_path, event)
-    @logger.debug("File, writing event to file.", :filename => log_path)
-    fd = open(log_path)
+  def write_event(event, data)
+    file_output_path = generate_filepath(event)
+    if path_with_field_ref? && !inside_file_root?(file_output_path)
+      @logger.warn("File: the event tried to write outside the files root, writing the event to the failure file",  :event => event, :filename => @failure_path)
+      file_output_path = @failure_path
+    elsif !@create_if_deleted && deleted?(file_output_path)
+      file_output_path = @failure_path
+    end
+    @logger.debug("File, writing event to file.", :filename => file_output_path)
+    fd = open(file_output_path)
     # TODO(sissel): Check if we should rotate the file.
-
-    fd.write(event)
-    fd.write("\n")
-
+    fd.write(data)
     flush(fd)
-    close_stale_files
   end
 
   private
@@ -152,15 +163,6 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
     path =~ FIELD_REF
   end
 
-  private
-  def format_message(event)
-    if @message_format
-      event.sprintf(@message_format)
-    else
-      event.to_json
-    end
-  end
-
   private
   def extract_file_root
     parts = File.expand_path(path).split(File::SEPARATOR)

data/logstash-output-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-file'
-  s.version         = '2.1.0'
+  s.version         = '2.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This output will write events to files on disk"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -21,8 +21,9 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
-  s.add_runtime_dependency 'logstash-input-generator'
+  s.add_runtime_dependency 'logstash-codec-json_lines'
+  s.add_runtime_dependency 'logstash-codec-line'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-input-generator'
 end
-

data/spec/outputs/file_spec.rb

@@ -30,7 +30,7 @@ describe LogStash::Outputs::File do
 
     agent do
       line_num = 0
-      
+
       # Now check all events for order and correctness.
       tmp_file.each_line do |line|
         event = LogStash::Event.new(LogStash::Json.load(line))
@@ -169,7 +169,7 @@ describe LogStash::Outputs::File do
 
         it 'writes the bad event in the specified error file' do
           Stud::Temporary.directory('filepath_error') do |path|
-            config = { 
+            config = {
               "path" => "#{path}/%{error}",
               "filename_failure" => "_error"
             }
@@ -297,5 +297,68 @@ describe LogStash::Outputs::File do
         end
       end
     end
+    context "output string format" do
+      context "when using default configuration" do
+        it 'write the event as a json line' do
+          good_event = LogStash::Event.new
+          good_event['message'] = 'hello world'
+
+          Stud::Temporary.directory do |path|
+            config = { "path" => "#{path}/output.txt" }
+            output = LogStash::Outputs::File.new(config)
+            output.register
+            output.receive(good_event)
+            good_file = File.join(path, 'output.txt')
+            expect(File.exist?(good_file)).to eq(true)
+            output.close #teardown first to allow reading the file
+            File.open(good_file) {|f|
+              event = LogStash::Event.new(LogStash::Json.load(f.readline))
+              expect(event["message"]).to eq("hello world")
+            }
+          end
+        end
+      end
+      context "when using line codec" do
+        it 'writes event using specified format' do
+          good_event = LogStash::Event.new
+          good_event['message'] = "hello world"
+
+          Stud::Temporary.directory do |path|
+            config = { "path" => "#{path}/output.txt" }
+            output = LogStash::Outputs::File.new(config)
+            output.codec = LogStash::Codecs::Line.new({ "format" => "Custom format: %{message}"})
+            output.register
+            output.receive(good_event)
+            good_file = File.join(path, 'output.txt')
+            expect(File.exist?(good_file)).to eq(true)
+            output.close #teardown first to allow reading the file
+            File.open(good_file) {|f|
+              line = f.readline
+              expect(line).to eq("Custom format: hello world\n")
+            }
+          end
+        end
+      end
+      context "when using deprecated message_format config" do
+        it 'falls back to line codec' do
+          good_event = LogStash::Event.new
+          good_event['message'] = 'hello world'
+
+          Stud::Temporary.directory do |path|
+            config = { "path" => "#{path}/output.txt", "message_format" => "Custom format: %{message}" }
+            output = LogStash::Outputs::File.new(config)
+            output.register
+            output.receive(good_event)
+            good_file = File.join(path, 'output.txt')
+            expect(File.exist?(good_file)).to eq(true)
+            output.close #teardown first to allow reading the file
+            File.open(good_file) {|f|
+              line = f.readline
+              expect(line).to eq("Custom format: hello world\n")
+            }
+          end
+        end
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-file
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-19 00:00:00.000000000 Z
+date: 2015-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -31,7 +31,21 @@ dependencies:
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
-  name: logstash-input-generator
+  name: logstash-codec-json_lines
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-line
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -58,6 +72,20 @@ dependencies:
         version: '0'
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: logstash-input-generator
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -95,10 +123,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.6
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: This output will write events to files on disk
 test_files:
 - spec/outputs/file_spec.rb
-has_rdoc: