logstash-output-file 2.1.0 → 2.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 34e5dcb5b4862273a9cc92f1c4ab73235db7df4a
4
- data.tar.gz: 802613ca119351bcb1970d3ec96efa52c18b6a38
3
+ metadata.gz: aa80999ed0b695c5970cccfc4ef9a84c6c688d60
4
+ data.tar.gz: c4f1584eff0dc5eae5bc4eb109b6102bba60f79a
5
5
  SHA512:
6
- metadata.gz: ab71908733217054a01c5688cd3a31a6257350440440ba4885d7ac15a8404207b7fd2b19b067790541fd6db40d5e99e3b7cfd9173fc82f3b38c16daa4d4b4f76
7
- data.tar.gz: 798c54d4b133f2aed3dba737254dc2807ad3d6f0253a2ae6e06e8b51bee5e7b691415532cab396868b60959a6c92c84ba1eb4424786fccf7a4b978914c338a36
6
+ metadata.gz: ef3b7ea580bb5f0278fda8cf7c0da9f2ddffc6b1dcec67469e4e1f70d57fbc0cf4eefe1177147833a8afd214d905719f41f7bf25bda3ef3b1d538282497035db
7
+ data.tar.gz: 07c59b12f42771a38d93b4021ed2d5992433e8ad6bda53bf951ad594a42845d93d502701678df20f977add39168cfdac06e0fbc59d13a3d1031f429b3dc695e4
data/CHANGELOG.md CHANGED
@@ -1,3 +1,7 @@
1
+ ## 2.2.0
2
+ - Add support for codec, using **json_lines** as default codec to keep default behavior.
3
+ Ref: https://github.com/logstash-plugins/logstash-output-file/pull/9
4
+
1
5
  ## 2.1.0
2
6
  - Add create_if_deleted option to create a destination file in case it
3
7
  was deleted by another agent in the machine. In case of being false
@@ -4,8 +4,18 @@ require "logstash/outputs/base"
4
4
  require "logstash/errors"
5
5
  require "zlib"
6
6
 
7
- # This output will write events to files on disk. You can use fields
7
+ # This output writes events to files on disk. You can use fields
8
8
  # from the event as parts of the filename and/or path.
9
+ #
10
+ # By default, this output writes one event per line in **json** format.
11
+ # You can customise the line format using the `line` codec like
12
+ # [source,ruby]
13
+ # output {
14
+ # file {
15
+ # path => ...
16
+ # codec => { line { format => "custom format: %{message}"}}
17
+ # }
18
+ # }
9
19
  class LogStash::Outputs::File < LogStash::Outputs::Base
10
20
  FIELD_REF = /%\{[^}]+\}/
11
21
 
@@ -31,7 +41,7 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
31
41
  #
32
42
  # If this setting is omitted, the full json representation of the
33
43
  # event will be written as a single line.
34
- config :message_format, :validate => :string
44
+ config :message_format, :validate => :string, :deprecated => "You can achieve the same behavior with the 'line' codec"
35
45
 
36
46
  # Flush interval (in seconds) for flushing writes to log files.
37
47
  # 0 will flush on every message.
@@ -47,7 +57,8 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
47
57
  # If the a file is deleted, but an event is comming with the need to be stored
48
58
  # in such a file, the plugin will created a gain this file. Default => true
49
59
  config :create_if_deleted, :validate => :boolean, :default => true
50
-
60
+
61
+ default :codec, "json_lines"
51
62
 
52
63
  public
53
64
  def register
@@ -73,6 +84,13 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
73
84
  @last_stale_cleanup_cycle = now
74
85
  @flush_interval = @flush_interval.to_i
75
86
  @stale_cleanup_interval = 10
87
+
88
+ if @message_format
89
+ @codec = LogStash::Plugin.lookup("codec", "line").new
90
+ @codec.format = @message_format
91
+ end
92
+
93
+ @codec.on_event(&method(:write_event))
76
94
  end # def register
77
95
 
78
96
  private
@@ -96,18 +114,8 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
96
114
 
97
115
  public
98
116
  def receive(event)
99
-
100
- file_output_path = generate_filepath(event)
101
-
102
- if path_with_field_ref? && !inside_file_root?(file_output_path)
103
- @logger.warn("File: the event tried to write outside the files root, writing the event to the failure file", :event => event, :filename => @failure_path)
104
- file_output_path = @failure_path
105
- elsif !@create_if_deleted && deleted?(file_output_path)
106
- file_output_path = @failure_path
107
- end
108
-
109
- output = format_message(event)
110
- write_event(file_output_path, output)
117
+ @codec.encode(event)
118
+ close_stale_files
111
119
  end # def receive
112
120
 
113
121
  public
@@ -130,16 +138,19 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
130
138
  end
131
139
 
132
140
  private
133
- def write_event(log_path, event)
134
- @logger.debug("File, writing event to file.", :filename => log_path)
135
- fd = open(log_path)
141
+ def write_event(event, data)
142
+ file_output_path = generate_filepath(event)
143
+ if path_with_field_ref? && !inside_file_root?(file_output_path)
144
+ @logger.warn("File: the event tried to write outside the files root, writing the event to the failure file", :event => event, :filename => @failure_path)
145
+ file_output_path = @failure_path
146
+ elsif !@create_if_deleted && deleted?(file_output_path)
147
+ file_output_path = @failure_path
148
+ end
149
+ @logger.debug("File, writing event to file.", :filename => file_output_path)
150
+ fd = open(file_output_path)
136
151
  # TODO(sissel): Check if we should rotate the file.
137
-
138
- fd.write(event)
139
- fd.write("\n")
140
-
152
+ fd.write(data)
141
153
  flush(fd)
142
- close_stale_files
143
154
  end
144
155
 
145
156
  private
@@ -152,15 +163,6 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
152
163
  path =~ FIELD_REF
153
164
  end
154
165
 
155
- private
156
- def format_message(event)
157
- if @message_format
158
- event.sprintf(@message_format)
159
- else
160
- event.to_json
161
- end
162
- end
163
-
164
166
  private
165
167
  def extract_file_root
166
168
  parts = File.expand_path(path).split(File::SEPARATOR)
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-file'
4
- s.version = '2.1.0'
4
+ s.version = '2.2.0'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "This output will write events to files on disk"
7
7
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -21,8 +21,9 @@ Gem::Specification.new do |s|
21
21
 
22
22
  # Gem dependencies
23
23
  s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
24
- s.add_runtime_dependency 'logstash-input-generator'
24
+ s.add_runtime_dependency 'logstash-codec-json_lines'
25
+ s.add_runtime_dependency 'logstash-codec-line'
25
26
 
26
27
  s.add_development_dependency 'logstash-devutils'
28
+ s.add_development_dependency 'logstash-input-generator'
27
29
  end
28
-
@@ -30,7 +30,7 @@ describe LogStash::Outputs::File do
30
30
 
31
31
  agent do
32
32
  line_num = 0
33
-
33
+
34
34
  # Now check all events for order and correctness.
35
35
  tmp_file.each_line do |line|
36
36
  event = LogStash::Event.new(LogStash::Json.load(line))
@@ -169,7 +169,7 @@ describe LogStash::Outputs::File do
169
169
 
170
170
  it 'writes the bad event in the specified error file' do
171
171
  Stud::Temporary.directory('filepath_error') do |path|
172
- config = {
172
+ config = {
173
173
  "path" => "#{path}/%{error}",
174
174
  "filename_failure" => "_error"
175
175
  }
@@ -297,5 +297,68 @@ describe LogStash::Outputs::File do
297
297
  end
298
298
  end
299
299
  end
300
+ context "output string format" do
301
+ context "when using default configuration" do
302
+ it 'write the event as a json line' do
303
+ good_event = LogStash::Event.new
304
+ good_event['message'] = 'hello world'
305
+
306
+ Stud::Temporary.directory do |path|
307
+ config = { "path" => "#{path}/output.txt" }
308
+ output = LogStash::Outputs::File.new(config)
309
+ output.register
310
+ output.receive(good_event)
311
+ good_file = File.join(path, 'output.txt')
312
+ expect(File.exist?(good_file)).to eq(true)
313
+ output.close #teardown first to allow reading the file
314
+ File.open(good_file) {|f|
315
+ event = LogStash::Event.new(LogStash::Json.load(f.readline))
316
+ expect(event["message"]).to eq("hello world")
317
+ }
318
+ end
319
+ end
320
+ end
321
+ context "when using line codec" do
322
+ it 'writes event using specified format' do
323
+ good_event = LogStash::Event.new
324
+ good_event['message'] = "hello world"
325
+
326
+ Stud::Temporary.directory do |path|
327
+ config = { "path" => "#{path}/output.txt" }
328
+ output = LogStash::Outputs::File.new(config)
329
+ output.codec = LogStash::Codecs::Line.new({ "format" => "Custom format: %{message}"})
330
+ output.register
331
+ output.receive(good_event)
332
+ good_file = File.join(path, 'output.txt')
333
+ expect(File.exist?(good_file)).to eq(true)
334
+ output.close #teardown first to allow reading the file
335
+ File.open(good_file) {|f|
336
+ line = f.readline
337
+ expect(line).to eq("Custom format: hello world\n")
338
+ }
339
+ end
340
+ end
341
+ end
342
+ context "when using deprecated message_format config" do
343
+ it 'falls back to line codec' do
344
+ good_event = LogStash::Event.new
345
+ good_event['message'] = 'hello world'
346
+
347
+ Stud::Temporary.directory do |path|
348
+ config = { "path" => "#{path}/output.txt", "message_format" => "Custom format: %{message}" }
349
+ output = LogStash::Outputs::File.new(config)
350
+ output.register
351
+ output.receive(good_event)
352
+ good_file = File.join(path, 'output.txt')
353
+ expect(File.exist?(good_file)).to eq(true)
354
+ output.close #teardown first to allow reading the file
355
+ File.open(good_file) {|f|
356
+ line = f.readline
357
+ expect(line).to eq("Custom format: hello world\n")
358
+ }
359
+ end
360
+ end
361
+ end
362
+ end
300
363
  end
301
364
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-file
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.1.0
4
+ version: 2.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-11-19 00:00:00.000000000 Z
11
+ date: 2015-11-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: logstash-core
@@ -31,7 +31,21 @@ dependencies:
31
31
  prerelease: false
32
32
  type: :runtime
33
33
  - !ruby/object:Gem::Dependency
34
- name: logstash-input-generator
34
+ name: logstash-codec-json_lines
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - '>='
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
40
+ requirement: !ruby/object:Gem::Requirement
41
+ requirements:
42
+ - - '>='
43
+ - !ruby/object:Gem::Version
44
+ version: '0'
45
+ prerelease: false
46
+ type: :runtime
47
+ - !ruby/object:Gem::Dependency
48
+ name: logstash-codec-line
35
49
  version_requirements: !ruby/object:Gem::Requirement
36
50
  requirements:
37
51
  - - '>='
@@ -58,6 +72,20 @@ dependencies:
58
72
  version: '0'
59
73
  prerelease: false
60
74
  type: :development
75
+ - !ruby/object:Gem::Dependency
76
+ name: logstash-input-generator
77
+ version_requirements: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - '>='
80
+ - !ruby/object:Gem::Version
81
+ version: '0'
82
+ requirement: !ruby/object:Gem::Requirement
83
+ requirements:
84
+ - - '>='
85
+ - !ruby/object:Gem::Version
86
+ version: '0'
87
+ prerelease: false
88
+ type: :development
61
89
  description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
62
90
  email: info@elastic.co
63
91
  executables: []
@@ -95,10 +123,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
95
123
  version: '0'
96
124
  requirements: []
97
125
  rubyforge_project:
98
- rubygems_version: 2.4.6
126
+ rubygems_version: 2.4.8
99
127
  signing_key:
100
128
  specification_version: 4
101
129
  summary: This output will write events to files on disk
102
130
  test_files:
103
131
  - spec/outputs/file_spec.rb
104
- has_rdoc: