fluentd 0.14.5-x64-mingw32 → 0.14.7-x64-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/ChangeLog +55 -0
- data/bin/fluent-binlog-reader +7 -0
- data/example/in_dummy_with_compression.conf +23 -0
- data/lib/fluent/agent.rb +8 -12
- data/lib/fluent/command/binlog_reader.rb +234 -0
- data/lib/fluent/command/fluentd.rb +17 -1
- data/lib/fluent/compat/file_util.rb +1 -1
- data/lib/fluent/compat/output.rb +5 -1
- data/lib/fluent/config/configure_proxy.rb +18 -3
- data/lib/fluent/config/element.rb +1 -1
- data/lib/fluent/config/section.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +1 -1
- data/lib/fluent/env.rb +1 -0
- data/lib/fluent/event.rb +49 -2
- data/lib/fluent/event_router.rb +6 -2
- data/lib/fluent/label.rb +8 -0
- data/lib/fluent/log.rb +30 -1
- data/lib/fluent/plugin.rb +1 -1
- data/lib/fluent/plugin/base.rb +3 -0
- data/lib/fluent/plugin/buf_file.rb +2 -2
- data/lib/fluent/plugin/buf_memory.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +12 -2
- data/lib/fluent/plugin/buffer/chunk.rb +68 -7
- data/lib/fluent/plugin/buffer/file_chunk.rb +4 -4
- data/lib/fluent/plugin/buffer/memory_chunk.rb +4 -4
- data/lib/fluent/plugin/compressable.rb +91 -0
- data/lib/fluent/plugin/filter_grep.rb +4 -4
- data/lib/fluent/plugin/formatter.rb +2 -2
- data/lib/fluent/plugin/formatter_json.rb +2 -1
- data/lib/fluent/plugin/formatter_out_file.rb +3 -30
- data/lib/fluent/plugin/in_forward.rb +6 -5
- data/lib/fluent/plugin/in_monitor_agent.rb +7 -21
- data/lib/fluent/plugin/in_syslog.rb +1 -1
- data/lib/fluent/plugin/in_tail.rb +11 -2
- data/lib/fluent/plugin/multi_output.rb +63 -3
- data/lib/fluent/plugin/out_exec.rb +1 -1
- data/lib/fluent/plugin/out_file.rb +5 -1
- data/lib/fluent/plugin/out_forward.rb +17 -5
- data/lib/fluent/plugin/out_stdout.rb +2 -1
- data/lib/fluent/plugin/output.rb +205 -19
- data/lib/fluent/plugin/parser.rb +5 -49
- data/lib/fluent/plugin/parser_apache2.rb +1 -1
- data/lib/fluent/plugin/parser_json.rb +4 -4
- data/lib/fluent/plugin/parser_multiline.rb +5 -5
- data/lib/fluent/plugin/parser_regexp.rb +1 -2
- data/lib/fluent/plugin/parser_syslog.rb +2 -2
- data/lib/fluent/plugin/storage_local.rb +2 -1
- data/lib/fluent/plugin_helper.rb +1 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +39 -21
- data/lib/fluent/plugin_helper/extract.rb +92 -0
- data/lib/fluent/plugin_helper/inject.rb +10 -12
- data/lib/fluent/plugin_helper/thread.rb +23 -3
- data/lib/fluent/registry.rb +1 -1
- data/lib/fluent/root_agent.rb +2 -1
- data/lib/fluent/supervisor.rb +28 -8
- data/lib/fluent/test/base.rb +0 -7
- data/lib/fluent/test/driver/base.rb +1 -0
- data/lib/fluent/test/driver/output.rb +3 -0
- data/lib/fluent/test/helpers.rb +18 -0
- data/lib/fluent/test/input_test.rb +4 -2
- data/lib/fluent/test/log.rb +3 -1
- data/lib/fluent/time.rb +232 -1
- data/lib/fluent/timezone.rb +1 -1
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_binlog_reader.rb +351 -0
- data/test/config/test_config_parser.rb +6 -0
- data/test/config/test_configurable.rb +47 -1
- data/test/helper.rb +0 -1
- data/test/plugin/test_buffer.rb +22 -2
- data/test/plugin/test_buffer_chunk.rb +34 -4
- data/test/plugin/test_buffer_file_chunk.rb +73 -0
- data/test/plugin/test_buffer_memory_chunk.rb +73 -0
- data/test/plugin/test_compressable.rb +81 -0
- data/test/plugin/test_formatter_json.rb +14 -1
- data/test/plugin/test_in_forward.rb +67 -3
- data/test/plugin/test_in_monitor_agent.rb +17 -1
- data/test/plugin/test_in_tail.rb +8 -8
- data/test/plugin/test_out_file.rb +0 -8
- data/test/plugin/test_out_forward.rb +85 -0
- data/test/plugin/test_out_secondary_file.rb +20 -12
- data/test/plugin/test_out_stdout.rb +11 -10
- data/test/plugin/test_output.rb +234 -0
- data/test/plugin/test_output_as_buffered.rb +223 -0
- data/test/plugin/test_output_as_buffered_compress.rb +165 -0
- data/test/plugin/test_parser_json.rb +8 -0
- data/test/plugin/test_parser_regexp.rb +1 -1
- data/test/plugin_helper/test_child_process.rb +2 -2
- data/test/plugin_helper/test_extract.rb +195 -0
- data/test/plugin_helper/test_inject.rb +0 -7
- data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
- data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
- data/test/test_event.rb +186 -0
- data/test/test_event_router.rb +1 -1
- data/test/test_formatter.rb +0 -7
- data/test/test_log.rb +121 -0
- data/test/test_plugin_classes.rb +62 -0
- data/test/test_root_agent.rb +125 -0
- data/test/test_supervisor.rb +25 -2
- data/test/test_time_formatter.rb +103 -7
- data/test/test_time_parser.rb +211 -0
- metadata +22 -4
- data/test/plugin/test_parser_time.rb +0 -46
@@ -102,7 +102,7 @@ module Fluent
|
|
102
102
|
end
|
103
103
|
|
104
104
|
def has_key?(key)
|
105
|
-
@unused_in = false # some sections, e.g. <store> in copy, is not defined by config_section so clear unused flag for better warning message in
|
105
|
+
@unused_in = false # some sections, e.g. <store> in copy, is not defined by config_section so clear unused flag for better warning message in check_not_fetched.
|
106
106
|
@unused.delete(key)
|
107
107
|
super
|
108
108
|
end
|
@@ -165,7 +165,7 @@ module Fluent
|
|
165
165
|
proxy.sections.each do |name, subproxy|
|
166
166
|
varname = subproxy.variable_name
|
167
167
|
elements = (conf.respond_to?(:elements) ? conf.elements : []).select{ |e| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
|
168
|
-
if elements.empty? && subproxy.init?
|
168
|
+
if elements.empty? && subproxy.init?
|
169
169
|
elements << Fluent::Config::Element.new(subproxy.name.to_s, '', {}, [])
|
170
170
|
end
|
171
171
|
|
@@ -106,7 +106,7 @@ module Fluent
|
|
106
106
|
else
|
107
107
|
k = scan_string(SPACING)
|
108
108
|
spacing_without_comment
|
109
|
-
if prev_match.include?("\n") # support 'tag_mapped' like "without value" configuration
|
109
|
+
if prev_match.include?("\n") || eof? # support 'tag_mapped' like "without value" configuration
|
110
110
|
attrs[k] = ""
|
111
111
|
else
|
112
112
|
if k == '@include'
|
data/lib/fluent/env.rb
CHANGED
@@ -18,6 +18,7 @@ module Fluent
|
|
18
18
|
DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
|
19
19
|
DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
|
20
20
|
DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
|
21
|
+
DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat}
|
21
22
|
IS_WINDOWS = /mswin|mingw/ === RUBY_PLATFORM
|
22
23
|
private_constant :IS_WINDOWS
|
23
24
|
|
data/lib/fluent/event.rb
CHANGED
@@ -15,11 +15,13 @@
|
|
15
15
|
#
|
16
16
|
|
17
17
|
require 'fluent/msgpack_factory'
|
18
|
+
require 'fluent/plugin/compressable'
|
18
19
|
|
19
20
|
module Fluent
|
20
21
|
class EventStream
|
21
22
|
include Enumerable
|
22
23
|
include MessagePackFactory::Mixin
|
24
|
+
include Fluent::Plugin::Compressable
|
23
25
|
|
24
26
|
# dup does deep copy for event stream
|
25
27
|
def dup
|
@@ -61,6 +63,11 @@ module Fluent
|
|
61
63
|
out.to_s
|
62
64
|
end
|
63
65
|
|
66
|
+
def to_compressed_msgpack_stream(time_int: false)
|
67
|
+
packed = to_msgpack_stream(time_int: time_int)
|
68
|
+
compress(packed)
|
69
|
+
end
|
70
|
+
|
64
71
|
def to_msgpack_stream_forced_integer
|
65
72
|
out = msgpack_packer
|
66
73
|
each {|time,record|
|
@@ -206,9 +213,9 @@ module Fluent
|
|
206
213
|
|
207
214
|
def dup
|
208
215
|
if @unpacked_times
|
209
|
-
|
216
|
+
self.class.new(@data.dup, nil, @size, unpacked_times: @unpacked_times, unpacked_records: @unpacked_records.map(&:dup))
|
210
217
|
else
|
211
|
-
|
218
|
+
self.class.new(@data.dup, nil, @size)
|
212
219
|
end
|
213
220
|
end
|
214
221
|
|
@@ -267,6 +274,46 @@ module Fluent
|
|
267
274
|
end
|
268
275
|
end
|
269
276
|
|
277
|
+
class CompressedMessagePackEventStream < MessagePackEventStream
|
278
|
+
def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)
|
279
|
+
super
|
280
|
+
@decompressed_data = nil
|
281
|
+
@compressed_data = data
|
282
|
+
end
|
283
|
+
|
284
|
+
def empty?
|
285
|
+
ensure_decompressed!
|
286
|
+
super
|
287
|
+
end
|
288
|
+
|
289
|
+
def ensure_unpacked!
|
290
|
+
ensure_decompressed!
|
291
|
+
super
|
292
|
+
end
|
293
|
+
|
294
|
+
def each(&block)
|
295
|
+
ensure_decompressed!
|
296
|
+
super
|
297
|
+
end
|
298
|
+
|
299
|
+
def to_msgpack_stream(time_int: false)
|
300
|
+
ensure_decompressed!
|
301
|
+
super
|
302
|
+
end
|
303
|
+
|
304
|
+
def to_compressed_msgpack_stream(time_int: false)
|
305
|
+
# time_int is always ignored because @data is always packed binary in this class
|
306
|
+
@compressed_data
|
307
|
+
end
|
308
|
+
|
309
|
+
private
|
310
|
+
|
311
|
+
def ensure_decompressed!
|
312
|
+
return if @decompressed_data
|
313
|
+
@data = @decompressed_data = decompress(@data)
|
314
|
+
end
|
315
|
+
end
|
316
|
+
|
270
317
|
module ChunkMessagePackEventStreamer
|
271
318
|
include MessagePackFactory::Mixin
|
272
319
|
# chunk.extend(ChunkEventStreamer)
|
data/lib/fluent/event_router.rb
CHANGED
@@ -207,10 +207,14 @@ module Fluent
|
|
207
207
|
|
208
208
|
def optimizable?
|
209
209
|
return @optimizable unless @optimizable.nil?
|
210
|
-
|
210
|
+
fs_filters = filters_having_filter_stream
|
211
|
+
@optimizable = if fs_filters.empty?
|
211
212
|
true
|
212
213
|
else
|
213
|
-
|
214
|
+
# skip log message when filter is only 1, because its performace is same as non optimized chain.
|
215
|
+
if @filters.size > 1 && fs_filters.size >= 1
|
216
|
+
$log.info "disable filter chain optimization because #{fs_filters.map(&:class)} uses `#filter_stream` method."
|
217
|
+
end
|
214
218
|
false
|
215
219
|
end
|
216
220
|
end
|
data/lib/fluent/label.rb
CHANGED
@@ -27,6 +27,14 @@ module Fluent
|
|
27
27
|
|
28
28
|
attr_accessor :root_agent
|
29
29
|
|
30
|
+
def configure(conf)
|
31
|
+
super
|
32
|
+
|
33
|
+
if conf.elements('match').size == 0
|
34
|
+
raise ConfigError, "Missing <match> sections in <label #{@context}> section"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
30
38
|
def emit_error_event(tag, time, record, e)
|
31
39
|
@root_agent.emit_error_event(tag, time, record, e)
|
32
40
|
end
|
data/lib/fluent/log.rb
CHANGED
@@ -15,6 +15,7 @@
|
|
15
15
|
#
|
16
16
|
|
17
17
|
require 'forwardable'
|
18
|
+
require 'logger'
|
18
19
|
|
19
20
|
module Fluent
|
20
21
|
class Log
|
@@ -122,7 +123,7 @@ module Fluent
|
|
122
123
|
end
|
123
124
|
|
124
125
|
def reopen!
|
125
|
-
# do
|
126
|
+
# do nothing in @logger.reopen! because it's already reopened in Supervisor.load_config
|
126
127
|
@logger.reopen! if @logger
|
127
128
|
nil
|
128
129
|
end
|
@@ -442,4 +443,32 @@ module Fluent
|
|
442
443
|
@log.reset
|
443
444
|
end
|
444
445
|
end
|
446
|
+
|
447
|
+
# This class delegetes some methods which are used in `Fluent::Logger` to a instance variable(`dev`) in `Logger::LogDevice` class
|
448
|
+
# https://github.com/ruby/ruby/blob/7b2d47132ff8ee950b0f978ab772dee868d9f1b0/lib/logger.rb#L661
|
449
|
+
class LogDeviceIO < Logger::LogDevice
|
450
|
+
def flush
|
451
|
+
if @dev.respond_to?(:flush)
|
452
|
+
@dev.flush
|
453
|
+
else
|
454
|
+
super
|
455
|
+
end
|
456
|
+
end
|
457
|
+
|
458
|
+
def tty?
|
459
|
+
if @dev.respond_to?(:tty?)
|
460
|
+
@dev.tty?
|
461
|
+
else
|
462
|
+
super
|
463
|
+
end
|
464
|
+
end
|
465
|
+
|
466
|
+
def sync=(v)
|
467
|
+
if @dev.respond_to?(:sync=)
|
468
|
+
@dev.sync = v
|
469
|
+
else
|
470
|
+
super
|
471
|
+
end
|
472
|
+
end
|
473
|
+
end
|
445
474
|
end
|
data/lib/fluent/plugin.rb
CHANGED
@@ -30,7 +30,7 @@ module Fluent
|
|
30
30
|
OUTPUT_REGISTRY = Registry.new(:output, 'fluent/plugin/out_', dir_search_prefix: 'out_')
|
31
31
|
FILTER_REGISTRY = Registry.new(:filter, 'fluent/plugin/filter_', dir_search_prefix: 'filter_')
|
32
32
|
|
33
|
-
# feature plugin: second class plugins (
|
33
|
+
# feature plugin: second class plugins (instantiated by Plugins or Helpers)
|
34
34
|
BUFFER_REGISTRY = Registry.new(:buffer, 'fluent/plugin/buf_', dir_search_prefix: 'buf_')
|
35
35
|
PARSER_REGISTRY = Registry.new(:parser, 'fluent/plugin/parser_', dir_search_prefix: 'parser_')
|
36
36
|
FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_', dir_search_prefix: 'formatter_')
|
data/lib/fluent/plugin/base.rb
CHANGED
@@ -26,9 +26,12 @@ module Fluent
|
|
26
26
|
|
27
27
|
State = Struct.new(:configure, :start, :after_start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate)
|
28
28
|
|
29
|
+
attr_accessor :under_plugin_development
|
30
|
+
|
29
31
|
def initialize
|
30
32
|
super
|
31
33
|
@_state = State.new(false, false, false, false, false, false, false, false, false)
|
34
|
+
@under_plugin_development = false
|
32
35
|
end
|
33
36
|
|
34
37
|
def has_router?
|
@@ -144,9 +144,9 @@ module Fluent
|
|
144
144
|
def generate_chunk(metadata)
|
145
145
|
# FileChunk generates real path with unique_id
|
146
146
|
if @file_permission
|
147
|
-
Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: @file_permission)
|
147
|
+
Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: @file_permission, compress: @compress)
|
148
148
|
else
|
149
|
-
Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create)
|
149
|
+
Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, compress: @compress)
|
150
150
|
end
|
151
151
|
end
|
152
152
|
end
|
data/lib/fluent/plugin/buffer.rb
CHANGED
@@ -55,6 +55,9 @@ module Fluent
|
|
55
55
|
# if chunk size (or records) is 95% or more after #write, then that chunk will be enqueued
|
56
56
|
config_param :chunk_full_threshold, :float, default: DEFAULT_CHUNK_FULL_THRESHOLD
|
57
57
|
|
58
|
+
desc 'Compress buffered data.'
|
59
|
+
config_param :compress, :enum, list: [:text, :gzip], default: :text
|
60
|
+
|
58
61
|
Metadata = Struct.new(:timekey, :tag, :variables) do
|
59
62
|
def empty?
|
60
63
|
timekey.nil? && tag.nil? && variables.nil?
|
@@ -157,6 +160,13 @@ module Fluent
|
|
157
160
|
end
|
158
161
|
end
|
159
162
|
|
163
|
+
# it's too dangerous, and use it so carefully to remove metadata for tests
|
164
|
+
def metadata_list_clear!
|
165
|
+
synchronize do
|
166
|
+
@metadata_list.clear
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
160
170
|
def new_metadata(timekey: nil, tag: nil, variables: nil)
|
161
171
|
Metadata.new(timekey, tag, variables)
|
162
172
|
end
|
@@ -458,7 +468,7 @@ module Fluent
|
|
458
468
|
serialized = format.call(data)
|
459
469
|
chunk.concat(serialized, size ? size.call : data.size)
|
460
470
|
else
|
461
|
-
chunk.append(data)
|
471
|
+
chunk.append(data, compress: @compress)
|
462
472
|
end
|
463
473
|
adding_bytesize = chunk.bytesize - original_bytesize
|
464
474
|
|
@@ -558,7 +568,7 @@ module Fluent
|
|
558
568
|
if format
|
559
569
|
chunk.concat(format.call(split), split.size)
|
560
570
|
else
|
561
|
-
chunk.append(split)
|
571
|
+
chunk.append(split, compress: @compress)
|
562
572
|
end
|
563
573
|
|
564
574
|
if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
|
@@ -15,18 +15,20 @@
|
|
15
15
|
#
|
16
16
|
|
17
17
|
require 'fluent/plugin/buffer'
|
18
|
+
require 'fluent/plugin/compressable'
|
18
19
|
require 'fluent/unique_id'
|
19
20
|
require 'fluent/event'
|
20
21
|
|
21
22
|
require 'monitor'
|
23
|
+
require 'tempfile'
|
24
|
+
require 'zlib'
|
22
25
|
|
23
26
|
module Fluent
|
24
27
|
module Plugin
|
25
|
-
class Buffer # fluent/plugin/buffer is
|
28
|
+
class Buffer # fluent/plugin/buffer is already loaded
|
26
29
|
class Chunk
|
27
30
|
include MonitorMixin
|
28
31
|
include UniqueId::Mixin
|
29
|
-
include ChunkMessagePackEventStreamer
|
30
32
|
|
31
33
|
# Chunks has 2 part:
|
32
34
|
# * metadata: contains metadata which should be restored after resume (if possible)
|
@@ -46,7 +48,7 @@ module Fluent
|
|
46
48
|
|
47
49
|
# TODO: CompressedPackedMessage of forward protocol?
|
48
50
|
|
49
|
-
def initialize(metadata)
|
51
|
+
def initialize(metadata, compress: :text)
|
50
52
|
super()
|
51
53
|
@unique_id = generate_unique_id
|
52
54
|
@metadata = metadata
|
@@ -57,12 +59,15 @@ module Fluent
|
|
57
59
|
@size = 0
|
58
60
|
@created_at = Time.now
|
59
61
|
@modified_at = Time.now
|
62
|
+
|
63
|
+
extend Decompressable if compress == :gzip
|
60
64
|
end
|
61
65
|
|
62
66
|
attr_reader :unique_id, :metadata, :created_at, :modified_at, :state
|
63
67
|
|
64
68
|
# data is array of formatted record string
|
65
|
-
def append(data)
|
69
|
+
def append(data, **kwargs)
|
70
|
+
raise ArgumentError, '`compress: gzip` can be used for Compressable module' if kwargs[:compress] == :gzip
|
66
71
|
adding = ''.b
|
67
72
|
data.each do |d|
|
68
73
|
adding << d.b
|
@@ -141,19 +146,75 @@ module Fluent
|
|
141
146
|
self
|
142
147
|
end
|
143
148
|
|
144
|
-
def read
|
149
|
+
def read(**kwargs)
|
150
|
+
raise ArgumentError, '`compressed: gzip` can be used for Compressable module' if kwargs[:compressed] == :gzip
|
145
151
|
raise NotImplementedError, "Implement this method in child class"
|
146
152
|
end
|
147
153
|
|
148
|
-
def open(&block)
|
154
|
+
def open(**kwargs, &block)
|
155
|
+
raise ArgumentError, '`compressed: gzip` can be used for Compressable module' if kwargs[:compressed] == :gzip
|
149
156
|
raise NotImplementedError, "Implement this method in child class"
|
150
157
|
end
|
151
158
|
|
152
|
-
def write_to(io)
|
159
|
+
def write_to(io, **kwargs)
|
160
|
+
raise ArgumentError, '`compressed: gzip` can be used for Compressable module' if kwargs[:compressed] == :gzip
|
153
161
|
open do |i|
|
154
162
|
IO.copy_stream(i, io)
|
155
163
|
end
|
156
164
|
end
|
165
|
+
|
166
|
+
module Decompressable
|
167
|
+
include Fluent::Plugin::Compressable
|
168
|
+
|
169
|
+
def append(data, **kwargs)
|
170
|
+
if kwargs[:compress] == :gzip
|
171
|
+
io = StringIO.new
|
172
|
+
Zlib::GzipWriter.wrap(io) do |gz|
|
173
|
+
data.each do |d|
|
174
|
+
gz.write d
|
175
|
+
end
|
176
|
+
end
|
177
|
+
concat(io.string, data.size)
|
178
|
+
else
|
179
|
+
super
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
def open(**kwargs, &block)
|
184
|
+
if kwargs[:compressed] == :gzip
|
185
|
+
super
|
186
|
+
else
|
187
|
+
super(kwargs) do |chunk_io|
|
188
|
+
output_io = if chunk_io.is_a?(StringIO)
|
189
|
+
StringIO.new
|
190
|
+
else
|
191
|
+
Tempfile.new('decompressed-data')
|
192
|
+
end
|
193
|
+
decompress(input_io: chunk_io, output_io: output_io)
|
194
|
+
output_io.seek(0, IO::SEEK_SET)
|
195
|
+
yield output_io
|
196
|
+
end
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
def read(**kwargs)
|
201
|
+
if kwargs[:compressed] == :gzip
|
202
|
+
super
|
203
|
+
else
|
204
|
+
decompress(super)
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
def write_to(io, **kwargs)
|
209
|
+
open(compressed: :gzip) do |chunk_io|
|
210
|
+
if kwargs[:compressed] == :gzip
|
211
|
+
IO.copy_stream(chunk_io, io)
|
212
|
+
else
|
213
|
+
decompress(input_io: chunk_io, output_io: io)
|
214
|
+
end
|
215
|
+
end
|
216
|
+
end
|
217
|
+
end
|
157
218
|
end
|
158
219
|
end
|
159
220
|
end
|
@@ -40,8 +40,8 @@ module Fluent
|
|
40
40
|
|
41
41
|
attr_reader :path, :permission
|
42
42
|
|
43
|
-
def initialize(metadata, path, mode, perm: system_config.file_permission || FILE_PERMISSION)
|
44
|
-
super(metadata)
|
43
|
+
def initialize(metadata, path, mode, perm: system_config.file_permission || FILE_PERMISSION, compress: :text)
|
44
|
+
super(metadata, compress: compress)
|
45
45
|
@permission = perm
|
46
46
|
@bytesize = @size = @adding_bytes = @adding_size = 0
|
47
47
|
@meta = nil
|
@@ -133,12 +133,12 @@ module Fluent
|
|
133
133
|
File.unlink(@path, @meta_path)
|
134
134
|
end
|
135
135
|
|
136
|
-
def read
|
136
|
+
def read(**kwargs)
|
137
137
|
@chunk.seek(0, IO::SEEK_SET)
|
138
138
|
@chunk.read
|
139
139
|
end
|
140
140
|
|
141
|
-
def open(&block)
|
141
|
+
def open(**kwargs, &block)
|
142
142
|
@chunk.seek(0, IO::SEEK_SET)
|
143
143
|
val = yield @chunk
|
144
144
|
@chunk.seek(0, IO::SEEK_END) if self.staged?
|
@@ -20,7 +20,7 @@ module Fluent
|
|
20
20
|
module Plugin
|
21
21
|
class Buffer
|
22
22
|
class MemoryChunk < Chunk
|
23
|
-
def initialize(metadata)
|
23
|
+
def initialize(metadata, compress: :text)
|
24
24
|
super
|
25
25
|
@chunk = ''.force_encoding(Encoding::ASCII_8BIT)
|
26
26
|
@chunk_bytes = 0
|
@@ -72,15 +72,15 @@ module Fluent
|
|
72
72
|
true
|
73
73
|
end
|
74
74
|
|
75
|
-
def read
|
75
|
+
def read(**kwargs)
|
76
76
|
@chunk
|
77
77
|
end
|
78
78
|
|
79
|
-
def open(&block)
|
79
|
+
def open(**kwargs, &block)
|
80
80
|
StringIO.open(@chunk, &block)
|
81
81
|
end
|
82
82
|
|
83
|
-
def write_to(io)
|
83
|
+
def write_to(io, **kwargs)
|
84
84
|
# re-implementation to optimize not to create StringIO
|
85
85
|
io.write @chunk
|
86
86
|
end
|