fluent-plugin-td 0.11.0.rc1 → 1.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: d1ae29d128d90dd331b4cc446ea185179c1a0a33
4
- data.tar.gz: 5d8ed306878be4384000dcee1f60e94cea1a591d
3
+ metadata.gz: 41182a2b265e1d4e1cfa4a1fdc6936c972b83a03
4
+ data.tar.gz: 5cec317a97fa59bd1f71a05f5cf72d2900618220
5
5
  SHA512:
6
- metadata.gz: d2a2dde68301f045c80edcedcd0deaa6830eab6f449cabc9c5c98fc698138396d7f1a2556bf93f771096eecd9b9bdc12bdf6d97b6f32476c93f13fb6ebca3860
7
- data.tar.gz: dcc9909052e4b5450f80d29b8f222803c0f70ea84c6db3ae00582fb70ecd9fa94fe69185604d09ac11ea41b76553d6897212b51bf8ef0e147458d7138d486b69
6
+ metadata.gz: 36d0dea559b0481a8b262cd1f7aad273d7ae5c213179d010623fcbf78f4fe94f1f58b91d83acd86f7093b9b27d661a0dad57c7fd59b2aa21576dc079c7f23182
7
+ data.tar.gz: 0cd0f5890c5132cfe622bd3cdb5a1ba14ca5b5a801d78befe2ca12a97847134bb256b28307c5618d5a11b09f2feb732876cc58f2b4575984adbea02ac8e6575f
@@ -7,7 +7,6 @@ rvm:
7
7
 
8
8
  gemfile:
9
9
  - Gemfile
10
- - Gemfile.v0.12
11
10
 
12
11
  before_install: gem update bundler
13
12
  script: bundle exec rake test
@@ -17,6 +16,3 @@ sudo: false
17
16
  matrix:
18
17
  allow_failures:
19
18
  - rvm: ruby-head
20
- exclude:
21
- - rvm: 2.4.0
22
- gemfile: Gemfile.v0.12
data/ChangeLog CHANGED
@@ -1,3 +1,9 @@
1
+ Release 1.0.0.rc1 - 2017/02/23
2
+
3
+ * Use new Plugin API
4
+ * Update fluentd dependency to v0.14.13 or later
5
+
6
+
1
7
  Release 0.11.0.rc1 - 2017/01/20
2
8
 
3
9
  * Fix event time handling for v0.14
@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
7
7
  gem.description = "Treasure Data Cloud Data Service plugin for Fluentd"
8
8
  gem.homepage = "http://www.treasuredata.com/"
9
9
  gem.summary = gem.description
10
- gem.version = Fluent::TreasureDataPlugin::VERSION
10
+ gem.version = Fluent::Plugin::TreasureDataPlugin::VERSION
11
11
  gem.authors = ["Treasure Data, Inc."]
12
12
  gem.email = "support@treasure-data.com"
13
13
  gem.has_rdoc = false
@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
18
18
  gem.require_paths = ['lib']
19
19
  gem.license = "Apache-2.0"
20
20
 
21
- gem.add_dependency "fluentd", [">= 0.12.0", "< 2"]
21
+ gem.add_dependency "fluentd", [">= 0.14.13", "< 2"]
22
22
  gem.add_dependency "td-client", "~> 1.0"
23
23
  gem.add_development_dependency "rake", ">= 0.9.2"
24
24
  gem.add_development_dependency "webmock", "~> 1.16"
@@ -1,41 +1,44 @@
1
+ require 'fileutils'
2
+ require 'tempfile'
3
+ require 'zlib'
4
+ require 'stringio'
1
5
  require 'td-client'
2
- require 'fluent/output'
6
+
7
+ require 'fluent/plugin/output'
3
8
  require 'fluent/plugin/td_plugin_version'
4
9
 
5
- module Fluent
6
- class TreasureDataLogOutput < BufferedOutput
7
- Plugin.register_output('tdlog', self)
10
+ module Fluent::Plugin
11
+ class TreasureDataLogOutput < Output
12
+ Fluent::Plugin.register_output('tdlog', self)
8
13
 
9
14
  IMPORT_SIZE_LIMIT = 32 * 1024 * 1024
15
+ UPLOAD_EXT = 'msgpack.gz'.freeze
10
16
 
11
- # To support log_level option since Fluentd v0.10.43
12
- unless method_defined?(:log)
13
- define_method(:log) { $log }
14
- end
17
+ helpers :event_emitter, :compat_parameters
15
18
 
16
19
  config_param :apikey, :string, :secret => true
17
20
  config_param :auto_create_table, :bool, :default => true
21
+ config_param :database, :string, :default => nil
22
+ config_param :table, :string, :default => nil
18
23
  config_param :use_gzip_command, :bool, :default => false
19
24
 
20
25
  config_param :endpoint, :string, :default => TreasureData::API::NEW_DEFAULT_ENDPOINT
21
26
  config_param :use_ssl, :bool, :default => true
27
+ config_param :tmpdir, :string, :default => nil
28
+ config_param :http_proxy, :string, :default => nil
22
29
  config_param :connect_timeout, :integer, :default => nil
23
30
  config_param :read_timeout, :integer, :default => nil
24
31
  config_param :send_timeout, :integer, :default => nil
25
- config_set_default :flush_interval, 300
32
+
33
+ config_section :buffer do
34
+ config_set_default :@type, 'file'
35
+ config_set_default :chunk_keys, ['tag']
36
+ config_set_default :flush_interval, 300
37
+ config_set_default :chunk_limit_size, IMPORT_SIZE_LIMIT
38
+ end
26
39
 
27
40
  def initialize
28
- require 'fileutils'
29
- require 'tempfile'
30
- require 'zlib'
31
- require 'net/http'
32
- require 'json'
33
- require 'cgi' # CGI.escape
34
- require 'time' # Time#rfc2822
35
- require 'digest/md5'
36
- require 'stringio'
37
41
  super
38
- @tmpdir = nil
39
42
  @key = nil
40
43
  @key_num_limit = 512 # TODO: Our one-time import has the restriction about the number of record keys.
41
44
  @record_size_limit = 32 * 1024 * 1024 # TODO
@@ -45,15 +48,7 @@ module Fluent
45
48
  end
46
49
 
47
50
  def configure(conf)
48
- # overwrite default value of buffer_chunk_limit
49
- unless conf.has_key?('buffer_chunk_limit')
50
- conf['buffer_chunk_limit'] = IMPORT_SIZE_LIMIT
51
- end
52
-
53
- # v0.14 seems to have a bug of config_set_default: https://github.com/treasure-data/fluent-plugin-td/pull/22#issuecomment-230782005
54
- unless conf.has_key?('buffer_type')
55
- conf['buffer_type'] = 'file'
56
- end
51
+ compat_parameters_convert(conf, :buffer, default_chunk_key: 'tag')
57
52
 
58
53
  super
59
54
 
@@ -67,19 +62,16 @@ module Fluent
67
62
  end
68
63
  end
69
64
 
70
- if conf.has_key?('tmpdir')
71
- @tmpdir = conf['tmpdir']
72
- FileUtils.mkdir_p(@tmpdir)
73
- end
65
+ FileUtils.mkdir_p(@tmpdir) if @tmpdir
74
66
 
75
- database = conf['database']
76
- table = conf['table']
77
- if database && table
78
- validate_database_and_table_name(database, table, conf)
79
- @key = "#{database}.#{table}"
67
+ if @database && @table
68
+ validate_database_and_table_name(@database, @table)
69
+ @key = "#{@database}.#{@table}"
70
+ else
71
+ unless @chunk_key_tag
72
+ raise Fluent::ConfigError, "'tag' must be included in <buffer ARG> when database and table are not specified"
73
+ end
80
74
  end
81
-
82
- @http_proxy = conf['http_proxy']
83
75
  end
84
76
 
85
77
  def start
@@ -93,72 +85,51 @@ module Fluent
93
85
 
94
86
  if @key
95
87
  if @auto_create_table
96
- database, table = @key.split('.',2)
97
- ensure_database_and_table(database, table)
88
+ ensure_database_and_table(@database, @table)
98
89
  else
99
90
  check_table_exists(@key)
100
91
  end
101
92
  end
102
93
  end
103
94
 
104
- def emit(tag, es, chain)
105
- if @key
106
- key = @key
107
- else
108
- database, table = tag.split('.')[-2,2]
109
- database = TreasureData::API.normalize_database_name(database)
110
- table = TreasureData::API.normalize_table_name(table)
111
- key = "#{database}.#{table}"
112
- end
113
-
114
- unless @auto_create_table
115
- check_table_exists(key)
116
- end
117
-
118
- super(tag, es, chain, key)
95
+ def multi_workers_ready?
96
+ true
119
97
  end
120
98
 
121
- def format_stream(tag, es)
122
- out = MessagePack::Buffer.new
123
- off = out.size # size is same as bytesize in ASCII-8BIT string
124
- es.each { |time, record|
125
- # Applications may send non-hash record or broken chunk may generate non-hash record so such records should be skipped
126
- next unless record.is_a?(Hash)
99
+ def formatted_to_msgpack_binary
100
+ true
101
+ end
127
102
 
128
- begin
129
- record['time'] = time.to_i
130
- record.delete(:time) if record.has_key?(:time)
103
+ def format(tag, time, record)
104
+ begin
105
+ record['time'] = time.to_i
106
+ record.delete(:time) if record.has_key?(:time)
131
107
 
132
- if record.size > @key_num_limit
133
- raise "Too many number of keys (#{record.size} keys)" # TODO include summary of the record
134
- end
135
- rescue => e
136
- # TODO (a) Remove the transaction mechanism of fluentd
137
- # or (b) keep transaction boundaries in in/out_forward.
138
- # This code disables the transaction mechanism (a).
139
- log.warn "Skipped a broken record (#{e}): #{summarize_record(record)}"
140
- log.warn_backtrace e.backtrace
141
- next
108
+ if record.size > @key_num_limit
109
+ # TODO include summary of the record
110
+ router.emit_error_event(tag, time, record, RuntimeError.new("too many number of keys (#{record.size} keys)"))
111
+ return nil
142
112
  end
113
+ rescue => e
114
+ router.emit_error_event(tag, time, {'record' => record}, RuntimeError.new("skipped a broken record: #{e}"))
115
+ return nil
116
+ end
143
117
 
144
- begin
145
- record.to_msgpack(out)
146
- rescue RangeError
147
- # In msgpack v0.5, 'out' becomes String, not Buffer. This is not a problem because Buffer has a compatibility with String
148
- out = out.to_s[0, off]
149
- TreasureData::API.normalized_msgpack(record, out)
150
- end
118
+ begin
119
+ result = record.to_msgpack
120
+ rescue RangeError
121
+ result = TreasureData::API.normalized_msgpack(record)
122
+ rescue => e
123
+ router.emit_error_event(tag, time, {'record' => record}, RuntimeError.new("can't convert record to msgpack: #{e}"))
124
+ return nil
125
+ end
151
126
 
152
- noff = out.size
153
- sz = noff - off
154
- if sz > @record_size_limit
155
- # TODO don't raise error
156
- #raise "Size of a record too large (#{sz} bytes)" # TODO include summary of the record
157
- log.warn "Size of a record too large (#{sz} bytes): #{summarize_record(record)}"
158
- end
159
- off = noff
160
- }
161
- out.to_s
127
+ if result.bytesize > @record_size_limit
128
+ # Don't raise error. Large size is not critical for streaming import
129
+ log.warn "Size of a record too large (#{result.bytesize} bytes): #{summarize_record(record)}"
130
+ end
131
+
132
+ result
162
133
  end
163
134
 
164
135
  def summarize_record(record)
@@ -172,10 +143,17 @@ module Fluent
172
143
 
173
144
  def write(chunk)
174
145
  unique_id = chunk.unique_id
175
- database, table = chunk.key.split('.', 2)
146
+
147
+ if @key
148
+ database, table = @database, @table
149
+ else
150
+ database, table = chunk.metadata.tag.split('.')[-2, 2]
151
+ database = TreasureData::API.normalize_database_name(database)
152
+ table = TreasureData::API.normalize_table_name(table)
153
+ end
176
154
 
177
155
  FileUtils.mkdir_p(@tmpdir) unless @tmpdir.nil?
178
- f = Tempfile.new("tdlog-#{chunk.key}-", @tmpdir)
156
+ f = Tempfile.new("tdlog-#{chunk.metadata.tag}-", @tmpdir)
179
157
  f.binmode
180
158
 
181
159
  size = if @use_gzip_command
@@ -191,11 +169,11 @@ module Fluent
191
169
 
192
170
  # TODO: Share this routine with s3 compressors
193
171
  def gzip_by_command(chunk, tmp)
194
- chunk_is_file = @buffer_type == 'file'
172
+ chunk_is_file = @buffer_config['@type'] == 'file'
195
173
  path = if chunk_is_file
196
174
  chunk.path
197
175
  else
198
- w = Tempfile.new("gzip-tdlog-#{chunk.key}-", @tmpdir)
176
+ w = Tempfile.new("gzip-tdlog-#{chunk.metadata.tag}-", @tmpdir)
199
177
  w.binmode
200
178
  chunk.write_to(w)
201
179
  w.close
@@ -235,7 +213,7 @@ module Fluent
235
213
  begin
236
214
  begin
237
215
  start = Time.now
238
- @client.import(database, table, "msgpack.gz", io, size, unique_str)
216
+ @client.import(database, table, UPLOAD_EXT, io, size, unique_str)
239
217
  rescue TreasureData::NotFoundError => e
240
218
  unless @auto_create_table
241
219
  raise e
@@ -258,7 +236,7 @@ module Fluent
258
236
  log.debug "checking whether table '#{database}.#{table}' exists on Treasure Data"
259
237
  io = StringIO.new(@empty_gz_data)
260
238
  begin
261
- @client.import(database, table, "msgpack.gz", io, io.size)
239
+ @client.import(database, table, UPLOAD_EXT, io, io.size)
262
240
  @table_list[key] = true
263
241
  rescue TreasureData::NotFoundError
264
242
  raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td table:create #{database} #{table}' to create it."
@@ -269,16 +247,16 @@ module Fluent
269
247
  end
270
248
  end
271
249
 
272
- def validate_database_and_table_name(database, table, conf)
250
+ def validate_database_and_table_name(database, table)
273
251
  begin
274
252
  TreasureData::API.validate_database_name(database)
275
253
  rescue => e
276
- raise ConfigError, "Invalid database name #{database.inspect}: #{e}: #{conf}"
254
+ raise ConfigError, "Invalid database name #{database.inspect}: #{e}"
277
255
  end
278
256
  begin
279
257
  TreasureData::API.validate_table_name(table)
280
258
  rescue => e
281
- raise ConfigError, "Invalid table name #{table.inspect}: #{e}: #{conf}"
259
+ raise ConfigError, "Invalid table name #{table.inspect}: #{e}"
282
260
  end
283
261
  end
284
262
 
@@ -1,5 +1,7 @@
1
1
  module Fluent
2
- module TreasureDataPlugin
3
- VERSION = '0.11.0.rc1'
2
+ module Plugin
3
+ module TreasureDataPlugin
4
+ VERSION = '1.0.0.rc1'
5
+ end
4
6
  end
5
7
  end
@@ -1,26 +1,36 @@
1
1
  require 'fluent/test'
2
+ require 'fluent/test/driver/output'
2
3
  require 'fluent/plugin/out_tdlog'
3
4
  require 'test_helper.rb'
4
5
 
5
6
  class TreasureDataLogOutputTest < Test::Unit::TestCase
7
+ TMP_DIR = File.dirname(__FILE__) + "/tmp"
8
+
6
9
  def setup
10
+ super
7
11
  Fluent::Test.setup
12
+ FileUtils.rm_rf(TMP_DIR, secure: true)
13
+ FileUtils.mkdir_p(TMP_DIR)
8
14
  end
9
15
 
10
- TMP_DIR = File.dirname(__FILE__) + "/tmp"
16
+ def teardown
17
+ super
18
+ Fluent::Engine.stop
19
+ end
11
20
 
21
+ BASE_CONFIG = %[
22
+ apikey testkey
23
+ buffer_path #{TMP_DIR}/buffer
24
+ ]
12
25
  DEFAULT_CONFIG = %[
13
26
  database test
14
27
  table table
15
28
  ]
16
29
 
17
30
  def create_driver(conf = DEFAULT_CONFIG)
18
- config = %[
19
- apikey testkey
20
- buffer_path #{TMP_DIR}/buffer
21
- ] + conf
31
+ config = BASE_CONFIG + conf
22
32
 
23
- Fluent::Test::BufferedOutputTestDriver.new(Fluent::TreasureDataLogOutput) do
33
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::TreasureDataLogOutput) do
24
34
  def write(chunk)
25
35
  chunk.instance_variable_set(:@key, @key)
26
36
  def chunk.key
@@ -34,15 +44,30 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
34
44
  def test_configure
35
45
  d = create_driver
36
46
 
37
- {:@apikey => 'testkey', :@use_ssl => true, :@auto_create_table => true,
38
- :@buffer_type => 'file', :@flush_interval => 300, :@use_gzip_command => false}.each { |k, v|
47
+ {:@apikey => 'testkey', :@use_ssl => true, :@auto_create_table => true, :@use_gzip_command => false}.each { |k, v|
39
48
  assert_equal(d.instance.instance_variable_get(k), v)
40
49
  }
50
+ {:@chunk_keys => ['tag'], :@flush_interval => 300, :@chunk_limit_size => Fluent::Plugin::TreasureDataLogOutput::IMPORT_SIZE_LIMIT}.each { |k, v|
51
+ assert_equal(d.instance.buffer.instance_variable_get(k), v)
52
+ }
53
+ end
54
+
55
+ def test_configure_for_chunk_key_tag
56
+ assert_raise Fluent::ConfigError.new("'tag' must be included in <buffer ARG> when database and table are not specified") do
57
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::TreasureDataLogOutput).configure(%[
58
+ apikey testkey
59
+ <buffer []>
60
+ flush_interval 10s
61
+ path #{TMP_DIR}/buffer
62
+ </buffer>
63
+ ])
64
+ end
41
65
  end
42
66
 
43
- def test_emit
67
+ data('evet_time' => 'event_time', 'int_time' => 'int')
68
+ def test_emit(time_class)
44
69
  d = create_driver
45
- time, records = stub_seed_values
70
+ time, records = stub_seed_values(time_class)
46
71
  database, table = d.instance.instance_variable_get(:@key).split(".", 2)
47
72
  stub_td_table_create_request(database, table)
48
73
  stub_td_import_request(stub_request_body(records, time), database, table)
@@ -51,10 +76,11 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
51
76
  # We need actual gzipped content to verify compressed body is correct or not.
52
77
  dont_allow(d.instance).gzip_by_command(anything, is_a(Tempfile))
53
78
 
54
- records.each { |record|
55
- d.emit(record, time)
79
+ d.run(default_tag: 'test') {
80
+ records.each { |record|
81
+ d.feed(time, record)
82
+ }
56
83
  }
57
- d.run
58
84
  }
59
85
 
60
86
  assert_equal('TD1 testkey', @auth_header)
@@ -69,11 +95,11 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
69
95
  assert_rr {
70
96
  # same as test_emit
71
97
  dont_allow(d.instance).gzip_by_writer(anything, is_a(Tempfile))
72
-
73
- records.each { |record|
74
- d.emit(record, time)
98
+ d.run(default_tag: 'test') {
99
+ records.each { |record|
100
+ d.feed(time, record)
101
+ }
75
102
  }
76
- d.run
77
103
  }
78
104
 
79
105
  assert_equal('TD1 testkey', @auth_header)
@@ -88,14 +114,14 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
88
114
  stub_td_table_create_request(database, table)
89
115
  stub_td_import_request(stub_request_body(records, time), database, table)
90
116
 
91
- records.each { |record|
92
- d.emit(record, time)
117
+ d.run(default_tag: 'test') {
118
+ d.feed_to_plugin('test', Fluent::ArrayEventStream.new(records.map { |e| [time, e] }))
93
119
  }
94
- d.run
95
120
 
96
- assert !d.instance.log.logs.any? { |line|
97
- line =~ /undefined method/
98
- }, 'nil record should be skipped'
121
+ error_events = d.error_events(tag: 'test')
122
+ assert_equal 2, error_events.size
123
+ assert_equal nil, error_events[0][2]['record']
124
+ assert_equal "string", error_events[1][2]['record']
99
125
  end
100
126
 
101
127
  def test_emit_with_bigint_record
@@ -111,29 +137,11 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
111
137
  test_time, test_records = stub_seed_values
112
138
  test_records[1]['k'] = ['hogehoge' * 1000]
113
139
  test_records[1]['kk'] = n
114
- test_records.each { |record|
115
- d.emit(record, test_time)
116
- }
117
- d.run
118
- end
119
-
120
- def test_emit_with_event_time
121
- omit "EventTime is not implemented with current Fluentd version" unless Fluent.const_defined?('EventTime')
122
-
123
- event_time_klass = Fluent.const_get('EventTime')
124
-
125
- event_time = event_time_klass.now
126
- d = create_driver
127
- _time, records = stub_seed_values
128
- database, table = d.instance.instance_variable_get(:@key).split(".", 2)
129
- stub_td_table_create_request(database, table)
130
- stub_td_import_request(stub_request_body(records, event_time.to_i), database, table)
131
-
132
- _test_time, test_records = stub_seed_values
133
- test_records.each { |record|
134
- d.emit(record, event_time)
140
+ d.run(default_tag: 'test') {
141
+ test_records.each { |record|
142
+ d.feed(test_time, record)
143
+ }
135
144
  }
136
- d.run
137
145
  end
138
146
 
139
147
  def test_emit_with_time_symbole
@@ -143,11 +151,12 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
143
151
  stub_td_table_create_request(database, table)
144
152
  stub_td_import_request(stub_request_body(records, time), database, table)
145
153
 
146
- records.each { |record|
147
- record[:time] = Time.now.to_i # emit removes this :time key
148
- d.emit(record, time)
154
+ d.run(default_tag: 'test') {
155
+ records.each { |record|
156
+ record[:time] = Time.now.to_i # emit removes this :time key
157
+ d.feed(time, record)
158
+ }
149
159
  }
150
- d.run
151
160
 
152
161
  assert_equal('TD1 testkey', @auth_header)
153
162
  end
@@ -160,10 +169,11 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
160
169
  stub_td_table_create_request(database, table, opts)
161
170
  stub_td_import_request(stub_request_body(records, time), database, table, opts)
162
171
 
163
- records.each { |record|
164
- d.emit(record, time)
172
+ d.run(default_tag: 'test') {
173
+ records.each { |record|
174
+ d.feed(time, record)
175
+ }
165
176
  }
166
- d.run
167
177
  end
168
178
 
169
179
  def test_emit_with_too_many_keys
@@ -174,45 +184,53 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
174
184
  stub_td_table_create_request(database, table, opts)
175
185
  stub_td_import_request(stub_request_body([], time), database, table, opts)
176
186
 
177
- d.emit(create_too_many_keys_record, time)
178
- d.run
187
+ d.run(default_tag: 'test') {
188
+ d.feed(time, create_too_many_keys_record)
189
+ }
179
190
 
180
- assert_equal 0, d.emits.size
181
- assert d.instance.log.logs.select{ |line|
182
- line =~ /Too many number of keys/
183
- }.size == 1, "too many keys error is not logged"
191
+ assert_equal 0, d.events.size
192
+ assert_equal 1, d.error_events.size
184
193
  end
185
194
 
186
- # TODO: add normalized_msgpack / tag split test
187
-
188
- ## TODO invalid names are normalized
189
- # def test_invalid_name
190
- # d = create_driver
191
- # d.instance.start
192
- #
193
- # es = Fluent::OneEventStream.new(Time.now.to_i, {})
194
- # chain = Fluent::NullOutputChain.instance
195
- # assert_raise(RuntimeError) do
196
- # d.instance.emit("test.invalid-name", es, chain)
197
- # end
198
- # assert_raise(RuntimeError) do
199
- # d.instance.emit("empty", es, chain)
200
- # end
201
- # assert_raise(RuntimeError) do
202
- # d.instance.emit("", es, chain)
203
- # end
204
- # end
205
-
206
- ## TODO invalid data is ignored
207
- # def test_invalid_data
208
- # d = create_driver
209
- # d.instance.start
210
- #
211
- # es = Fluent::OneEventStream.new(Time.now.to_i, "invalid")
212
- # chain = Fluent::NullOutputChain.instance
213
- # assert_nothing_raised do
214
- # d.instance.emit("test.name", es, chain)
215
- # end
216
- # end
195
+ sub_test_case 'tag splitting for database and table' do
196
+ def create_driver(conf = %[auto_create_table true])
197
+ config = BASE_CONFIG + conf
198
+
199
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::TreasureDataLogOutput).configure(config)
200
+ end
201
+
202
+ data('evet_time' => 'event_time', 'int_time' => 'int')
203
+ def test_tag_split(time_class)
204
+ d = create_driver
205
+
206
+ time, records = stub_seed_values(time_class)
207
+ database = 'db1'
208
+ table = 'table1'
209
+ stub_td_table_create_request(database, table)
210
+ stub_td_import_request(stub_request_body(records, time), database, table)
211
+
212
+ d.run(default_tag: 'td.db1.table1') {
213
+ records.each { |record|
214
+ d.feed(time, record)
215
+ }
216
+ }
217
+ end
218
+
219
+ def test_tag_split_with_normalization
220
+ d = create_driver
221
+
222
+ time, records = stub_seed_values
223
+ database = 'db_'
224
+ table = 'tb_'
225
+ stub_td_table_create_request(database, table)
226
+ stub_td_import_request(stub_request_body(records, time), database, table)
227
+
228
+ d.run(default_tag: 'td.db.tb') {
229
+ records.each { |record|
230
+ d.feed(time, record)
231
+ }
232
+ }
233
+ end
234
+ end
217
235
  end
218
236
 
@@ -1,26 +1,31 @@
1
1
  require 'json'
2
2
  require 'msgpack'
3
- require 'fluent/test'
4
3
  require 'webmock/test_unit'
5
4
  require 'stringio'
6
5
  require 'td-client'
7
6
  require 'zlib'
8
7
  require 'test/unit/rr'
9
8
 
9
+ require 'fluent/test'
10
+ require 'fluent/test/helpers'
11
+
10
12
  def e(s)
11
13
  require 'cgi'
12
14
  CGI.escape(s.to_s)
13
15
  end
14
16
 
15
17
  class Test::Unit::TestCase
18
+ include Fluent::Test::Helpers
19
+
16
20
  def create_too_many_keys_record
17
21
  record = {}
18
22
  5012.times { |i| record["k#{i}"] = i }
19
23
  record
20
24
  end
21
25
 
22
- def stub_seed_values
23
- time = Time.parse("2014-01-01 00:00:00 UTC").to_i
26
+ def stub_seed_values(time_class = 'int')
27
+ time = event_time("2014-01-01 00:00:00 UTC")
28
+ time = time.to_i if time_class == 'int'
24
29
  records = [{"a" => 1}, {"a" => 2}]
25
30
  return time, records
26
31
  end
@@ -32,7 +37,7 @@ class Test::Unit::TestCase
32
37
 
33
38
  r = record.dup
34
39
  if time
35
- r['time'] = time
40
+ r['time'] = time.to_i
36
41
  end
37
42
  r.to_msgpack(out)
38
43
  }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-td
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.11.0.rc1
4
+ version: 1.0.0.rc1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Treasure Data, Inc.
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-01-23 00:00:00.000000000 Z
11
+ date: 2017-02-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -16,7 +16,7 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 0.12.0
19
+ version: 0.14.13
20
20
  - - "<"
21
21
  - !ruby/object:Gem::Version
22
22
  version: '2'
@@ -26,7 +26,7 @@ dependencies:
26
26
  requirements:
27
27
  - - ">="
28
28
  - !ruby/object:Gem::Version
29
- version: 0.12.0
29
+ version: 0.14.13
30
30
  - - "<"
31
31
  - !ruby/object:Gem::Version
32
32
  version: '2'