fluent-plugin-td 1.1.0 → 1.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f8e132d9df4dd5ed1344497a0680b12caa75c8d9b6361feef6d36e6d50eef0d2
4
- data.tar.gz: 475920dcb5e14528ce9e2c5d5baab44811fa1fcab9b613f22c8212e10e56cf4c
3
+ metadata.gz: a08410b0e8078a20d2150bacc0cde7cc14cfb70be600fbcc86309b5df1b7076c
4
+ data.tar.gz: 4aca6ca191fa4401f198b323cd0ae12c1616de67e640bc8111237af6eca8f976
5
5
  SHA512:
6
- metadata.gz: bf337f321840654a6ace12117b899234bfe9b144756d96f34986381e84eb8a9f4d6d2f970ae9d55a3b49ed4fce6bf3c9d514aaaaa6e44c441f567c75d3238b0a
7
- data.tar.gz: 74fc971d23dac8608901ff7bcc9add4c969f20bdf3a293a653f26ebc703e3b951479ac9e98d296f83e222ad3dc8fc0c6636e5a0f1533e4e2fa6397225b7cc79b
6
+ metadata.gz: a776e4f86f5793451a04e76a9db1eb746ffb762d53e3781f1f20c8187663dcd8c575ef9814207c546831bd6192674661c6c38041998928f98ec159bbcd40d85c
7
+ data.tar.gz: 03a5a89b24a49813255dbe340b84b9dcd3abba74373dbb655d2220f3803f6beac8dbe6e9211312750ef00f59cf0f0b7e794c68810321b00aab20d4124d1997e3
@@ -0,0 +1,51 @@
1
+ version: 2.1
2
+ orbs:
3
+ ruby: circleci/ruby@1.8
4
+ jobs:
5
+ test:
6
+ parameters:
7
+ ruby-version:
8
+ type: string
9
+ docker:
10
+ - image: cimg/ruby:<< parameters.ruby-version >>
11
+ steps:
12
+ - checkout
13
+ - ruby/install-deps
14
+ - run:
15
+ name: Run tests
16
+ command: bundle exec rake test
17
+ publish:
18
+ docker:
19
+ - image: cimg/ruby:2.4
20
+ steps:
21
+ - checkout
22
+ - ruby/install-deps
23
+ - run:
24
+ name: Publish gem
25
+ command: |
26
+ mkdir -p $HOME/.gem
27
+ touch $HOME/.gem/credentials
28
+ chmod 0600 $HOME/.gem/credentials
29
+ printf -- "---\n:rubygems_api_key: ${RUBYGEMS_AUTH_TOKEN}\n" > $HOME/.gem/credentials
30
+ gem build *.gemspec
31
+ gem push *.gem
32
+ workflows:
33
+ version: 2
34
+ test:
35
+ jobs:
36
+ - test:
37
+ matrix:
38
+ parameters:
39
+ ruby-version: [ "2.4", "2.5", "2.6", "2.7", "3.0", "3.1" ]
40
+ filters:
41
+ tags:
42
+ only: /v[0-9]+(\.[0-9]+)*/
43
+ - publish:
44
+ context: rubygems
45
+ requires:
46
+ - test
47
+ filters:
48
+ branches:
49
+ ignore: /.*/
50
+ tags:
51
+ only: /v[0-9]+(\.[0-9]+)*/
@@ -0,0 +1,6 @@
1
+ version: 2
2
+ updates:
3
+ - package-ecosystem: "bundler"
4
+ directory: "/"
5
+ schedule:
6
+ interval: "weekly"
@@ -0,0 +1,31 @@
1
+ name: "CodeQL"
2
+ on:
3
+ push:
4
+ branches: [ master ]
5
+ pull_request:
6
+ branches: [ master ]
7
+ schedule:
8
+ - cron: '37 9 * * 3'
9
+ jobs:
10
+ analyze:
11
+ name: Analyze
12
+ runs-on: ubuntu-latest
13
+ permissions:
14
+ actions: read
15
+ contents: read
16
+ security-events: write
17
+ strategy:
18
+ fail-fast: false
19
+ matrix:
20
+ language: [ 'ruby' ]
21
+ steps:
22
+ - name: Checkout repository
23
+ uses: actions/checkout@v3
24
+ - name: Initialize CodeQL
25
+ uses: github/codeql-action/init@v2
26
+ with:
27
+ languages: ${{ matrix.language }}
28
+ - name: Autobuild
29
+ uses: github/codeql-action/autobuild@v2
30
+ - name: Perform CodeQL Analysis
31
+ uses: github/codeql-action/analyze@v2
data/.gitignore CHANGED
@@ -1,2 +1,2 @@
1
- Gemfile.lock
2
1
  pkg/*
2
+ .idea
data/Gemfile CHANGED
@@ -1,3 +1,3 @@
1
- source "http://rubygems.org"
1
+ source "https://rubygems.org"
2
2
 
3
3
  gemspec
data/Gemfile.lock ADDED
@@ -0,0 +1,95 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ fluent-plugin-td (1.2.0)
5
+ fluentd (>= 0.14.13, < 2)
6
+ td-client (>= 1.0.8)
7
+
8
+ GEM
9
+ remote: https://rubygems.org/
10
+ specs:
11
+ addressable (2.8.0)
12
+ public_suffix (>= 2.0.2, < 5.0)
13
+ ast (2.4.2)
14
+ concurrent-ruby (1.1.10)
15
+ cool.io (1.7.1)
16
+ crack (0.4.5)
17
+ rexml
18
+ fluentd (1.14.6)
19
+ bundler
20
+ cool.io (>= 1.4.5, < 2.0.0)
21
+ http_parser.rb (>= 0.5.1, < 0.9.0)
22
+ msgpack (>= 1.3.1, < 2.0.0)
23
+ serverengine (>= 2.2.5, < 3.0.0)
24
+ sigdump (~> 0.2.2)
25
+ strptime (>= 0.2.4, < 1.0.0)
26
+ tzinfo (>= 1.0, < 3.0)
27
+ tzinfo-data (~> 1.0)
28
+ webrick (>= 1.4.2, < 1.8.0)
29
+ yajl-ruby (~> 1.0)
30
+ hashdiff (1.0.1)
31
+ http_parser.rb (0.8.0)
32
+ httpclient (2.8.3)
33
+ msgpack (1.5.1)
34
+ parallel (1.20.1)
35
+ parser (3.1.2.0)
36
+ ast (~> 2.4.1)
37
+ power_assert (2.0.1)
38
+ public_suffix (4.0.7)
39
+ rainbow (3.1.1)
40
+ rake (13.0.6)
41
+ regexp_parser (2.4.0)
42
+ rexml (3.2.5)
43
+ rr (3.0.9)
44
+ rubocop (1.12.1)
45
+ parallel (~> 1.10)
46
+ parser (>= 3.0.0.0)
47
+ rainbow (>= 2.2.2, < 4.0)
48
+ regexp_parser (>= 1.8, < 3.0)
49
+ rexml
50
+ rubocop-ast (>= 1.2.0, < 2.0)
51
+ ruby-progressbar (~> 1.7)
52
+ unicode-display_width (>= 1.4.0, < 3.0)
53
+ rubocop-ast (1.4.1)
54
+ parser (>= 2.7.1.5)
55
+ rubocop-rake (0.5.1)
56
+ rubocop
57
+ ruby-progressbar (1.11.0)
58
+ serverengine (2.2.5)
59
+ sigdump (~> 0.2.2)
60
+ sigdump (0.2.4)
61
+ strptime (0.2.5)
62
+ td-client (1.0.8)
63
+ httpclient (>= 2.7)
64
+ msgpack (>= 0.5.6, < 2)
65
+ test-unit (3.5.3)
66
+ power_assert
67
+ test-unit-rr (1.0.5)
68
+ rr (>= 1.1.1)
69
+ test-unit (>= 2.5.2)
70
+ tzinfo (2.0.4)
71
+ concurrent-ruby (~> 1.0)
72
+ tzinfo-data (1.2022.1)
73
+ tzinfo (>= 1.0.0)
74
+ unicode-display_width (2.1.0)
75
+ webmock (3.14.0)
76
+ addressable (>= 2.8.0)
77
+ crack (>= 0.3.2)
78
+ hashdiff (>= 0.4.0, < 2.0.0)
79
+ webrick (1.7.0)
80
+ yajl-ruby (1.4.1)
81
+
82
+ PLATFORMS
83
+ ruby
84
+
85
+ DEPENDENCIES
86
+ fluent-plugin-td!
87
+ rake (>= 0.9.2)
88
+ rubocop
89
+ rubocop-rake
90
+ test-unit (~> 3.5.3)
91
+ test-unit-rr (~> 1.0.5)
92
+ webmock (~> 3.14.0)
93
+
94
+ BUNDLED WITH
95
+ 2.1.4
@@ -1,26 +1,26 @@
1
- # encoding: utf-8
2
- $:.push File.expand_path('../lib', __FILE__)
3
- require 'fluent/plugin/td_plugin_version'
1
+ require File.expand_path('../lib/fluent/plugin/td_plugin_version', __FILE__)
4
2
 
5
3
  Gem::Specification.new do |gem|
6
- gem.name = "fluent-plugin-td"
7
- gem.description = "Treasure Data Cloud Data Service plugin for Fluentd"
8
- gem.homepage = "http://www.treasuredata.com/"
4
+ gem.name = 'fluent-plugin-td'
5
+ gem.description = 'Treasure Data Cloud Data Service plugin for Fluentd'
6
+ gem.homepage = 'https://www.treasuredata.com/'
9
7
  gem.summary = gem.description
10
8
  gem.version = Fluent::Plugin::TreasureDataPlugin::VERSION
11
- gem.authors = ["Treasure Data, Inc."]
12
- gem.email = "support@treasure-data.com"
13
- #gem.platform = Gem::Platform::RUBY
9
+ gem.authors = ['Treasure Data, Inc.']
10
+ gem.email = 'support@treasure-data.com'
14
11
  gem.files = `git ls-files`.split("\n")
15
12
  gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
16
- gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
13
+ gem.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
17
14
  gem.require_paths = ['lib']
18
- gem.license = "Apache-2.0"
15
+ gem.license = 'Apache-2.0'
19
16
 
20
- gem.add_dependency "fluentd", [">= 0.14.13", "< 2"]
21
- gem.add_dependency "td-client", "~> 1.0"
22
- gem.add_development_dependency "rake", ">= 0.9.2"
23
- gem.add_development_dependency "webmock", "~> 1.16"
24
- gem.add_development_dependency "test-unit", "~> 3.0.8"
25
- gem.add_development_dependency "test-unit-rr", "~> 1.0.3"
17
+ gem.required_ruby_version = '>= 2.4'
18
+ gem.add_dependency 'fluentd', ['>= 0.14.13', '< 2']
19
+ gem.add_dependency 'td-client', '>= 1.0.8'
20
+ gem.add_development_dependency 'rake', '>= 0.9.2'
21
+ gem.add_development_dependency 'rubocop'
22
+ gem.add_development_dependency 'rubocop-rake'
23
+ gem.add_development_dependency 'test-unit', '~> 3.5.3'
24
+ gem.add_development_dependency 'test-unit-rr', '~> 1.0.5'
25
+ gem.add_development_dependency 'webmock', '~> 3.14.0'
26
26
  end
@@ -4,6 +4,7 @@ require 'zlib'
4
4
  require 'stringio'
5
5
  require 'td-client'
6
6
 
7
+ require 'fluent/error'
7
8
  require 'fluent/plugin/output'
8
9
  require 'fluent/plugin/td_plugin_version'
9
10
 
@@ -12,6 +13,7 @@ module Fluent::Plugin
12
13
  Fluent::Plugin.register_output('tdlog', self)
13
14
 
14
15
  IMPORT_SIZE_LIMIT = 32 * 1024 * 1024
16
+ IMPORT_RECORDS_LIMIT = 8096
15
17
  UPLOAD_EXT = 'msgpack.gz'.freeze
16
18
 
17
19
  helpers :event_emitter, :compat_parameters
@@ -36,13 +38,14 @@ module Fluent::Plugin
36
38
  config_set_default :chunk_keys, ['tag']
37
39
  config_set_default :flush_interval, 300
38
40
  config_set_default :chunk_limit_size, IMPORT_SIZE_LIMIT
41
+ config_set_default :chunk_limit_records, IMPORT_RECORDS_LIMIT
39
42
  end
40
43
 
41
44
  def initialize
42
45
  super
43
46
  @key = nil
44
- @key_num_limit = 512 # TODO: Our one-time import has the restriction about the number of record keys.
45
- @record_size_limit = 32 * 1024 * 1024 # TODO
47
+ @key_num_limit = 512
48
+ @record_size_limit = 32 * 1024 * 1024
46
49
  @table_list = {}
47
50
  @empty_gz_data = TreasureData::API.create_empty_gz_data
48
51
  @user_agent = "fluent-plugin-td: #{TreasureDataPlugin::VERSION}".freeze
@@ -107,7 +110,6 @@ module Fluent::Plugin
107
110
  record.delete(:time) if record.has_key?(:time)
108
111
 
109
112
  if record.size > @key_num_limit
110
- # TODO include summary of the record
111
113
  router.emit_error_event(tag, time, record, RuntimeError.new("too many number of keys (#{record.size} keys)"))
112
114
  return nil
113
115
  end
@@ -168,7 +170,6 @@ module Fluent::Plugin
168
170
  f.close(true) if f
169
171
  end
170
172
 
171
- # TODO: Share this routine with s3 compressors
172
173
  def gzip_by_command(chunk, tmp)
173
174
  chunk_is_file = @buffer_config['@type'] == 'file'
174
175
  path = if chunk_is_file
@@ -183,10 +184,8 @@ module Fluent::Plugin
183
184
  res = system "gzip -c #{path} > #{tmp.path}"
184
185
  unless res
185
186
  log.warn "failed to execute gzip command. Fallback to GzipWriter. status = #{$?}"
186
- begin
187
- tmp.truncate(0)
188
- return gzip_by_writer(chunk, tmp)
189
- end
187
+ tmp.truncate(0)
188
+ return gzip_by_writer(chunk, tmp)
190
189
  end
191
190
  File.size(tmp.path)
192
191
  ensure
@@ -211,18 +210,22 @@ module Fluent::Plugin
211
210
  unique_str = unique_id.unpack('C*').map { |x| "%02x" % x }.join
212
211
  log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }
213
212
 
213
+ start = Time.now
214
214
  begin
215
215
  begin
216
- start = Time.now
217
216
  @client.import(database, table, UPLOAD_EXT, io, size, unique_str)
218
- rescue TreasureData::NotFoundError => e
217
+ rescue TreasureData::NotFoundError
219
218
  unless @auto_create_table
220
- raise e
219
+ raise
221
220
  end
222
221
  ensure_database_and_table(database, table)
223
222
  io.pos = 0
224
223
  retry
225
224
  end
225
+ rescue TreasureData::TooManyRequestsError
226
+ raise
227
+ rescue TreasureData::ClientError => e
228
+ raise Fluent::UnrecoverableError.new(e.message)
226
229
  rescue => e
227
230
  elapsed = Time.now - start
228
231
  ne = RuntimeError.new("Failed to upload to Treasure Data '#{database}.#{table}' table: #{e.inspect} (#{size} bytes; #{elapsed} seconds)")
@@ -269,6 +272,7 @@ module Fluent::Plugin
269
272
  @api_client.create_database(database)
270
273
  @api_client.create_log_table(database, table)
271
274
  rescue TreasureData::AlreadyExistsError
275
+ # ignored
272
276
  end
273
277
  end
274
278
  end
@@ -1,7 +1,7 @@
1
1
  module Fluent
2
2
  module Plugin
3
3
  module TreasureDataPlugin
4
- VERSION = '1.1.0'
4
+ VERSION = '1.2.0'
5
5
  end
6
6
  end
7
7
  end
@@ -1,7 +1,7 @@
1
1
  require 'fluent/test'
2
2
  require 'fluent/test/driver/output'
3
3
  require 'fluent/plugin/out_tdlog'
4
- require 'test_helper.rb'
4
+ require 'test_helper'
5
5
 
6
6
  class TreasureDataLogOutputTest < Test::Unit::TestCase
7
7
  TMP_DIR = File.dirname(__FILE__) + "/tmp"
@@ -144,7 +144,7 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
144
144
  }
145
145
  end
146
146
 
147
- def test_emit_with_time_symbole
147
+ def test_emit_with_time_symbol
148
148
  d = create_driver
149
149
  time, records = stub_seed_values
150
150
  database, table = d.instance.instance_variable_get(:@key).split(".", 2)
@@ -190,6 +190,38 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
190
190
  assert_equal 1, d.error_events.size
191
191
  end
192
192
 
193
+ def test_write_with_client_error
194
+ d = create_driver(DEFAULT_CONFIG)
195
+ time, records = stub_seed_values
196
+ database, table = d.instance.instance_variable_get(:@key).split(".", 2)
197
+ stub_td_table_create_request(database, table)
198
+ stub_td_import_request(stub_request_body(records, time), database, table, status: 400)
199
+
200
+ assert_nothing_raised(Fluent::UnrecoverableError) do
201
+ d.run(default_tag: 'test') {
202
+ records.each { |record|
203
+ d.feed(time, record)
204
+ }
205
+ }
206
+ end
207
+ end
208
+
209
+ def test_write_retry_if_too_many_requests
210
+ d = create_driver(DEFAULT_CONFIG)
211
+ time, records = stub_seed_values
212
+ database, table = d.instance.instance_variable_get(:@key).split(".", 2)
213
+ stub_td_table_create_request(database, table)
214
+ stub_td_import_request(stub_request_body(records, time), database, table, status: 429)
215
+
216
+ assert_raise(TreasureData::TooManyRequestsError) do
217
+ d.run(default_tag: 'test') {
218
+ records.each { |record|
219
+ d.feed(time, record)
220
+ }
221
+ }
222
+ end
223
+ end
224
+
193
225
  sub_test_case 'tag splitting for database and table' do
194
226
  def create_driver(conf = %[auto_create_table true])
195
227
  config = BASE_CONFIG + conf
@@ -197,7 +229,7 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
197
229
  Fluent::Test::Driver::Output.new(Fluent::Plugin::TreasureDataLogOutput).configure(config)
198
230
  end
199
231
 
200
- data('evet_time' => 'event_time', 'int_time' => 'int')
232
+ data('event_time' => 'event_time', 'int_time' => 'int')
201
233
  def test_tag_split(time_class)
202
234
  d = create_driver
203
235
 
data/test/test_helper.rb CHANGED
@@ -69,6 +69,7 @@ class Test::Unit::TestCase
69
69
  opts[:use_ssl] = true unless opts.has_key?(:use_ssl)
70
70
  format = opts[:format] || 'msgpack.gz'
71
71
  schema = opts[:use_ssl] ? 'https' : 'http'
72
+ status = opts[:status] || 200
72
73
  response = {"database" => db, "table" => table, "elapsed_time" => 0}.to_json
73
74
  endpoint = opts[:endpoint] ? opts[:endpoint] : TreasureData::API::DEFAULT_IMPORT_ENDPOINT
74
75
 
@@ -80,6 +81,6 @@ class Test::Unit::TestCase
80
81
  stub_request(:put, url_with_unique).with(:headers => {'Content-Type' => 'application/octet-stream'}) { |req|
81
82
  @auth_header = req.headers["Authorization"]
82
83
  stub_gzip_unwrap(req.body) == stub_gzip_unwrap(body)
83
- }.to_return(:status => 200, :body => response)
84
+ }.to_return(:status => status, :body => response)
84
85
  end
85
86
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-td
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.0
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Treasure Data, Inc.
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-12-18 00:00:00.000000000 Z
11
+ date: 2022-05-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -34,16 +34,16 @@ dependencies:
34
34
  name: td-client
35
35
  requirement: !ruby/object:Gem::Requirement
36
36
  requirements:
37
- - - "~>"
37
+ - - ">="
38
38
  - !ruby/object:Gem::Version
39
- version: '1.0'
39
+ version: 1.0.8
40
40
  type: :runtime
41
41
  prerelease: false
42
42
  version_requirements: !ruby/object:Gem::Requirement
43
43
  requirements:
44
- - - "~>"
44
+ - - ">="
45
45
  - !ruby/object:Gem::Version
46
- version: '1.0'
46
+ version: 1.0.8
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: rake
49
49
  requirement: !ruby/object:Gem::Requirement
@@ -59,70 +59,98 @@ dependencies:
59
59
  - !ruby/object:Gem::Version
60
60
  version: 0.9.2
61
61
  - !ruby/object:Gem::Dependency
62
- name: webmock
62
+ name: rubocop
63
63
  requirement: !ruby/object:Gem::Requirement
64
64
  requirements:
65
- - - "~>"
65
+ - - ">="
66
66
  - !ruby/object:Gem::Version
67
- version: '1.16'
67
+ version: '0'
68
68
  type: :development
69
69
  prerelease: false
70
70
  version_requirements: !ruby/object:Gem::Requirement
71
71
  requirements:
72
- - - "~>"
72
+ - - ">="
73
73
  - !ruby/object:Gem::Version
74
- version: '1.16'
74
+ version: '0'
75
+ - !ruby/object:Gem::Dependency
76
+ name: rubocop-rake
77
+ requirement: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - ">="
80
+ - !ruby/object:Gem::Version
81
+ version: '0'
82
+ type: :development
83
+ prerelease: false
84
+ version_requirements: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - ">="
87
+ - !ruby/object:Gem::Version
88
+ version: '0'
75
89
  - !ruby/object:Gem::Dependency
76
90
  name: test-unit
77
91
  requirement: !ruby/object:Gem::Requirement
78
92
  requirements:
79
93
  - - "~>"
80
94
  - !ruby/object:Gem::Version
81
- version: 3.0.8
95
+ version: 3.5.3
82
96
  type: :development
83
97
  prerelease: false
84
98
  version_requirements: !ruby/object:Gem::Requirement
85
99
  requirements:
86
100
  - - "~>"
87
101
  - !ruby/object:Gem::Version
88
- version: 3.0.8
102
+ version: 3.5.3
89
103
  - !ruby/object:Gem::Dependency
90
104
  name: test-unit-rr
91
105
  requirement: !ruby/object:Gem::Requirement
92
106
  requirements:
93
107
  - - "~>"
94
108
  - !ruby/object:Gem::Version
95
- version: 1.0.3
109
+ version: 1.0.5
96
110
  type: :development
97
111
  prerelease: false
98
112
  version_requirements: !ruby/object:Gem::Requirement
99
113
  requirements:
100
114
  - - "~>"
101
115
  - !ruby/object:Gem::Version
102
- version: 1.0.3
116
+ version: 1.0.5
117
+ - !ruby/object:Gem::Dependency
118
+ name: webmock
119
+ requirement: !ruby/object:Gem::Requirement
120
+ requirements:
121
+ - - "~>"
122
+ - !ruby/object:Gem::Version
123
+ version: 3.14.0
124
+ type: :development
125
+ prerelease: false
126
+ version_requirements: !ruby/object:Gem::Requirement
127
+ requirements:
128
+ - - "~>"
129
+ - !ruby/object:Gem::Version
130
+ version: 3.14.0
103
131
  description: Treasure Data Cloud Data Service plugin for Fluentd
104
132
  email: support@treasure-data.com
105
133
  executables: []
106
134
  extensions: []
107
135
  extra_rdoc_files: []
108
136
  files:
137
+ - ".circleci/config.yml"
138
+ - ".github/dependabot.yml"
139
+ - ".github/workflows/codeql-analysis.yml"
109
140
  - ".gitignore"
110
- - ".travis.yml"
111
141
  - AUTHORS
112
142
  - ChangeLog
113
143
  - Gemfile
114
- - Gemfile.msgpack.0.4
115
- - Gemfile.v0.12
144
+ - Gemfile.lock
116
145
  - README.rdoc
117
146
  - Rakefile
118
147
  - example.conf
119
148
  - fluent-plugin-td.gemspec
120
149
  - lib/fluent/plugin/out_tdlog.rb
121
- - lib/fluent/plugin/td_plugin_util.rb
122
150
  - lib/fluent/plugin/td_plugin_version.rb
123
151
  - test/plugin/test_out_tdlog.rb
124
152
  - test/test_helper.rb
125
- homepage: http://www.treasuredata.com/
153
+ homepage: https://www.treasuredata.com/
126
154
  licenses:
127
155
  - Apache-2.0
128
156
  metadata: {}
@@ -134,17 +162,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
134
162
  requirements:
135
163
  - - ">="
136
164
  - !ruby/object:Gem::Version
137
- version: '0'
165
+ version: '2.4'
138
166
  required_rubygems_version: !ruby/object:Gem::Requirement
139
167
  requirements:
140
168
  - - ">="
141
169
  - !ruby/object:Gem::Version
142
170
  version: '0'
143
171
  requirements: []
144
- rubygems_version: 3.0.3
172
+ rubygems_version: 3.1.2
145
173
  signing_key:
146
174
  specification_version: 4
147
175
  summary: Treasure Data Cloud Data Service plugin for Fluentd
148
- test_files:
149
- - test/plugin/test_out_tdlog.rb
150
- - test/test_helper.rb
176
+ test_files: []
data/.travis.yml DELETED
@@ -1,18 +0,0 @@
1
- rvm:
2
- - 2.1
3
- - 2.2.4
4
- - 2.3.1
5
- - 2.4.0
6
- - ruby-head
7
-
8
- gemfile:
9
- - Gemfile
10
-
11
- before_install: gem update bundler
12
- script: bundle exec rake test
13
-
14
- sudo: false
15
-
16
- matrix:
17
- allow_failures:
18
- - rvm: ruby-head
data/Gemfile.msgpack.0.4 DELETED
@@ -1,5 +0,0 @@
1
- source 'https://rubygems.org/'
2
-
3
- gem 'msgpack', '= 0.4.7'
4
- gem 'fluentd', '~> 0.10.55'
5
- gemspec
data/Gemfile.v0.12 DELETED
@@ -1,6 +0,0 @@
1
- source "http://rubygems.org"
2
-
3
- gem 'json', '= 1.8.6'
4
- gem 'fluentd', '~> 0.12.0'
5
-
6
- gemspec
@@ -1,107 +0,0 @@
1
- module Fluent
2
- module TDPluginUtil
3
- require 'fileutils'
4
- require 'stringio'
5
- require 'tempfile'
6
- require 'zlib'
7
- require 'td-client'
8
-
9
- def validate_database_and_table_name(database, table, conf)
10
- begin
11
- TreasureData::API.validate_database_name(database)
12
- rescue => e
13
- raise ConfigError, "Invalid database name #{database.inspect}: #{e}: #{conf}"
14
- end
15
- begin
16
- TreasureData::API.validate_table_name(table)
17
- rescue => e
18
- raise ConfigError, "Invalid table name #{table.inspect}: #{e}: #{conf}"
19
- end
20
- end
21
-
22
- def parse_bool_parameter(param)
23
- if param.empty?
24
- true
25
- else
26
- param = Config.bool_value(param)
27
- raise ConfigError, "'true' or 'false' is required for #{key} option on tdlog output" if param.nil?
28
- param
29
- end
30
- end
31
-
32
- def summarize_record(record)
33
- json = record.to_json
34
- if json.size > 100
35
- json[0..97] + "..."
36
- else
37
- json
38
- end
39
- end
40
-
41
- def check_table_existence(database, table)
42
- @table_list ||= {}
43
- key = "#{database}.#{table}"
44
- unless @table_list.has_key?(key)
45
- log.debug "checking whether table '#{key}' exists on Treasure Data"
46
- io = StringIO.new(@empty_gz_data)
47
- begin
48
- # here doesn't check whether target table is item table or not because import-only user can't read the table status.
49
- # So I use empty import request to check table existence.
50
- @client.import(database, table, "msgpack.gz", io, io.size)
51
- @table_list[key] = true
52
- rescue TreasureData::NotFoundError
53
- args = self.class == TreasureDataItemOutput ? ' -t item' : ''
54
- raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td table:create #{database} #{table}#{args}' to create it."
55
- rescue => e
56
- log.warn "failed to check table existence on Treasure Data", :error => e.inspect
57
- log.debug_backtrace e
58
- end
59
- end
60
- end
61
-
62
- def write(chunk)
63
- unique_id = chunk.unique_id
64
- database, table = chunk.key.split('.', 2)
65
-
66
- FileUtils.mkdir_p(@tmpdir) unless @tmpdir.nil?
67
- f = Tempfile.new(@tmpdir_prefix, @tmpdir)
68
- w = Zlib::GzipWriter.new(f)
69
-
70
- chunk.write_to(w)
71
- w.finish
72
- w = nil
73
-
74
- size = f.pos
75
- f.pos = 0
76
- upload(database, table, f, size, unique_id)
77
- ensure
78
- w.close if w
79
- f.close if f
80
- end
81
-
82
- # assume @client and @auto_create_table variable exist
83
- def upload(database, table, io, size, unique_id)
84
- unique_str = unique_id.unpack('C*').map {|x| "%02x" % x }.join
85
- log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }
86
-
87
- begin
88
- begin
89
- start = Time.now
90
- @client.import(database, table, "msgpack.gz", io, size, unique_str)
91
- rescue TreasureData::NotFoundError => e
92
- unless @auto_create_table
93
- raise e
94
- end
95
- ensure_database_and_table(database, table)
96
- io.pos = 0
97
- retry
98
- end
99
- rescue => e
100
- elapsed = Time.now - start
101
- ne = RuntimeError.new("Failed to upload to TreasureData: #{e} (#{size} bytes; #{elapsed} seconds)")
102
- ne.set_backtrace(e.backtrace)
103
- raise ne
104
- end
105
- end
106
- end
107
- end