fluent-plugin-td 1.0.0.rc1 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.circleci/config.yml +51 -0
- data/.github/dependabot.yml +6 -0
- data/.github/workflows/codeql-analysis.yml +31 -0
- data/.gitignore +1 -1
- data/ChangeLog +7 -1
- data/Gemfile +1 -1
- data/Gemfile.lock +95 -0
- data/README.rdoc +11 -4
- data/fluent-plugin-td.gemspec +17 -18
- data/lib/fluent/plugin/out_tdlog.rb +23 -18
- data/lib/fluent/plugin/td_plugin_version.rb +1 -1
- data/test/plugin/test_out_tdlog.rb +42 -12
- data/test/test_helper.rb +4 -3
- metadata +53 -28
- data/.travis.yml +0 -18
- data/Gemfile.msgpack.0.4 +0 -5
- data/Gemfile.v0.12 +0 -6
- data/lib/fluent/plugin/td_plugin_util.rb +0 -107
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: a08410b0e8078a20d2150bacc0cde7cc14cfb70be600fbcc86309b5df1b7076c
|
|
4
|
+
data.tar.gz: 4aca6ca191fa4401f198b323cd0ae12c1616de67e640bc8111237af6eca8f976
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a776e4f86f5793451a04e76a9db1eb746ffb762d53e3781f1f20c8187663dcd8c575ef9814207c546831bd6192674661c6c38041998928f98ec159bbcd40d85c
|
|
7
|
+
data.tar.gz: 03a5a89b24a49813255dbe340b84b9dcd3abba74373dbb655d2220f3803f6beac8dbe6e9211312750ef00f59cf0f0b7e794c68810321b00aab20d4124d1997e3
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
version: 2.1
|
|
2
|
+
orbs:
|
|
3
|
+
ruby: circleci/ruby@1.8
|
|
4
|
+
jobs:
|
|
5
|
+
test:
|
|
6
|
+
parameters:
|
|
7
|
+
ruby-version:
|
|
8
|
+
type: string
|
|
9
|
+
docker:
|
|
10
|
+
- image: cimg/ruby:<< parameters.ruby-version >>
|
|
11
|
+
steps:
|
|
12
|
+
- checkout
|
|
13
|
+
- ruby/install-deps
|
|
14
|
+
- run:
|
|
15
|
+
name: Run tests
|
|
16
|
+
command: bundle exec rake test
|
|
17
|
+
publish:
|
|
18
|
+
docker:
|
|
19
|
+
- image: cimg/ruby:2.4
|
|
20
|
+
steps:
|
|
21
|
+
- checkout
|
|
22
|
+
- ruby/install-deps
|
|
23
|
+
- run:
|
|
24
|
+
name: Publish gem
|
|
25
|
+
command: |
|
|
26
|
+
mkdir -p $HOME/.gem
|
|
27
|
+
touch $HOME/.gem/credentials
|
|
28
|
+
chmod 0600 $HOME/.gem/credentials
|
|
29
|
+
printf -- "---\n:rubygems_api_key: ${RUBYGEMS_AUTH_TOKEN}\n" > $HOME/.gem/credentials
|
|
30
|
+
gem build *.gemspec
|
|
31
|
+
gem push *.gem
|
|
32
|
+
workflows:
|
|
33
|
+
version: 2
|
|
34
|
+
test:
|
|
35
|
+
jobs:
|
|
36
|
+
- test:
|
|
37
|
+
matrix:
|
|
38
|
+
parameters:
|
|
39
|
+
ruby-version: [ "2.4", "2.5", "2.6", "2.7", "3.0", "3.1" ]
|
|
40
|
+
filters:
|
|
41
|
+
tags:
|
|
42
|
+
only: /v[0-9]+(\.[0-9]+)*/
|
|
43
|
+
- publish:
|
|
44
|
+
context: rubygems
|
|
45
|
+
requires:
|
|
46
|
+
- test
|
|
47
|
+
filters:
|
|
48
|
+
branches:
|
|
49
|
+
ignore: /.*/
|
|
50
|
+
tags:
|
|
51
|
+
only: /v[0-9]+(\.[0-9]+)*/
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
name: "CodeQL"
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
branches: [ master ]
|
|
5
|
+
pull_request:
|
|
6
|
+
branches: [ master ]
|
|
7
|
+
schedule:
|
|
8
|
+
- cron: '37 9 * * 3'
|
|
9
|
+
jobs:
|
|
10
|
+
analyze:
|
|
11
|
+
name: Analyze
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
permissions:
|
|
14
|
+
actions: read
|
|
15
|
+
contents: read
|
|
16
|
+
security-events: write
|
|
17
|
+
strategy:
|
|
18
|
+
fail-fast: false
|
|
19
|
+
matrix:
|
|
20
|
+
language: [ 'ruby' ]
|
|
21
|
+
steps:
|
|
22
|
+
- name: Checkout repository
|
|
23
|
+
uses: actions/checkout@v3
|
|
24
|
+
- name: Initialize CodeQL
|
|
25
|
+
uses: github/codeql-action/init@v2
|
|
26
|
+
with:
|
|
27
|
+
languages: ${{ matrix.language }}
|
|
28
|
+
- name: Autobuild
|
|
29
|
+
uses: github/codeql-action/autobuild@v2
|
|
30
|
+
- name: Perform CodeQL Analysis
|
|
31
|
+
uses: github/codeql-action/analyze@v2
|
data/.gitignore
CHANGED
data/ChangeLog
CHANGED
|
@@ -1,4 +1,10 @@
|
|
|
1
|
-
Release 1.
|
|
1
|
+
Release 1.1.0 - 2019/12/18
|
|
2
|
+
|
|
3
|
+
* Add api_endpoint and import_endpoint to use proper endpoint for each request.
|
|
4
|
+
endpoint is now alias of import_endpoint
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
Release 1.0.0 - 2017/11/26
|
|
2
8
|
|
|
3
9
|
* Use new Plugin API
|
|
4
10
|
* Update fluentd dependency to v0.14.13 or later
|
data/Gemfile
CHANGED
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
fluent-plugin-td (1.2.0)
|
|
5
|
+
fluentd (>= 0.14.13, < 2)
|
|
6
|
+
td-client (>= 1.0.8)
|
|
7
|
+
|
|
8
|
+
GEM
|
|
9
|
+
remote: https://rubygems.org/
|
|
10
|
+
specs:
|
|
11
|
+
addressable (2.8.0)
|
|
12
|
+
public_suffix (>= 2.0.2, < 5.0)
|
|
13
|
+
ast (2.4.2)
|
|
14
|
+
concurrent-ruby (1.1.10)
|
|
15
|
+
cool.io (1.7.1)
|
|
16
|
+
crack (0.4.5)
|
|
17
|
+
rexml
|
|
18
|
+
fluentd (1.14.6)
|
|
19
|
+
bundler
|
|
20
|
+
cool.io (>= 1.4.5, < 2.0.0)
|
|
21
|
+
http_parser.rb (>= 0.5.1, < 0.9.0)
|
|
22
|
+
msgpack (>= 1.3.1, < 2.0.0)
|
|
23
|
+
serverengine (>= 2.2.5, < 3.0.0)
|
|
24
|
+
sigdump (~> 0.2.2)
|
|
25
|
+
strptime (>= 0.2.4, < 1.0.0)
|
|
26
|
+
tzinfo (>= 1.0, < 3.0)
|
|
27
|
+
tzinfo-data (~> 1.0)
|
|
28
|
+
webrick (>= 1.4.2, < 1.8.0)
|
|
29
|
+
yajl-ruby (~> 1.0)
|
|
30
|
+
hashdiff (1.0.1)
|
|
31
|
+
http_parser.rb (0.8.0)
|
|
32
|
+
httpclient (2.8.3)
|
|
33
|
+
msgpack (1.5.1)
|
|
34
|
+
parallel (1.20.1)
|
|
35
|
+
parser (3.1.2.0)
|
|
36
|
+
ast (~> 2.4.1)
|
|
37
|
+
power_assert (2.0.1)
|
|
38
|
+
public_suffix (4.0.7)
|
|
39
|
+
rainbow (3.1.1)
|
|
40
|
+
rake (13.0.6)
|
|
41
|
+
regexp_parser (2.4.0)
|
|
42
|
+
rexml (3.2.5)
|
|
43
|
+
rr (3.0.9)
|
|
44
|
+
rubocop (1.12.1)
|
|
45
|
+
parallel (~> 1.10)
|
|
46
|
+
parser (>= 3.0.0.0)
|
|
47
|
+
rainbow (>= 2.2.2, < 4.0)
|
|
48
|
+
regexp_parser (>= 1.8, < 3.0)
|
|
49
|
+
rexml
|
|
50
|
+
rubocop-ast (>= 1.2.0, < 2.0)
|
|
51
|
+
ruby-progressbar (~> 1.7)
|
|
52
|
+
unicode-display_width (>= 1.4.0, < 3.0)
|
|
53
|
+
rubocop-ast (1.4.1)
|
|
54
|
+
parser (>= 2.7.1.5)
|
|
55
|
+
rubocop-rake (0.5.1)
|
|
56
|
+
rubocop
|
|
57
|
+
ruby-progressbar (1.11.0)
|
|
58
|
+
serverengine (2.2.5)
|
|
59
|
+
sigdump (~> 0.2.2)
|
|
60
|
+
sigdump (0.2.4)
|
|
61
|
+
strptime (0.2.5)
|
|
62
|
+
td-client (1.0.8)
|
|
63
|
+
httpclient (>= 2.7)
|
|
64
|
+
msgpack (>= 0.5.6, < 2)
|
|
65
|
+
test-unit (3.5.3)
|
|
66
|
+
power_assert
|
|
67
|
+
test-unit-rr (1.0.5)
|
|
68
|
+
rr (>= 1.1.1)
|
|
69
|
+
test-unit (>= 2.5.2)
|
|
70
|
+
tzinfo (2.0.4)
|
|
71
|
+
concurrent-ruby (~> 1.0)
|
|
72
|
+
tzinfo-data (1.2022.1)
|
|
73
|
+
tzinfo (>= 1.0.0)
|
|
74
|
+
unicode-display_width (2.1.0)
|
|
75
|
+
webmock (3.14.0)
|
|
76
|
+
addressable (>= 2.8.0)
|
|
77
|
+
crack (>= 0.3.2)
|
|
78
|
+
hashdiff (>= 0.4.0, < 2.0.0)
|
|
79
|
+
webrick (1.7.0)
|
|
80
|
+
yajl-ruby (1.4.1)
|
|
81
|
+
|
|
82
|
+
PLATFORMS
|
|
83
|
+
ruby
|
|
84
|
+
|
|
85
|
+
DEPENDENCIES
|
|
86
|
+
fluent-plugin-td!
|
|
87
|
+
rake (>= 0.9.2)
|
|
88
|
+
rubocop
|
|
89
|
+
rubocop-rake
|
|
90
|
+
test-unit (~> 3.5.3)
|
|
91
|
+
test-unit-rr (~> 1.0.5)
|
|
92
|
+
webmock (~> 3.14.0)
|
|
93
|
+
|
|
94
|
+
BUNDLED WITH
|
|
95
|
+
2.1.4
|
data/README.rdoc
CHANGED
|
@@ -5,7 +5,7 @@ Treasure Data's REST APIs.
|
|
|
5
5
|
|
|
6
6
|
This plugin relies on the Treasure Data's {Ruby client library}[https://github.com/treasure-data/td-client-ruby] to communicate with the Treasure Data's REST APIs.
|
|
7
7
|
|
|
8
|
-
For more information, please visit the official {Fluentd TD Output plugin}[
|
|
8
|
+
For more information, please visit the official {Fluentd TD Output plugin}[https://docs.fluentd.org/how-to-guides/http-to-td] page at https://docs.fluentd.org.
|
|
9
9
|
|
|
10
10
|
== Configuration
|
|
11
11
|
|
|
@@ -37,7 +37,7 @@ The configuration options currently supported are:
|
|
|
37
37
|
*NOTE*
|
|
38
38
|
|
|
39
39
|
depending on the access control permissions associated to the API key, a database can or cannot be created if not already available.
|
|
40
|
-
See the {Treasure Data Access Control documentation}[
|
|
40
|
+
See the {Treasure Data Access Control documentation}[https://docs.treasuredata.com/articles/access-control] page for details.
|
|
41
41
|
|
|
42
42
|
+database+::
|
|
43
43
|
Specifies the destination database in the Treasure Data cloud.
|
|
@@ -54,12 +54,19 @@ The configuration options currently supported are:
|
|
|
54
54
|
This parameter is *required* unless the +auto_create_table+ option is used.
|
|
55
55
|
|
|
56
56
|
+endpoint+::
|
|
57
|
-
Specifies the Treasure Data's REST API endpoint.
|
|
57
|
+
Specifies the Treasure Data's REST API endpoint for import requests.
|
|
58
58
|
|
|
59
59
|
Requires the endpoint as argument.
|
|
60
60
|
|
|
61
|
-
If not specified, a default is used; please refer to the {td-client-ruby's :endpoint}[https://github.com/treasure-data/td-client-ruby#endpoint] option for more details.
|
|
61
|
+
If not specified, a default for import requests is used; please refer to the {td-client-ruby's :endpoint}[https://github.com/treasure-data/td-client-ruby#endpoint] option for more details.
|
|
62
62
|
|
|
63
|
+
+api_endpoint+::
|
|
64
|
+
Specifies the Treasure Data's REST API endpoint.
|
|
65
|
+
|
|
66
|
+
Requires the endpoint as argument.
|
|
67
|
+
|
|
68
|
+
If not specified, a default is used; please refer to the {td-client-ruby's :endpoint}[https://github.com/treasure-data/td-client-ruby#endpoint] option for more details.
|
|
69
|
+
|
|
63
70
|
+use_ssl+::
|
|
64
71
|
Specifies whether to communicate using SSL encryption over HTTPS.
|
|
65
72
|
|
data/fluent-plugin-td.gemspec
CHANGED
|
@@ -1,27 +1,26 @@
|
|
|
1
|
-
|
|
2
|
-
$:.push File.expand_path('../lib', __FILE__)
|
|
3
|
-
require 'fluent/plugin/td_plugin_version'
|
|
1
|
+
require File.expand_path('../lib/fluent/plugin/td_plugin_version', __FILE__)
|
|
4
2
|
|
|
5
3
|
Gem::Specification.new do |gem|
|
|
6
|
-
gem.name =
|
|
7
|
-
gem.description =
|
|
8
|
-
gem.homepage =
|
|
4
|
+
gem.name = 'fluent-plugin-td'
|
|
5
|
+
gem.description = 'Treasure Data Cloud Data Service plugin for Fluentd'
|
|
6
|
+
gem.homepage = 'https://www.treasuredata.com/'
|
|
9
7
|
gem.summary = gem.description
|
|
10
8
|
gem.version = Fluent::Plugin::TreasureDataPlugin::VERSION
|
|
11
|
-
gem.authors = [
|
|
12
|
-
gem.email =
|
|
13
|
-
gem.has_rdoc = false
|
|
14
|
-
#gem.platform = Gem::Platform::RUBY
|
|
9
|
+
gem.authors = ['Treasure Data, Inc.']
|
|
10
|
+
gem.email = 'support@treasure-data.com'
|
|
15
11
|
gem.files = `git ls-files`.split("\n")
|
|
16
12
|
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
|
17
|
-
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
|
13
|
+
gem.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
|
|
18
14
|
gem.require_paths = ['lib']
|
|
19
|
-
gem.license =
|
|
15
|
+
gem.license = 'Apache-2.0'
|
|
20
16
|
|
|
21
|
-
gem.
|
|
22
|
-
gem.add_dependency
|
|
23
|
-
gem.
|
|
24
|
-
gem.add_development_dependency
|
|
25
|
-
gem.add_development_dependency
|
|
26
|
-
gem.add_development_dependency
|
|
17
|
+
gem.required_ruby_version = '>= 2.4'
|
|
18
|
+
gem.add_dependency 'fluentd', ['>= 0.14.13', '< 2']
|
|
19
|
+
gem.add_dependency 'td-client', '>= 1.0.8'
|
|
20
|
+
gem.add_development_dependency 'rake', '>= 0.9.2'
|
|
21
|
+
gem.add_development_dependency 'rubocop'
|
|
22
|
+
gem.add_development_dependency 'rubocop-rake'
|
|
23
|
+
gem.add_development_dependency 'test-unit', '~> 3.5.3'
|
|
24
|
+
gem.add_development_dependency 'test-unit-rr', '~> 1.0.5'
|
|
25
|
+
gem.add_development_dependency 'webmock', '~> 3.14.0'
|
|
27
26
|
end
|
|
@@ -4,6 +4,7 @@ require 'zlib'
|
|
|
4
4
|
require 'stringio'
|
|
5
5
|
require 'td-client'
|
|
6
6
|
|
|
7
|
+
require 'fluent/error'
|
|
7
8
|
require 'fluent/plugin/output'
|
|
8
9
|
require 'fluent/plugin/td_plugin_version'
|
|
9
10
|
|
|
@@ -12,6 +13,7 @@ module Fluent::Plugin
|
|
|
12
13
|
Fluent::Plugin.register_output('tdlog', self)
|
|
13
14
|
|
|
14
15
|
IMPORT_SIZE_LIMIT = 32 * 1024 * 1024
|
|
16
|
+
IMPORT_RECORDS_LIMIT = 8096
|
|
15
17
|
UPLOAD_EXT = 'msgpack.gz'.freeze
|
|
16
18
|
|
|
17
19
|
helpers :event_emitter, :compat_parameters
|
|
@@ -22,7 +24,8 @@ module Fluent::Plugin
|
|
|
22
24
|
config_param :table, :string, :default => nil
|
|
23
25
|
config_param :use_gzip_command, :bool, :default => false
|
|
24
26
|
|
|
25
|
-
config_param :
|
|
27
|
+
config_param :import_endpoint, :string, :alias => :endpoint, :default => TreasureData::API::DEFAULT_IMPORT_ENDPOINT
|
|
28
|
+
config_param :api_endpoint, :string, :default => TreasureData::API::DEFAULT_ENDPOINT
|
|
26
29
|
config_param :use_ssl, :bool, :default => true
|
|
27
30
|
config_param :tmpdir, :string, :default => nil
|
|
28
31
|
config_param :http_proxy, :string, :default => nil
|
|
@@ -35,13 +38,14 @@ module Fluent::Plugin
|
|
|
35
38
|
config_set_default :chunk_keys, ['tag']
|
|
36
39
|
config_set_default :flush_interval, 300
|
|
37
40
|
config_set_default :chunk_limit_size, IMPORT_SIZE_LIMIT
|
|
41
|
+
config_set_default :chunk_limit_records, IMPORT_RECORDS_LIMIT
|
|
38
42
|
end
|
|
39
43
|
|
|
40
44
|
def initialize
|
|
41
45
|
super
|
|
42
46
|
@key = nil
|
|
43
|
-
@key_num_limit = 512
|
|
44
|
-
@record_size_limit = 32 * 1024 * 1024
|
|
47
|
+
@key_num_limit = 512
|
|
48
|
+
@record_size_limit = 32 * 1024 * 1024
|
|
45
49
|
@table_list = {}
|
|
46
50
|
@empty_gz_data = TreasureData::API.create_empty_gz_data
|
|
47
51
|
@user_agent = "fluent-plugin-td: #{TreasureDataPlugin::VERSION}".freeze
|
|
@@ -78,11 +82,11 @@ module Fluent::Plugin
|
|
|
78
82
|
super
|
|
79
83
|
|
|
80
84
|
client_opts = {
|
|
81
|
-
:ssl => @use_ssl, :http_proxy => @http_proxy, :user_agent => @user_agent,
|
|
85
|
+
:ssl => @use_ssl, :http_proxy => @http_proxy, :user_agent => @user_agent,
|
|
82
86
|
:connect_timeout => @connect_timeout, :read_timeout => @read_timeout, :send_timeout => @send_timeout
|
|
83
87
|
}
|
|
84
|
-
@client = TreasureData::Client.new(@apikey, client_opts)
|
|
85
|
-
|
|
88
|
+
@client = TreasureData::Client.new(@apikey, client_opts.merge({:endpoint => @import_endpoint}))
|
|
89
|
+
@api_client = TreasureData::Client.new(@apikey, client_opts.merge({:endpoint => @api_endpoint}))
|
|
86
90
|
if @key
|
|
87
91
|
if @auto_create_table
|
|
88
92
|
ensure_database_and_table(@database, @table)
|
|
@@ -106,7 +110,6 @@ module Fluent::Plugin
|
|
|
106
110
|
record.delete(:time) if record.has_key?(:time)
|
|
107
111
|
|
|
108
112
|
if record.size > @key_num_limit
|
|
109
|
-
# TODO include summary of the record
|
|
110
113
|
router.emit_error_event(tag, time, record, RuntimeError.new("too many number of keys (#{record.size} keys)"))
|
|
111
114
|
return nil
|
|
112
115
|
end
|
|
@@ -167,7 +170,6 @@ module Fluent::Plugin
|
|
|
167
170
|
f.close(true) if f
|
|
168
171
|
end
|
|
169
172
|
|
|
170
|
-
# TODO: Share this routine with s3 compressors
|
|
171
173
|
def gzip_by_command(chunk, tmp)
|
|
172
174
|
chunk_is_file = @buffer_config['@type'] == 'file'
|
|
173
175
|
path = if chunk_is_file
|
|
@@ -182,10 +184,8 @@ module Fluent::Plugin
|
|
|
182
184
|
res = system "gzip -c #{path} > #{tmp.path}"
|
|
183
185
|
unless res
|
|
184
186
|
log.warn "failed to execute gzip command. Fallback to GzipWriter. status = #{$?}"
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
return gzip_by_writer(chunk, tmp)
|
|
188
|
-
end
|
|
187
|
+
tmp.truncate(0)
|
|
188
|
+
return gzip_by_writer(chunk, tmp)
|
|
189
189
|
end
|
|
190
190
|
File.size(tmp.path)
|
|
191
191
|
ensure
|
|
@@ -210,18 +210,22 @@ module Fluent::Plugin
|
|
|
210
210
|
unique_str = unique_id.unpack('C*').map { |x| "%02x" % x }.join
|
|
211
211
|
log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }
|
|
212
212
|
|
|
213
|
+
start = Time.now
|
|
213
214
|
begin
|
|
214
215
|
begin
|
|
215
|
-
start = Time.now
|
|
216
216
|
@client.import(database, table, UPLOAD_EXT, io, size, unique_str)
|
|
217
|
-
rescue TreasureData::NotFoundError
|
|
217
|
+
rescue TreasureData::NotFoundError
|
|
218
218
|
unless @auto_create_table
|
|
219
|
-
raise
|
|
219
|
+
raise
|
|
220
220
|
end
|
|
221
221
|
ensure_database_and_table(database, table)
|
|
222
222
|
io.pos = 0
|
|
223
223
|
retry
|
|
224
224
|
end
|
|
225
|
+
rescue TreasureData::TooManyRequestsError
|
|
226
|
+
raise
|
|
227
|
+
rescue TreasureData::ClientError => e
|
|
228
|
+
raise Fluent::UnrecoverableError.new(e.message)
|
|
225
229
|
rescue => e
|
|
226
230
|
elapsed = Time.now - start
|
|
227
231
|
ne = RuntimeError.new("Failed to upload to Treasure Data '#{database}.#{table}' table: #{e.inspect} (#{size} bytes; #{elapsed} seconds)")
|
|
@@ -263,11 +267,12 @@ module Fluent::Plugin
|
|
|
263
267
|
def ensure_database_and_table(database, table)
|
|
264
268
|
log.info "Creating table #{database}.#{table} on TreasureData"
|
|
265
269
|
begin
|
|
266
|
-
@
|
|
270
|
+
@api_client.create_log_table(database, table)
|
|
267
271
|
rescue TreasureData::NotFoundError
|
|
268
|
-
@
|
|
269
|
-
@
|
|
272
|
+
@api_client.create_database(database)
|
|
273
|
+
@api_client.create_log_table(database, table)
|
|
270
274
|
rescue TreasureData::AlreadyExistsError
|
|
275
|
+
# ignored
|
|
271
276
|
end
|
|
272
277
|
end
|
|
273
278
|
end
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
require 'fluent/test'
|
|
2
2
|
require 'fluent/test/driver/output'
|
|
3
3
|
require 'fluent/plugin/out_tdlog'
|
|
4
|
-
require 'test_helper
|
|
4
|
+
require 'test_helper'
|
|
5
5
|
|
|
6
6
|
class TreasureDataLogOutputTest < Test::Unit::TestCase
|
|
7
7
|
TMP_DIR = File.dirname(__FILE__) + "/tmp"
|
|
@@ -144,7 +144,7 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
|
|
|
144
144
|
}
|
|
145
145
|
end
|
|
146
146
|
|
|
147
|
-
def
|
|
147
|
+
def test_emit_with_time_symbol
|
|
148
148
|
d = create_driver
|
|
149
149
|
time, records = stub_seed_values
|
|
150
150
|
database, table = d.instance.instance_variable_get(:@key).split(".", 2)
|
|
@@ -162,12 +162,11 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
|
|
|
162
162
|
end
|
|
163
163
|
|
|
164
164
|
def test_emit_with_endpoint
|
|
165
|
-
d = create_driver(DEFAULT_CONFIG + "endpoint foo.bar.baz")
|
|
166
|
-
opts = {:endpoint => 'foo.bar.baz'}
|
|
165
|
+
d = create_driver(DEFAULT_CONFIG + "endpoint foo.bar.baz\napi_endpoint boo.bar.baz")
|
|
167
166
|
time, records = stub_seed_values
|
|
168
167
|
database, table = d.instance.instance_variable_get(:@key).split(".", 2)
|
|
169
|
-
stub_td_table_create_request(database, table,
|
|
170
|
-
stub_td_import_request(stub_request_body(records, time), database, table,
|
|
168
|
+
stub_td_table_create_request(database, table, {:endpoint => 'boo.bar.baz'})
|
|
169
|
+
stub_td_import_request(stub_request_body(records, time), database, table, {:endpoint => 'foo.bar.baz'})
|
|
171
170
|
|
|
172
171
|
d.run(default_tag: 'test') {
|
|
173
172
|
records.each { |record|
|
|
@@ -177,12 +176,11 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
|
|
|
177
176
|
end
|
|
178
177
|
|
|
179
178
|
def test_emit_with_too_many_keys
|
|
180
|
-
d = create_driver(DEFAULT_CONFIG
|
|
181
|
-
|
|
182
|
-
time, records = stub_seed_values
|
|
179
|
+
d = create_driver(DEFAULT_CONFIG)
|
|
180
|
+
time, _ = stub_seed_values
|
|
183
181
|
database, table = d.instance.instance_variable_get(:@key).split(".", 2)
|
|
184
|
-
stub_td_table_create_request(database, table
|
|
185
|
-
stub_td_import_request(stub_request_body([], time), database, table
|
|
182
|
+
stub_td_table_create_request(database, table)
|
|
183
|
+
stub_td_import_request(stub_request_body([], time), database, table)
|
|
186
184
|
|
|
187
185
|
d.run(default_tag: 'test') {
|
|
188
186
|
d.feed(time, create_too_many_keys_record)
|
|
@@ -192,6 +190,38 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
|
|
|
192
190
|
assert_equal 1, d.error_events.size
|
|
193
191
|
end
|
|
194
192
|
|
|
193
|
+
def test_write_with_client_error
|
|
194
|
+
d = create_driver(DEFAULT_CONFIG)
|
|
195
|
+
time, records = stub_seed_values
|
|
196
|
+
database, table = d.instance.instance_variable_get(:@key).split(".", 2)
|
|
197
|
+
stub_td_table_create_request(database, table)
|
|
198
|
+
stub_td_import_request(stub_request_body(records, time), database, table, status: 400)
|
|
199
|
+
|
|
200
|
+
assert_nothing_raised(Fluent::UnrecoverableError) do
|
|
201
|
+
d.run(default_tag: 'test') {
|
|
202
|
+
records.each { |record|
|
|
203
|
+
d.feed(time, record)
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def test_write_retry_if_too_many_requests
|
|
210
|
+
d = create_driver(DEFAULT_CONFIG)
|
|
211
|
+
time, records = stub_seed_values
|
|
212
|
+
database, table = d.instance.instance_variable_get(:@key).split(".", 2)
|
|
213
|
+
stub_td_table_create_request(database, table)
|
|
214
|
+
stub_td_import_request(stub_request_body(records, time), database, table, status: 429)
|
|
215
|
+
|
|
216
|
+
assert_raise(TreasureData::TooManyRequestsError) do
|
|
217
|
+
d.run(default_tag: 'test') {
|
|
218
|
+
records.each { |record|
|
|
219
|
+
d.feed(time, record)
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
end
|
|
223
|
+
end
|
|
224
|
+
|
|
195
225
|
sub_test_case 'tag splitting for database and table' do
|
|
196
226
|
def create_driver(conf = %[auto_create_table true])
|
|
197
227
|
config = BASE_CONFIG + conf
|
|
@@ -199,7 +229,7 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
|
|
|
199
229
|
Fluent::Test::Driver::Output.new(Fluent::Plugin::TreasureDataLogOutput).configure(config)
|
|
200
230
|
end
|
|
201
231
|
|
|
202
|
-
data('
|
|
232
|
+
data('event_time' => 'event_time', 'int_time' => 'int')
|
|
203
233
|
def test_tag_split(time_class)
|
|
204
234
|
d = create_driver
|
|
205
235
|
|
data/test/test_helper.rb
CHANGED
|
@@ -59,7 +59,7 @@ class Test::Unit::TestCase
|
|
|
59
59
|
opts[:use_ssl] = true unless opts.has_key?(:use_ssl)
|
|
60
60
|
schema = opts[:use_ssl] ? 'https' : 'http'
|
|
61
61
|
response = {"database" => database, "table" => table}.to_json
|
|
62
|
-
endpoint = opts[:endpoint] ? opts[:endpoint] : TreasureData::API::
|
|
62
|
+
endpoint = opts[:endpoint] ? opts[:endpoint] : TreasureData::API::DEFAULT_ENDPOINT
|
|
63
63
|
|
|
64
64
|
url = "#{schema}://#{endpoint}/v3/table/create/#{e(database)}/#{e(table)}/log"
|
|
65
65
|
stub_request(:post, url).to_return(:status => 200, :body => response)
|
|
@@ -69,8 +69,9 @@ class Test::Unit::TestCase
|
|
|
69
69
|
opts[:use_ssl] = true unless opts.has_key?(:use_ssl)
|
|
70
70
|
format = opts[:format] || 'msgpack.gz'
|
|
71
71
|
schema = opts[:use_ssl] ? 'https' : 'http'
|
|
72
|
+
status = opts[:status] || 200
|
|
72
73
|
response = {"database" => db, "table" => table, "elapsed_time" => 0}.to_json
|
|
73
|
-
endpoint = opts[:endpoint] ? opts[:endpoint] : TreasureData::API::
|
|
74
|
+
endpoint = opts[:endpoint] ? opts[:endpoint] : TreasureData::API::DEFAULT_IMPORT_ENDPOINT
|
|
74
75
|
|
|
75
76
|
# for check_table_existence
|
|
76
77
|
url_with_empty = "#{schema}://#{endpoint}/v3/table/import/#{e(db)}/#{e(table)}/#{format}"
|
|
@@ -80,6 +81,6 @@ class Test::Unit::TestCase
|
|
|
80
81
|
stub_request(:put, url_with_unique).with(:headers => {'Content-Type' => 'application/octet-stream'}) { |req|
|
|
81
82
|
@auth_header = req.headers["Authorization"]
|
|
82
83
|
stub_gzip_unwrap(req.body) == stub_gzip_unwrap(body)
|
|
83
|
-
}.to_return(:status =>
|
|
84
|
+
}.to_return(:status => status, :body => response)
|
|
84
85
|
end
|
|
85
86
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-td
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Treasure Data, Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-05-24 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: fluentd
|
|
@@ -34,16 +34,16 @@ dependencies:
|
|
|
34
34
|
name: td-client
|
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
|
36
36
|
requirements:
|
|
37
|
-
- - "
|
|
37
|
+
- - ">="
|
|
38
38
|
- !ruby/object:Gem::Version
|
|
39
|
-
version:
|
|
39
|
+
version: 1.0.8
|
|
40
40
|
type: :runtime
|
|
41
41
|
prerelease: false
|
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
|
43
43
|
requirements:
|
|
44
|
-
- - "
|
|
44
|
+
- - ">="
|
|
45
45
|
- !ruby/object:Gem::Version
|
|
46
|
-
version:
|
|
46
|
+
version: 1.0.8
|
|
47
47
|
- !ruby/object:Gem::Dependency
|
|
48
48
|
name: rake
|
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -59,70 +59,98 @@ dependencies:
|
|
|
59
59
|
- !ruby/object:Gem::Version
|
|
60
60
|
version: 0.9.2
|
|
61
61
|
- !ruby/object:Gem::Dependency
|
|
62
|
-
name:
|
|
62
|
+
name: rubocop
|
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
|
64
64
|
requirements:
|
|
65
|
-
- - "
|
|
65
|
+
- - ">="
|
|
66
66
|
- !ruby/object:Gem::Version
|
|
67
|
-
version: '
|
|
67
|
+
version: '0'
|
|
68
68
|
type: :development
|
|
69
69
|
prerelease: false
|
|
70
70
|
version_requirements: !ruby/object:Gem::Requirement
|
|
71
71
|
requirements:
|
|
72
|
-
- - "
|
|
72
|
+
- - ">="
|
|
73
73
|
- !ruby/object:Gem::Version
|
|
74
|
-
version: '
|
|
74
|
+
version: '0'
|
|
75
|
+
- !ruby/object:Gem::Dependency
|
|
76
|
+
name: rubocop-rake
|
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
|
78
|
+
requirements:
|
|
79
|
+
- - ">="
|
|
80
|
+
- !ruby/object:Gem::Version
|
|
81
|
+
version: '0'
|
|
82
|
+
type: :development
|
|
83
|
+
prerelease: false
|
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
85
|
+
requirements:
|
|
86
|
+
- - ">="
|
|
87
|
+
- !ruby/object:Gem::Version
|
|
88
|
+
version: '0'
|
|
75
89
|
- !ruby/object:Gem::Dependency
|
|
76
90
|
name: test-unit
|
|
77
91
|
requirement: !ruby/object:Gem::Requirement
|
|
78
92
|
requirements:
|
|
79
93
|
- - "~>"
|
|
80
94
|
- !ruby/object:Gem::Version
|
|
81
|
-
version: 3.
|
|
95
|
+
version: 3.5.3
|
|
82
96
|
type: :development
|
|
83
97
|
prerelease: false
|
|
84
98
|
version_requirements: !ruby/object:Gem::Requirement
|
|
85
99
|
requirements:
|
|
86
100
|
- - "~>"
|
|
87
101
|
- !ruby/object:Gem::Version
|
|
88
|
-
version: 3.
|
|
102
|
+
version: 3.5.3
|
|
89
103
|
- !ruby/object:Gem::Dependency
|
|
90
104
|
name: test-unit-rr
|
|
91
105
|
requirement: !ruby/object:Gem::Requirement
|
|
92
106
|
requirements:
|
|
93
107
|
- - "~>"
|
|
94
108
|
- !ruby/object:Gem::Version
|
|
95
|
-
version: 1.0.
|
|
109
|
+
version: 1.0.5
|
|
110
|
+
type: :development
|
|
111
|
+
prerelease: false
|
|
112
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
113
|
+
requirements:
|
|
114
|
+
- - "~>"
|
|
115
|
+
- !ruby/object:Gem::Version
|
|
116
|
+
version: 1.0.5
|
|
117
|
+
- !ruby/object:Gem::Dependency
|
|
118
|
+
name: webmock
|
|
119
|
+
requirement: !ruby/object:Gem::Requirement
|
|
120
|
+
requirements:
|
|
121
|
+
- - "~>"
|
|
122
|
+
- !ruby/object:Gem::Version
|
|
123
|
+
version: 3.14.0
|
|
96
124
|
type: :development
|
|
97
125
|
prerelease: false
|
|
98
126
|
version_requirements: !ruby/object:Gem::Requirement
|
|
99
127
|
requirements:
|
|
100
128
|
- - "~>"
|
|
101
129
|
- !ruby/object:Gem::Version
|
|
102
|
-
version:
|
|
130
|
+
version: 3.14.0
|
|
103
131
|
description: Treasure Data Cloud Data Service plugin for Fluentd
|
|
104
132
|
email: support@treasure-data.com
|
|
105
133
|
executables: []
|
|
106
134
|
extensions: []
|
|
107
135
|
extra_rdoc_files: []
|
|
108
136
|
files:
|
|
137
|
+
- ".circleci/config.yml"
|
|
138
|
+
- ".github/dependabot.yml"
|
|
139
|
+
- ".github/workflows/codeql-analysis.yml"
|
|
109
140
|
- ".gitignore"
|
|
110
|
-
- ".travis.yml"
|
|
111
141
|
- AUTHORS
|
|
112
142
|
- ChangeLog
|
|
113
143
|
- Gemfile
|
|
114
|
-
- Gemfile.
|
|
115
|
-
- Gemfile.v0.12
|
|
144
|
+
- Gemfile.lock
|
|
116
145
|
- README.rdoc
|
|
117
146
|
- Rakefile
|
|
118
147
|
- example.conf
|
|
119
148
|
- fluent-plugin-td.gemspec
|
|
120
149
|
- lib/fluent/plugin/out_tdlog.rb
|
|
121
|
-
- lib/fluent/plugin/td_plugin_util.rb
|
|
122
150
|
- lib/fluent/plugin/td_plugin_version.rb
|
|
123
151
|
- test/plugin/test_out_tdlog.rb
|
|
124
152
|
- test/test_helper.rb
|
|
125
|
-
homepage:
|
|
153
|
+
homepage: https://www.treasuredata.com/
|
|
126
154
|
licenses:
|
|
127
155
|
- Apache-2.0
|
|
128
156
|
metadata: {}
|
|
@@ -134,18 +162,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
134
162
|
requirements:
|
|
135
163
|
- - ">="
|
|
136
164
|
- !ruby/object:Gem::Version
|
|
137
|
-
version: '
|
|
165
|
+
version: '2.4'
|
|
138
166
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
139
167
|
requirements:
|
|
140
|
-
- - "
|
|
168
|
+
- - ">="
|
|
141
169
|
- !ruby/object:Gem::Version
|
|
142
|
-
version:
|
|
170
|
+
version: '0'
|
|
143
171
|
requirements: []
|
|
144
|
-
|
|
145
|
-
rubygems_version: 2.6.8
|
|
172
|
+
rubygems_version: 3.1.2
|
|
146
173
|
signing_key:
|
|
147
174
|
specification_version: 4
|
|
148
175
|
summary: Treasure Data Cloud Data Service plugin for Fluentd
|
|
149
|
-
test_files:
|
|
150
|
-
- test/plugin/test_out_tdlog.rb
|
|
151
|
-
- test/test_helper.rb
|
|
176
|
+
test_files: []
|
data/.travis.yml
DELETED
data/Gemfile.msgpack.0.4
DELETED
data/Gemfile.v0.12
DELETED
|
@@ -1,107 +0,0 @@
|
|
|
1
|
-
module Fluent
|
|
2
|
-
module TDPluginUtil
|
|
3
|
-
require 'fileutils'
|
|
4
|
-
require 'stringio'
|
|
5
|
-
require 'tempfile'
|
|
6
|
-
require 'zlib'
|
|
7
|
-
require 'td-client'
|
|
8
|
-
|
|
9
|
-
def validate_database_and_table_name(database, table, conf)
|
|
10
|
-
begin
|
|
11
|
-
TreasureData::API.validate_database_name(database)
|
|
12
|
-
rescue => e
|
|
13
|
-
raise ConfigError, "Invalid database name #{database.inspect}: #{e}: #{conf}"
|
|
14
|
-
end
|
|
15
|
-
begin
|
|
16
|
-
TreasureData::API.validate_table_name(table)
|
|
17
|
-
rescue => e
|
|
18
|
-
raise ConfigError, "Invalid table name #{table.inspect}: #{e}: #{conf}"
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
def parse_bool_parameter(param)
|
|
23
|
-
if param.empty?
|
|
24
|
-
true
|
|
25
|
-
else
|
|
26
|
-
param = Config.bool_value(param)
|
|
27
|
-
raise ConfigError, "'true' or 'false' is required for #{key} option on tdlog output" if param.nil?
|
|
28
|
-
param
|
|
29
|
-
end
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
def summarize_record(record)
|
|
33
|
-
json = record.to_json
|
|
34
|
-
if json.size > 100
|
|
35
|
-
json[0..97] + "..."
|
|
36
|
-
else
|
|
37
|
-
json
|
|
38
|
-
end
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
def check_table_existence(database, table)
|
|
42
|
-
@table_list ||= {}
|
|
43
|
-
key = "#{database}.#{table}"
|
|
44
|
-
unless @table_list.has_key?(key)
|
|
45
|
-
log.debug "checking whether table '#{key}' exists on Treasure Data"
|
|
46
|
-
io = StringIO.new(@empty_gz_data)
|
|
47
|
-
begin
|
|
48
|
-
# here doesn't check whether target table is item table or not because import-only user can't read the table status.
|
|
49
|
-
# So I use empty import request to check table existence.
|
|
50
|
-
@client.import(database, table, "msgpack.gz", io, io.size)
|
|
51
|
-
@table_list[key] = true
|
|
52
|
-
rescue TreasureData::NotFoundError
|
|
53
|
-
args = self.class == TreasureDataItemOutput ? ' -t item' : ''
|
|
54
|
-
raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td table:create #{database} #{table}#{args}' to create it."
|
|
55
|
-
rescue => e
|
|
56
|
-
log.warn "failed to check table existence on Treasure Data", :error => e.inspect
|
|
57
|
-
log.debug_backtrace e
|
|
58
|
-
end
|
|
59
|
-
end
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
def write(chunk)
|
|
63
|
-
unique_id = chunk.unique_id
|
|
64
|
-
database, table = chunk.key.split('.', 2)
|
|
65
|
-
|
|
66
|
-
FileUtils.mkdir_p(@tmpdir) unless @tmpdir.nil?
|
|
67
|
-
f = Tempfile.new(@tmpdir_prefix, @tmpdir)
|
|
68
|
-
w = Zlib::GzipWriter.new(f)
|
|
69
|
-
|
|
70
|
-
chunk.write_to(w)
|
|
71
|
-
w.finish
|
|
72
|
-
w = nil
|
|
73
|
-
|
|
74
|
-
size = f.pos
|
|
75
|
-
f.pos = 0
|
|
76
|
-
upload(database, table, f, size, unique_id)
|
|
77
|
-
ensure
|
|
78
|
-
w.close if w
|
|
79
|
-
f.close if f
|
|
80
|
-
end
|
|
81
|
-
|
|
82
|
-
# assume @client and @auto_create_table variable exist
|
|
83
|
-
def upload(database, table, io, size, unique_id)
|
|
84
|
-
unique_str = unique_id.unpack('C*').map {|x| "%02x" % x }.join
|
|
85
|
-
log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }
|
|
86
|
-
|
|
87
|
-
begin
|
|
88
|
-
begin
|
|
89
|
-
start = Time.now
|
|
90
|
-
@client.import(database, table, "msgpack.gz", io, size, unique_str)
|
|
91
|
-
rescue TreasureData::NotFoundError => e
|
|
92
|
-
unless @auto_create_table
|
|
93
|
-
raise e
|
|
94
|
-
end
|
|
95
|
-
ensure_database_and_table(database, table)
|
|
96
|
-
io.pos = 0
|
|
97
|
-
retry
|
|
98
|
-
end
|
|
99
|
-
rescue => e
|
|
100
|
-
elapsed = Time.now - start
|
|
101
|
-
ne = RuntimeError.new("Failed to upload to TreasureData: #{e} (#{size} bytes; #{elapsed} seconds)")
|
|
102
|
-
ne.set_backtrace(e.backtrace)
|
|
103
|
-
raise ne
|
|
104
|
-
end
|
|
105
|
-
end
|
|
106
|
-
end
|
|
107
|
-
end
|