fluent-plugin-td 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/ChangeLog +6 -0
- data/README.rdoc +11 -4
- data/fluent-plugin-td.gemspec +0 -1
- data/lib/fluent/plugin/out_tdlog.rb +8 -7
- data/lib/fluent/plugin/td_plugin_version.rb +1 -1
- data/test/plugin/test_out_tdlog.rb +7 -9
- data/test/test_helper.rb +2 -2
- metadata +3 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: f8e132d9df4dd5ed1344497a0680b12caa75c8d9b6361feef6d36e6d50eef0d2
|
4
|
+
data.tar.gz: 475920dcb5e14528ce9e2c5d5baab44811fa1fcab9b613f22c8212e10e56cf4c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bf337f321840654a6ace12117b899234bfe9b144756d96f34986381e84eb8a9f4d6d2f970ae9d55a3b49ed4fce6bf3c9d514aaaaa6e44c441f567c75d3238b0a
|
7
|
+
data.tar.gz: 74fc971d23dac8608901ff7bcc9add4c969f20bdf3a293a653f26ebc703e3b951479ac9e98d296f83e222ad3dc8fc0c6636e5a0f1533e4e2fa6397225b7cc79b
|
data/ChangeLog
CHANGED
data/README.rdoc
CHANGED
@@ -5,7 +5,7 @@ Treasure Data's REST APIs.
|
|
5
5
|
|
6
6
|
This plugin relies on the Treasure Data's {Ruby client library}[https://github.com/treasure-data/td-client-ruby] to communicate with the Treasure Data's REST APIs.
|
7
7
|
|
8
|
-
For more information, please visit the official {Fluentd TD Output plugin}[
|
8
|
+
For more information, please visit the official {Fluentd TD Output plugin}[https://docs.fluentd.org/how-to-guides/http-to-td] page at https://docs.fluentd.org.
|
9
9
|
|
10
10
|
== Configuration
|
11
11
|
|
@@ -37,7 +37,7 @@ The configuration options currently supported are:
|
|
37
37
|
*NOTE*
|
38
38
|
|
39
39
|
depending on the access control permissions associated to the API key, a database can or cannot be created if not already available.
|
40
|
-
See the {Treasure Data Access Control documentation}[
|
40
|
+
See the {Treasure Data Access Control documentation}[https://docs.treasuredata.com/articles/access-control] page for details.
|
41
41
|
|
42
42
|
+database+::
|
43
43
|
Specifies the destination database in the Treasure Data cloud.
|
@@ -54,12 +54,19 @@ The configuration options currently supported are:
|
|
54
54
|
This parameter is *required* unless the +auto_create_table+ option is used.
|
55
55
|
|
56
56
|
+endpoint+::
|
57
|
-
Specifies the Treasure Data's REST API endpoint.
|
57
|
+
Specifies the Treasure Data's REST API endpoint for import requests.
|
58
58
|
|
59
59
|
Requires the endpoint as argument.
|
60
60
|
|
61
|
-
If not specified, a default is used; please refer to the {td-client-ruby's :endpoint}[https://github.com/treasure-data/td-client-ruby#endpoint] option for more details.
|
61
|
+
If not specified, a default for import requests is used; please refer to the {td-client-ruby's :endpoint}[https://github.com/treasure-data/td-client-ruby#endpoint] option for more details.
|
62
62
|
|
63
|
+
+api_endpoint+::
|
64
|
+
Specifies the Treasure Data's REST API endpoint.
|
65
|
+
|
66
|
+
Requires the endpoint as argument.
|
67
|
+
|
68
|
+
If not specified, a default is used; please refer to the {td-client-ruby's :endpoint}[https://github.com/treasure-data/td-client-ruby#endpoint] option for more details.
|
69
|
+
|
63
70
|
+use_ssl+::
|
64
71
|
Specifies whether to communicate using SSL encryption over HTTPS.
|
65
72
|
|
data/fluent-plugin-td.gemspec
CHANGED
@@ -10,7 +10,6 @@ Gem::Specification.new do |gem|
|
|
10
10
|
gem.version = Fluent::Plugin::TreasureDataPlugin::VERSION
|
11
11
|
gem.authors = ["Treasure Data, Inc."]
|
12
12
|
gem.email = "support@treasure-data.com"
|
13
|
-
gem.has_rdoc = false
|
14
13
|
#gem.platform = Gem::Platform::RUBY
|
15
14
|
gem.files = `git ls-files`.split("\n")
|
16
15
|
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
@@ -22,7 +22,8 @@ module Fluent::Plugin
|
|
22
22
|
config_param :table, :string, :default => nil
|
23
23
|
config_param :use_gzip_command, :bool, :default => false
|
24
24
|
|
25
|
-
config_param :
|
25
|
+
config_param :import_endpoint, :string, :alias => :endpoint, :default => TreasureData::API::DEFAULT_IMPORT_ENDPOINT
|
26
|
+
config_param :api_endpoint, :string, :default => TreasureData::API::DEFAULT_ENDPOINT
|
26
27
|
config_param :use_ssl, :bool, :default => true
|
27
28
|
config_param :tmpdir, :string, :default => nil
|
28
29
|
config_param :http_proxy, :string, :default => nil
|
@@ -78,11 +79,11 @@ module Fluent::Plugin
|
|
78
79
|
super
|
79
80
|
|
80
81
|
client_opts = {
|
81
|
-
:ssl => @use_ssl, :http_proxy => @http_proxy, :user_agent => @user_agent,
|
82
|
+
:ssl => @use_ssl, :http_proxy => @http_proxy, :user_agent => @user_agent,
|
82
83
|
:connect_timeout => @connect_timeout, :read_timeout => @read_timeout, :send_timeout => @send_timeout
|
83
84
|
}
|
84
|
-
@client = TreasureData::Client.new(@apikey, client_opts)
|
85
|
-
|
85
|
+
@client = TreasureData::Client.new(@apikey, client_opts.merge({:endpoint => @import_endpoint}))
|
86
|
+
@api_client = TreasureData::Client.new(@apikey, client_opts.merge({:endpoint => @api_endpoint}))
|
86
87
|
if @key
|
87
88
|
if @auto_create_table
|
88
89
|
ensure_database_and_table(@database, @table)
|
@@ -263,10 +264,10 @@ module Fluent::Plugin
|
|
263
264
|
def ensure_database_and_table(database, table)
|
264
265
|
log.info "Creating table #{database}.#{table} on TreasureData"
|
265
266
|
begin
|
266
|
-
@
|
267
|
+
@api_client.create_log_table(database, table)
|
267
268
|
rescue TreasureData::NotFoundError
|
268
|
-
@
|
269
|
-
@
|
269
|
+
@api_client.create_database(database)
|
270
|
+
@api_client.create_log_table(database, table)
|
270
271
|
rescue TreasureData::AlreadyExistsError
|
271
272
|
end
|
272
273
|
end
|
@@ -162,12 +162,11 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
|
|
162
162
|
end
|
163
163
|
|
164
164
|
def test_emit_with_endpoint
|
165
|
-
d = create_driver(DEFAULT_CONFIG + "endpoint foo.bar.baz")
|
166
|
-
opts = {:endpoint => 'foo.bar.baz'}
|
165
|
+
d = create_driver(DEFAULT_CONFIG + "endpoint foo.bar.baz\napi_endpoint boo.bar.baz")
|
167
166
|
time, records = stub_seed_values
|
168
167
|
database, table = d.instance.instance_variable_get(:@key).split(".", 2)
|
169
|
-
stub_td_table_create_request(database, table,
|
170
|
-
stub_td_import_request(stub_request_body(records, time), database, table,
|
168
|
+
stub_td_table_create_request(database, table, {:endpoint => 'boo.bar.baz'})
|
169
|
+
stub_td_import_request(stub_request_body(records, time), database, table, {:endpoint => 'foo.bar.baz'})
|
171
170
|
|
172
171
|
d.run(default_tag: 'test') {
|
173
172
|
records.each { |record|
|
@@ -177,12 +176,11 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
|
|
177
176
|
end
|
178
177
|
|
179
178
|
def test_emit_with_too_many_keys
|
180
|
-
d = create_driver(DEFAULT_CONFIG
|
181
|
-
|
182
|
-
time, records = stub_seed_values
|
179
|
+
d = create_driver(DEFAULT_CONFIG)
|
180
|
+
time, _ = stub_seed_values
|
183
181
|
database, table = d.instance.instance_variable_get(:@key).split(".", 2)
|
184
|
-
stub_td_table_create_request(database, table
|
185
|
-
stub_td_import_request(stub_request_body([], time), database, table
|
182
|
+
stub_td_table_create_request(database, table)
|
183
|
+
stub_td_import_request(stub_request_body([], time), database, table)
|
186
184
|
|
187
185
|
d.run(default_tag: 'test') {
|
188
186
|
d.feed(time, create_too_many_keys_record)
|
data/test/test_helper.rb
CHANGED
@@ -59,7 +59,7 @@ class Test::Unit::TestCase
|
|
59
59
|
opts[:use_ssl] = true unless opts.has_key?(:use_ssl)
|
60
60
|
schema = opts[:use_ssl] ? 'https' : 'http'
|
61
61
|
response = {"database" => database, "table" => table}.to_json
|
62
|
-
endpoint = opts[:endpoint] ? opts[:endpoint] : TreasureData::API::
|
62
|
+
endpoint = opts[:endpoint] ? opts[:endpoint] : TreasureData::API::DEFAULT_ENDPOINT
|
63
63
|
|
64
64
|
url = "#{schema}://#{endpoint}/v3/table/create/#{e(database)}/#{e(table)}/log"
|
65
65
|
stub_request(:post, url).to_return(:status => 200, :body => response)
|
@@ -70,7 +70,7 @@ class Test::Unit::TestCase
|
|
70
70
|
format = opts[:format] || 'msgpack.gz'
|
71
71
|
schema = opts[:use_ssl] ? 'https' : 'http'
|
72
72
|
response = {"database" => db, "table" => table, "elapsed_time" => 0}.to_json
|
73
|
-
endpoint = opts[:endpoint] ? opts[:endpoint] : TreasureData::API::
|
73
|
+
endpoint = opts[:endpoint] ? opts[:endpoint] : TreasureData::API::DEFAULT_IMPORT_ENDPOINT
|
74
74
|
|
75
75
|
# for check_table_existence
|
76
76
|
url_with_empty = "#{schema}://#{endpoint}/v3/table/import/#{e(db)}/#{e(table)}/#{format}"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-td
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Treasure Data, Inc.
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-12-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -141,8 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
141
141
|
- !ruby/object:Gem::Version
|
142
142
|
version: '0'
|
143
143
|
requirements: []
|
144
|
-
|
145
|
-
rubygems_version: 2.6.13
|
144
|
+
rubygems_version: 3.0.3
|
146
145
|
signing_key:
|
147
146
|
specification_version: 4
|
148
147
|
summary: Treasure Data Cloud Data Service plugin for Fluentd
|