fluent-plugin-bcdb 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +9 -8
  3. data/CODE_OF_CONDUCT.md +74 -74
  4. data/Gemfile +4 -4
  5. data/Gemfile.lock +51 -51
  6. data/LICENSE +204 -204
  7. data/README.md +76 -78
  8. data/Rakefile +2 -2
  9. data/bin/console +14 -14
  10. data/bin/setup +8 -8
  11. data/dev/csv-conf/AB_NYC_2019.csv +49081 -0
  12. data/dev/csv-conf/fluent.conf +39 -0
  13. data/dev/fluent.conf +37 -37
  14. data/dev/http-conf/fluent.conf +37 -0
  15. data/dev/readme.md +64 -0
  16. data/fluent-plugin-bcdb.gemspec +48 -48
  17. data/lib/fluent/plugin/bcdb/version.rb +7 -7
  18. data/lib/fluent/plugin/out_bcdb.rb +467 -459
  19. metadata +20 -12
  20. data/Dockerfile +0 -28
  21. data/docker-compose.yaml +0 -38
data/dev/csv-conf/fluent.conf ADDED
@@ -0,0 +1,39 @@
1
+ <source>
2
+ @type tail
3
+ @id input1
4
+ @label @mainstream
5
+ path C:/Users/dorin.stan/Documents/workspace/0_CUSTOMERS-PEERS/Parteners/logstash-output-bcdb/AB_NYC_2019.csv
6
+ pos_file C:/Users/dorin.stan/Documents/fluentd_sincedb_path_file.pos # This is where you record file position
7
+ tag testcsv
8
+ <parse>
9
+ @type csv
10
+ delimiter (',')
11
+ keys id,name,host_id,host_name,neighbourhood_group,neighbourhood,latitude,longitude,room_type,price,minimum_nights,number_of_reviews,last_review,reviews_per_month,calculated_host_listings_count,availability_365
12
+ #time_key timestamp
13
+ </parse>
14
+ </source>
15
+
16
+ <label @mainstream>
17
+ <match **>
18
+ @type bcdb
19
+ base_url "http://10.0.11.141:32018/services/core/v1/api" # Api endpoint like https://bcdb.modex.tech/node-03/services/core/v1/api
20
+ auth_url "http://10.0.11.160:32018/services/oauth/token" # OAuth token endpoint like https://bcdb.modex.tech/oauth/token
21
+ bcdb_entity "log-csv" # default: logline
22
+ ssl_no_verify false # default: false
23
+ rate_limit_msec 0 # default: 0 = no rate limiting
24
+ raise_on_error true # default: true
25
+ recoverable_status_codes 5000, 400 # default: 503
26
+ cacert_file /etc/ssl/endpoint1.cert # default: ''
27
+ client_cert_path /path/to/client_cert.crt # default: ''
28
+ private_key_path /path/to/private_key.key # default: ''
29
+ private_key_passphrase yourpassphrase # default: ''
30
+ username admin@ibt.ro # ex. bcdb.admin@modex.tech default: ''
31
+ password 123123 # ex. admin default: '', secret: true
32
+ client_id 0x01 # default: ''. BDCB client_id
33
+ client_secret 0x000001 # default: ''. BDCB client_secret
34
+ buffered false # default: false. Switch non-buffered/buffered mode
35
+ bulk_request true # default: false. Send events as application/x-ndjson
36
+ compress_request false # default: false. Send compressed events
37
+ @log_level trace
38
+ </match>
39
+ </label>
data/dev/fluent.conf CHANGED
@@ -1,37 +1,37 @@
1
- <source>
2
- @type http
3
- @id input1
4
- @label @mainstream
5
- @log_level trace
6
- port 24224
7
- </source>
8
-
9
- <system>
10
- # equal to -qq option
11
- log_level trace
12
- </system>
13
-
14
- <label @mainstream>
15
- <match **>
16
- @type bcdb
17
- base_url "http://<your_domain_or_hostIP:port>/services/core/v1/api" # Api endpoint like https://bcdb.modex.tech/node-03/services/core/v1/api
18
- auth_url "http://<your_domain_or_hostIP:port>/token" # OAuth token endpoint like https://bcdb.modex.tech/oauth/token
19
- bcdb_entity "log" # default: logline
20
- ssl_no_verify false # default: false
21
- rate_limit_msec 100 # default: 0 = no rate limiting
22
- raise_on_error true # default: true
23
- recoverable_status_codes 5000, 400 # default: 503
24
- cacert_file /etc/ssl/endpoint1.cert # default: ''
25
- client_cert_path /path/to/client_cert.crt # default: ''
26
- private_key_path /path/to/private_key.key # default: ''
27
- private_key_passphrase yourpassphrase # default: ''
28
- username <username> # ex. bcdb.admin@modex.tech default: ''
29
- password <password> # ex. admin default: '', secret: true
30
- client_id 0x01 # default: ''. BDCB client_id
31
- client_secret 0x000001 # default: ''. BDCB client_secret
32
- buffered true # default: false. Switch non-buffered/buffered mode
33
- bulk_request true # default: false. Send events as application/x-ndjson
34
- compress_request false # default: false. Send compressed events
35
- @log_level trace
36
- </match>
37
- </label>
1
+ <source>
2
+ @type http
3
+ @id input1
4
+ @label @mainstream
5
+ @log_level trace
6
+ port 24224
7
+ </source>
8
+
9
+ <system>
10
+ # equal to -qq option
11
+ log_level trace
12
+ </system>
13
+
14
+ <label @mainstream>
15
+ <match **>
16
+ @type bcdb
17
+ base_url "http://<your_domain_or_hostIP:port>/services/core/v1/api" # Api endpoint like https://bcdb.modex.tech/node-03/services/core/v1/api
18
+ auth_url "http://<your_domain_or_hostIP:port>/token" # OAuth token endpoint like https://bcdb.modex.tech/oauth/token
19
+ bcdb_entity "log" # default: logline
20
+ ssl_no_verify false # default: false
21
+ rate_limit_msec 100 # default: 0 = no rate limiting
22
+ raise_on_error true # default: true
23
+ recoverable_status_codes 5000, 400 # default: 503
24
+ cacert_file /etc/ssl/endpoint1.cert # default: ''
25
+ client_cert_path /path/to/client_cert.crt # default: ''
26
+ private_key_path /path/to/private_key.key # default: ''
27
+ private_key_passphrase yourpassphrase # default: ''
28
+ username <username> # ex. bcdb.admin@modex.tech default: ''
29
+ password <password> # ex. admin default: '', secret: true
30
+ client_id 0x01 # default: ''. BDCB client_id
31
+ client_secret 0x000001 # default: ''. BDCB client_secret
32
+ buffered true # default: false. Switch non-buffered/buffered mode
33
+ bulk_request true # default: false. Send events as application/x-ndjson
34
+ compress_request false # default: false. Send compressed events
35
+ @log_level trace
36
+ </match>
37
+ </label>
data/dev/http-conf/fluent.conf ADDED
@@ -0,0 +1,37 @@
1
+ <source>
2
+ @type http
3
+ @id input1
4
+ @label @mainstream
5
+ @log_level trace
6
+ port 24224
7
+ </source>
8
+
9
+ <system>
10
+ # equal to -qq option
11
+ log_level trace
12
+ </system>
13
+
14
+ <label @mainstream>
15
+ <match **>
16
+ @type bcdb
17
+ base_url "http://<your_domain_or_hostIP:port>/services/core/v1/api" # Api endpoint like https://bcdb.modex.tech/node-03/services/core/v1/api
18
+ auth_url "http://<your_domain_or_hostIP:port>/token" # OAuth token endpoint like https://bcdb.modex.tech/oauth/token
19
+ bcdb_entity "log" # default: logline
20
+ ssl_no_verify false # default: false
21
+ rate_limit_msec 100 # default: 0 = no rate limiting
22
+ raise_on_error true # default: true
23
+ recoverable_status_codes 5000, 400 # default: 503
24
+ cacert_file /etc/ssl/endpoint1.cert # default: ''
25
+ client_cert_path /path/to/client_cert.crt # default: ''
26
+ private_key_path /path/to/private_key.key # default: ''
27
+ private_key_passphrase yourpassphrase # default: ''
28
+ username <username> # ex. bcdb.admin@modex.tech default: ''
29
+ password <password> # ex. admin default: '', secret: true
30
+ client_id 0x01 # default: ''. BDCB client_id
31
+ client_secret 0x000001 # default: ''. BDCB client_secret
32
+ buffered true # default: false. Switch non-buffered/buffered mode
33
+ bulk_request true # default: false. Send events as application/x-ndjson
34
+ compress_request false # default: false. Send compressed events
35
+ @log_level trace
36
+ </match>
37
+ </label>
data/dev/readme.md ADDED
@@ -0,0 +1,64 @@
1
+ # Fluentd:
2
+
3
+ ## 1. Installation steps
4
+ ---
5
+ ### **Install Ruby environment**
6
+
7
+ 1. Go to [JRuby download location]https://rubyinstaller.org/downloads/)
8
+
9
+ 1. Install from file JRuby with Dev tools as a bundle
10
+
11
+ ### **Install Fluentd**
12
+ 1. Go to [Fluentd download location]https://www.fluentd.org/download)
13
+
14
+ 2. Add Fluentd /usr/sbin folder to your PATH variable
15
+
16
+ ### **Use plugin**
17
+
18
+ 1. Build the custom plugin
19
+
20
+ ```bash
21
+ gem build fluent-plugin-bcdb
22
+ ```
23
+
24
+ 2. Install custom plugin to logstash
25
+
26
+ ```bash
27
+ gem install fluent-plugin-bcdb.gem
28
+ ```
29
+
30
+ 3. Verify installed gem plugins
31
+ ```bash
32
+ gem list
33
+ ```
34
+
35
+ 4. Edit fluent.conf with the configuration appropriate to your BCDB endpoints:
36
+ ```
37
+ <match **>
38
+ @type bcdb
39
+ base_url "http://<your_domain_or_hostIP:port>/services/core/v1/api" # Api endpoint like https://bcdb.modex.tech/node-03/services/core/v1/api
40
+ auth_url "http://<your_domain_or_hostIP:port>/token" # OAuth token endpoint like https://bcdb.modex.tech/oauth/token
41
+ bcdb_entity "log" # default: logline
42
+ ssl_no_verify false # default: false
43
+ rate_limit_msec 100 # default: 0 = no rate limiting
44
+ raise_on_error true # default: true
45
+ recoverable_status_codes 5000, 400 # default: 503
46
+ cacert_file /etc/ssl/endpoint1.cert # default: ''
47
+ client_cert_path /path/to/client_cert.crt # default: ''
48
+ private_key_path /path/to/private_key.key # default: ''
49
+ private_key_passphrase yourpassphrase # default: ''
50
+ username <username> # ex. bcdb.admin@modex.tech default: ''
51
+ password <password> # ex. admin default: '', secret: true
52
+ client_id 0x01 # default: ''. BDCB client_id
53
+ client_secret 0x000001 # default: ''. BDCB client_secret
54
+ buffered true # default: false. Switch non-buffered/buffered mode
55
+ compress_request false # default: false. Send compressed events
56
+ @log_level trace
57
+ </match>
58
+ ```
59
+
60
+ 5. Execute logstash based on logstash.conf
61
+
62
+ ```bash
63
+ fluentd -c <absolute-path>/fluent.conf
64
+ ```
data/fluent-plugin-bcdb.gemspec CHANGED
@@ -1,48 +1,48 @@
1
- #
2
- # Copyright ©2019. MODEX (Gibraltar) LIMITED
3
- #
4
- # Licensed under the Apache License, Version 2.0 (the "License");
5
- # you may not use this file except in compliance with the License.
6
- # You may obtain a copy of the License at
7
- #
8
- # http://www.apache.org/licenses/LICENSE-2.0
9
- #
10
- # Unless required by applicable law or agreed to in writing, software
11
- # distributed under the License is distributed on an "AS IS" BASIS,
12
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- # See the License for the specific language governing permissions and
14
- # limitations under the License.
15
-
16
- lib = File.expand_path("lib", __dir__)
17
- $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
18
- require "fluent/plugin/bcdb/version"
19
-
20
- Gem::Specification.new do |spec|
21
- spec.name = "fluent-plugin-bcdb"
22
- spec.version = Fluent::Plugin::Bcdb::VERSION
23
- spec.authors = ["Modex Team"]
24
- spec.email = ["support@modex.tech"]
25
-
26
- spec.summary = %q{Fluent output plugin to Modex Blockchain Database}
27
- spec.homepage = "https://github.com/messbusters/fluent-plugin-bcdb"
28
- spec.license = "APACHE"
29
-
30
- spec.metadata["homepage_uri"] = spec.homepage
31
- spec.metadata["source_code_uri"] = "https://github.com/messbusters/fluent-plugin-bcdb"
32
- spec.metadata["changelog_uri"] = "https://github.com/messbusters/fluent-plugin-bcdb"
33
-
34
- # Specify which files should be added to the gem when it is released.
35
- # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
36
- spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
37
- `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
38
- end
39
- spec.bindir = "exe"
40
- spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
41
- spec.require_paths = ["lib"]
42
-
43
- spec.add_dependency "yajl-ruby", "~> 1.0"
44
- spec.add_dependency "fluentd", [">= 0.14.22", "< 2"]
45
- spec.add_development_dependency "bundler", "~> 2.0"
46
- spec.add_development_dependency "rake", "~> 10.0"
47
- spec.add_development_dependency "test-unit", ">= 3.1.0"
48
- end
1
+ #
2
+ # Copyright ©2019. MODEX (Gibraltar) LIMITED
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ lib = File.expand_path("lib", __dir__)
17
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
18
+ require "fluent/plugin/bcdb/version"
19
+
20
+ Gem::Specification.new do |spec|
21
+ spec.name = "fluent-plugin-bcdb"
22
+ spec.version = Fluent::Plugin::Bcdb::VERSION
23
+ spec.authors = ["Modex Team"]
24
+ spec.email = ["support@modex.tech"]
25
+
26
+ spec.summary = %q{Fluent output plugin to Modex Blockchain Database}
27
+ spec.homepage = "https://github.com/modex-bcdb/fluent-plugin-bcdb"
28
+ spec.license = "APACHE"
29
+
30
+ spec.metadata["homepage_uri"] = spec.homepage
31
+ spec.metadata["source_code_uri"] = "https://github.com/modex-bcdb/fluent-plugin-bcdb"
32
+ spec.metadata["changelog_uri"] = "https://github.com/modex-bcdb/fluent-plugin-bcdb"
33
+
34
+ # Specify which files should be added to the gem when it is released.
35
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
36
+ spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
37
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
38
+ end
39
+ spec.bindir = "exe"
40
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
41
+ spec.require_paths = ["lib"]
42
+
43
+ spec.add_dependency "yajl-ruby", "~> 1.0"
44
+ spec.add_dependency "fluentd", [">= 0.14.22", "< 2"]
45
+ spec.add_development_dependency "bundler", "~> 2.0"
46
+ spec.add_development_dependency "rake", "~> 13.0"
47
+ spec.add_development_dependency "test-unit", ">= 3.1.0"
48
+ end
data/lib/fluent/plugin/bcdb/version.rb CHANGED
@@ -1,7 +1,7 @@
1
- module Fluent
2
- module Plugin
3
- module Bcdb
4
- VERSION = "0.2.0"
5
- end
6
- end
7
- end
1
+ module Fluent
2
+ module Plugin
3
+ module Bcdb
4
+ VERSION = "0.2.1"
5
+ end
6
+ end
7
+ end
data/lib/fluent/plugin/out_bcdb.rb CHANGED
@@ -1,459 +1,467 @@
1
- #
2
- # Copyright ©2019. MODEX (Gibraltar) LIMITED
3
- #
4
- # Licensed under the Apache License, Version 2.0 (the "License");
5
- # you may not use this file except in compliance with the License.
6
- # You may obtain a copy of the License at
7
- #
8
- # http://www.apache.org/licenses/LICENSE-2.0
9
- #
10
- # Unless required by applicable law or agreed to in writing, software
11
- # distributed under the License is distributed on an "AS IS" BASIS,
12
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- # See the License for the specific language governing permissions and
14
- # limitations under the License.
15
-
16
- require 'net/http'
17
- require 'uri'
18
- require 'yajl'
19
- require 'fluent/plugin/output'
20
- require "fluent/plugin/bcdb/version"
21
- require 'tempfile'
22
- require 'openssl'
23
- require 'zlib'
24
-
25
- class BcdbOut < Fluent::Plugin::Output
26
- Fluent::Plugin.register_output('bcdb', self)
27
-
28
- class RecoverableResponse < StandardError; end
29
-
30
- helpers :compat_parameters, :formatter
31
-
32
- DEFAULT_BUFFER_TYPE = "memory"
33
- DEFAULT_FORMATTER = "json"
34
-
35
- def initialize
36
- super
37
- end
38
-
39
- # BCDB data endpoint
40
- config_param :base_url, :string
41
-
42
- # BCDB Auth endpoint
43
- config_param :auth_url, :string
44
-
45
- # BCDB database entity model name
46
- config_param :bcdb_entity, :string, :default => 'loglines'
47
-
48
- # Set Net::HTTP.verify_mode to `OpenSSL::SSL::VERIFY_NONE`
49
- config_param :ssl_no_verify, :bool, :default => false
50
-
51
- # HTTP method
52
- config_param :http_method, :enum, list: [:get, :put, :post, :delete], :default => :post
53
-
54
- # form | json | text | raw
55
- config_param :serializer, :enum, list: [:json, :form, :text, :raw], :default => :json
56
-
57
- # Simple rate limiting: ignore any records within `rate_limit_msec`
58
- # since the last one.
59
- config_param :rate_limit_msec, :integer, :default => 0
60
-
61
- # Raise errors that were rescued during HTTP requests?
62
- config_param :raise_on_error, :bool, :default => true
63
-
64
- # Specify recoverable error codes
65
- config_param :recoverable_status_codes, :array, value_type: :integer, default: [503]
66
-
67
- # ca file to use for https request
68
- config_param :cacert_file, :string, :default => ''
69
-
70
- # specify client sertificate
71
- config_param :client_cert_path, :string, :default => ''
72
-
73
- # specify private key path
74
- config_param :private_key_path, :string, :default => ''
75
-
76
- # specify private key passphrase
77
- config_param :private_key_passphrase, :string, :default => '', :secret => true
78
-
79
- # custom headers
80
- config_param :custom_headers, :hash, :default => nil
81
-
82
- # 'none' | 'basic' | 'jwt' | 'bearer'
83
- config_param :authentication, :enum, list: [:none, :basic, :jwt, :bearer, :oauth], :default => :oauth
84
- config_param :username, :string, :default => ''
85
- config_param :password, :string, :default => '', :secret => true
86
- config_param :client_id, :string, :default => ''
87
- config_param :client_secret, :string, :default => '', :secret => true
88
- config_param :grant_type, :enum, list: [:password, :authorization_code], :default => :password
89
- config_param :token, :string, :default => ''
90
- # Switch non-buffered/buffered plugin
91
- config_param :buffered, :bool, :default => false
92
- config_param :bulk_request, :bool, :default => false
93
- # Compress with gzip except for form serializer
94
- config_param :compress_request, :bool, :default => false
95
-
96
- config_section :buffer do
97
- config_set_default :@type, DEFAULT_BUFFER_TYPE
98
- config_set_default :chunk_keys, ['tag']
99
- end
100
-
101
- config_section :format do
102
- config_set_default :@type, DEFAULT_FORMATTER
103
- end
104
-
105
- def configure(conf)
106
- compat_parameters_convert(conf, :buffer, :formatter)
107
- super
108
- @create_schema_url = "#{@base_url}" + "/catalog/_JsonSchema/" + "#{@bcdb_entity}"
109
- if @bulk_request
110
- @base_url = "#{@base_url}" + "/data/bulk/" + "#{@bcdb_entity}"
111
- else
112
- @base_url = "#{@base_url}" + "/data/" + "#{@bcdb_entity}"
113
- end
114
-
115
- bcdb_authorise() if @authentication == :oauth
116
-
117
- @ssl_verify_mode = if @ssl_no_verify
118
- OpenSSL::SSL::VERIFY_NONE
119
- else
120
- OpenSSL::SSL::VERIFY_PEER
121
- end
122
-
123
- @ca_file = @cacert_file
124
- @last_request_time = nil
125
- raise Fluent::ConfigError, "'tag' in chunk_keys is required." if !@chunk_key_tag && @buffered
126
-
127
- if @formatter_config = conf.elements('format').first
128
- @formatter = formatter_create
129
- end
130
-
131
- if @bulk_request
132
- class << self
133
- alias_method :format, :bulk_request_format
134
- end
135
- @formatter = formatter_create(type: :json)
136
- @serializer = :x_ndjson # secret settings for bulk_request
137
- else
138
- class << self
139
- alias_method :format, :split_request_format
140
- end
141
- end
142
- end
143
-
144
- def bcdb_authorise()
145
- auth_uri = URI.parse(@auth_url)
146
- auth_data = {
147
- :username => @username,
148
- :password => @password,
149
- :client_id => @client_id,
150
- :client_secret => @client_secret,
151
- :grant_type => @grant_type
152
- }
153
- status = true
154
- unless @token_oauth || (@expires_token && Time.now.utc > @expires_token)
155
- https= Net::HTTP.new(auth_uri.host,auth_uri.port)
156
- https.use_ssl = auth_uri.scheme == 'https'
157
-
158
- request = Net::HTTP::Post.new(auth_uri.path)
159
- request.set_form_data(auth_data)
160
- request['Content-Type'] = "application/x-www-form-urlencoded"
161
- resp = https.request(request)
162
- log.debug("#{resp.body}")
163
- bcdb_response = JSON.parse(resp.body)
164
- if bcdb_response["code"] == 5000
165
- status = false
166
- log.error("Authentification failed please check your credentials")
167
- else
168
- @token_oauth = bcdb_response['access_token']
169
- @expires_token = Time.now.utc + bcdb_response['expires_in'].to_i
170
- end
171
- end
172
- return status
173
- end
174
-
175
- def bcdb_update_schema(data, cached_keys=false)
176
- schema_uri = URI.parse(@create_schema_url)
177
- schema_properties = {}
178
- data.each do |key|
179
- log.debug("KEY #{key.inspect}")
180
- schema_properties["#{key}"] = {
181
- :"$id" => "/properties/#{schema_properties["#{key}"]}",
182
- :type => "string",
183
- :title => "The #{schema_properties["#{key}"]} Schema"
184
- }
185
- end
186
- schema_data = {
187
- :type => "object",
188
- :"$id" => @bcdb_entity,
189
- :"$schema" => "http://json-schema.org/draft-07/schema#",
190
- :title => "The Root Schema",
191
- :properties => schema_properties,
192
- :autoId => true
193
- }
194
- body = JSON(schema_data)
195
-
196
- if cached_keys
197
- request = bcdb_url(schema_uri,'put', body)
198
- else
199
- request = bcdb_url(schema_uri,'post',body)
200
- if JSON.parse(request.body)["code"] == 4009
201
- request = bcdb_url(schema_uri,'put', body)
202
- end
203
- end
204
- log.debug("UPDATE SCHEMA: #{body}")
205
-
206
- log.debug("UPDATE SCHEMA RESPONSE: #{request.body}")
207
- return data, true
208
- end
209
- def bcdb_url(uri,type,body)
210
- bcdb_request = Net::HTTP.new(uri.host,uri.port)
211
- bcdb_request.use_ssl = uri.scheme == 'https'
212
- case type
213
- when 'post'
214
- request = Net::HTTP::Post.new(uri.path)
215
- when 'put'
216
- request = Net::HTTP::Put.new(uri.path)
217
- end
218
- request.body = body
219
- request['Content-Type'] = "application/json"
220
- request['authorization'] = "Bearer #{@token_oauth}"
221
- response = bcdb_request.request(request)
222
- return response
223
- end
224
-
225
- def start
226
- super
227
- end
228
-
229
- def shutdown
230
- super
231
- end
232
-
233
- def format_url(tag, time, record)
234
- @base_url
235
- end
236
-
237
- def set_body(req, tag, time, record)
238
- if @serializer == :json
239
- set_json_body(req, record)
240
- elsif @serializer == :text
241
- set_text_body(req, record)
242
- elsif @serializer == :raw
243
- set_raw_body(req, record)
244
- elsif @serializer == :x_ndjson
245
- set_bulk_body(req, record)
246
- else
247
- req.set_form_data(record)
248
- end
249
- req
250
- end
251
-
252
- def set_header(req, tag, time, record)
253
- if @custom_headers
254
- @custom_headers.each do |k,v|
255
- req[k] = v
256
- end
257
- req
258
- else
259
- req
260
- end
261
- end
262
-
263
- def compress_body(req, data)
264
- return unless @compress_request
265
- gz = Zlib::GzipWriter.new(StringIO.new)
266
- gz << data
267
-
268
- req['Content-Encoding'] = "gzip"
269
- req.body = gz.close.string
270
- end
271
-
272
- def set_json_body(req, data)
273
- bcdb_authorise()
274
- unless @cached_keys && @keys.sort == data.keys.sort
275
- @keys, @cached_keys = bcdb_update_schema(data.keys, @cached_keys)
276
- end
277
- # data = { :records => [data] } if @bulk_request
278
- req.body = Yajl.dump(data)
279
- req['Content-Type'] = "application/json"
280
- compress_body(req, req.body)
281
- end
282
-
283
- def set_text_body(req, data)
284
- req.body = data["message"]
285
- req['Content-Type'] = 'text/plain'
286
- compress_body(req, req.body)
287
- end
288
-
289
- def set_raw_body(req, data)
290
- req.body = data.to_s
291
- req['Content-Type'] = 'application/octet-stream'
292
- compress_body(req, req.body)
293
- end
294
-
295
- def set_bulk_body(req, data)
296
- bcdb_authorise()
297
- if data.is_a? String
298
- flat_keys = []
299
- bcdb_data = data.split("\n").map{ |x| JSON.parse(x) }
300
- bcdb_data.each do |data|
301
- flat_keys = flat_keys + data.keys
302
- end
303
- flat_keys.uniq!
304
- unless @cached_keys && @keys.sort == flat_keys.sort
305
- @keys, @cached_keys = bcdb_update_schema(flat_keys, @cached_keys)
306
- end
307
- data = { :records => bcdb_data }
308
- @base_url = "#{@base_url_}" + "/data/bulk/" + "#{@bcdb_entity}"
309
- else
310
- log.debug("DATA: #{data.inspect}")
311
- unless @cached_keys && @keys.sort == data.keys.sort
312
- @keys, @cached_keys = bcdb_update_schema(data.keys, @cached_keys)
313
- end
314
- data = { :records => [data] }
315
- end
316
- req.body = Yajl.dump(data)
317
- # req['Content-Type'] = 'application/x-ndjson'
318
- req['Content-Type'] = 'application/json'
319
- compress_body(req, req.body)
320
- end
321
-
322
- def create_request(tag, time, record)
323
- url = format_url(tag, time, record)
324
- uri = URI.parse(url)
325
- req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.request_uri)
326
- set_body(req, tag, time, record)
327
- set_header(req, tag, time, record)
328
- log.trace("CREATE REQUEST: #{req}, #{uri}")
329
- return req, uri
330
- end
331
-
332
- def http_opts(uri)
333
- opts = {
334
- :use_ssl => uri.scheme == 'https'
335
- }
336
- opts[:verify_mode] = @ssl_verify_mode if opts[:use_ssl]
337
- opts[:ca_file] = File.join(@ca_file) if File.file?(@ca_file)
338
- opts[:cert] = OpenSSL::X509::Certificate.new(File.read(@client_cert_path)) if File.file?(@client_cert_path)
339
- opts[:key] = OpenSSL::PKey.read(File.read(@private_key_path), @private_key_passphrase) if File.file?(@private_key_path)
340
- opts
341
- end
342
-
343
- def proxies
344
- ENV['HTTPS_PROXY'] || ENV['HTTP_PROXY'] || ENV['http_proxy'] || ENV['https_proxy']
345
- end
346
-
347
- def send_request(req, uri)
348
- is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
349
- if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
350
- log.info('Dropped request due to rate limiting')
351
- return
352
- end
353
-
354
- res = nil
355
-
356
- begin
357
- if @authentication == :basic
358
- req.basic_auth(@username, @password)
359
- elsif @authentication == :bearer
360
- req['authorization'] = "bearer #{@token}"
361
- elsif @authentication == :jwt
362
- req['authorization'] = "jwt #{@token}"
363
- elsif @authentication == :oauth
364
- req['authorization'] = "Bearer #{@token_oauth}"
365
- end
366
- @last_request_time = Time.now.to_f
367
-
368
- if proxy = proxies
369
- proxy_uri = URI.parse(proxy)
370
-
371
- res = Net::HTTP.start(uri.host, uri.port,
372
- proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password,
373
- **http_opts(uri)) {|http| http.request(req) }
374
- else
375
- res = Net::HTTP.start(uri.host, uri.port, **http_opts(uri)) {|http| http.request(req) }
376
- log.debug("REQUEST BODY: #{req.body}")
377
- log.debug("RESPONSE BODY: #{res.body}")
378
- end
379
- rescue => e # rescue all StandardErrors
380
- # server didn't respond
381
- log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{e.class}, '#{e.message}'"
382
- raise e if @raise_on_error
383
- else
384
- unless res and res.is_a?(Net::HTTPSuccess)
385
- res_summary = if res
386
- "#{res.code} #{res.message} #{res.body}"
387
- else
388
- "res=nil"
389
- end
390
- if @recoverable_status_codes.include?(res.code.to_i)
391
- raise RecoverableResponse, res_summary
392
- else
393
- log.warn "failed to #{req.method} #{uri} (#{res_summary})"
394
- end
395
- end #end unless
396
- end # end begin
397
- end # end send_request
398
-
399
- def handle_record(tag, time, record)
400
- if @formatter_config
401
- record = @formatter.format(tag, time, record)
402
- end
403
- req, uri = create_request(tag, time, record)
404
- send_request(req, uri)
405
- end
406
-
407
- def handle_records(tag, time, chunk)
408
- req, uri = create_request(tag, time, chunk.read)
409
- send_request(req, uri)
410
- end
411
-
412
- def prefer_buffered_processing
413
- @buffered
414
- end
415
-
416
- def format(tag, time, record)
417
- # For safety.
418
- end
419
-
420
- def split_request_format(tag, time, record)
421
- [time, record].to_msgpack
422
- end
423
-
424
- def bulk_request_format(tag, time, record)
425
- @formatter.format(tag, time, record)
426
- end
427
-
428
- def formatted_to_msgpack_binary?
429
- if @bulk_request
430
- false
431
- else
432
- true
433
- end
434
- end
435
-
436
- def multi_workers_ready?
437
- true
438
- end
439
-
440
- def process(tag, es)
441
- log.trace("TRACE PROCESS: #{tag}, #{es}")
442
- es.each do |time, record|
443
- handle_record(tag, time, record)
444
- end
445
- end
446
-
447
- def write(chunk)
448
- tag = chunk.metadata.tag
449
- @base_url = extract_placeholders(@base_url, chunk)
450
- if @bulk_request
451
- time = Fluent::Engine.now
452
- handle_records(tag, time, chunk)
453
- else
454
- chunk.msgpack_each do |time, record|
455
- handle_record(tag, time, record)
456
- end
457
- end
458
- end
459
- end
1
+ #
2
+ # Copyright ©2019. MODEX (Gibraltar) LIMITED
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ require 'net/http'
17
+ require 'uri'
18
+ require 'yajl'
19
+ require 'fluent/plugin/output'
20
+ require "fluent/plugin/bcdb/version"
21
+ require 'tempfile'
22
+ require 'openssl'
23
+ require 'zlib'
24
+
25
+ class BcdbOut < Fluent::Plugin::Output
26
+ Fluent::Plugin.register_output('bcdb', self)
27
+
28
+ class RecoverableResponse < StandardError; end
29
+
30
+ helpers :compat_parameters, :formatter
31
+
32
+ DEFAULT_BUFFER_TYPE = "memory"
33
+ DEFAULT_FORMATTER = "json"
34
+
35
+ def initialize
36
+ super
37
+ end
38
+
39
+ # BCDB data endpoint
40
+ config_param :base_url, :string
41
+
42
+ # BCDB Auth endpoint
43
+ config_param :auth_url, :string
44
+
45
+ # BCDB database entity model name
46
+ config_param :bcdb_entity, :string, :default => 'loglines'
47
+
48
+ # Set Net::HTTP.verify_mode to `OpenSSL::SSL::VERIFY_NONE`
49
+ config_param :ssl_no_verify, :bool, :default => false
50
+
51
+ # HTTP method
52
+ config_param :http_method, :enum, list: [:get, :put, :post, :delete], :default => :post
53
+
54
+ # form | json | text | raw
55
+ config_param :serializer, :enum, list: [:json, :form, :text, :raw], :default => :json
56
+
57
+ # Simple rate limiting: ignore any records within `rate_limit_msec`
58
+ # since the last one.
59
+ config_param :rate_limit_msec, :integer, :default => 0
60
+
61
+ # Raise errors that were rescued during HTTP requests?
62
+ config_param :raise_on_error, :bool, :default => true
63
+
64
+ # Specify recoverable error codes
65
+ config_param :recoverable_status_codes, :array, value_type: :integer, default: [503]
66
+
67
+ # ca file to use for https request
68
+ config_param :cacert_file, :string, :default => ''
69
+
70
+ # specify client sertificate
71
+ config_param :client_cert_path, :string, :default => ''
72
+
73
+ # specify private key path
74
+ config_param :private_key_path, :string, :default => ''
75
+
76
+ # specify private key passphrase
77
+ config_param :private_key_passphrase, :string, :default => '', :secret => true
78
+
79
+ # custom headers
80
+ config_param :custom_headers, :hash, :default => nil
81
+
82
+ # 'none' | 'basic' | 'jwt' | 'bearer'
83
+ config_param :authentication, :enum, list: [:none, :basic, :jwt, :bearer, :oauth], :default => :oauth
84
+ config_param :username, :string, :default => ''
85
+ config_param :password, :string, :default => '', :secret => true
86
+ config_param :client_id, :string, :default => ''
87
+ config_param :client_secret, :string, :default => '', :secret => true
88
+ config_param :grant_type, :enum, list: [:password, :authorization_code], :default => :password
89
+ config_param :token, :string, :default => ''
90
+ # Switch non-buffered/buffered plugin
91
+ config_param :buffered, :bool, :default => false
92
+ config_param :bulk_request, :bool, :default => false
93
+ # Compress with gzip except for form serializer
94
+ config_param :compress_request, :bool, :default => false
95
+
96
+ config_section :buffer do
97
+ config_set_default :@type, DEFAULT_BUFFER_TYPE
98
+ config_set_default :chunk_keys, ['tag']
99
+ end
100
+
101
+ config_section :format do
102
+ config_set_default :@type, DEFAULT_FORMATTER
103
+ end
104
+
105
+ def configure(conf)
106
+ compat_parameters_convert(conf, :buffer, :formatter)
107
+ super
108
+ @create_schema_url = "#{@base_url}" + "/data/catalog/_JsonSchema/" + "#{@bcdb_entity}"
109
+ if @bulk_request
110
+ @base_url = "#{@base_url}" + "/data/bulkAsync/" + "#{@bcdb_entity}"
111
+ else
112
+ @base_url = "#{@base_url}" + "/data/" + "#{@bcdb_entity}"
113
+ end
114
+
115
+ bcdb_authorise() if @authentication == :oauth
116
+
117
+ @ssl_verify_mode = if @ssl_no_verify
118
+ OpenSSL::SSL::VERIFY_NONE
119
+ else
120
+ OpenSSL::SSL::VERIFY_PEER
121
+ end
122
+
123
+ @ca_file = @cacert_file
124
+ @last_request_time = nil
125
+ raise Fluent::ConfigError, "'tag' in chunk_keys is required." if !@chunk_key_tag && @buffered
126
+
127
+ if @formatter_config = conf.elements('format').first
128
+ @formatter = formatter_create
129
+ end
130
+
131
+ if @bulk_request
132
+ class << self
133
+ alias_method :format, :bulk_request_format
134
+ end
135
+ @formatter = formatter_create(type: :json)
136
+ @serializer = :x_ndjson # secret settings for bulk_request
137
+ else
138
+ class << self
139
+ alias_method :format, :split_request_format
140
+ end
141
+ end
142
+ end
143
+
144
+ def bcdb_authorise()
145
+ auth_uri = URI.parse(@auth_url)
146
+ auth_data = {
147
+ :username => @username,
148
+ :password => @password,
149
+ :client_id => @client_id,
150
+ :client_secret => @client_secret,
151
+ :grant_type => @grant_type
152
+ }
153
+ status = true
154
+ begin
155
+ unless (@token_oauth && (@expires_token && Time.now.utc > @expires_token))
156
+ https= Net::HTTP.new(auth_uri.host,auth_uri.port)
157
+ https.use_ssl = auth_uri.scheme == 'https'
158
+
159
+ request = Net::HTTP::Post.new(auth_uri.path)
160
+ request.set_form_data(auth_data)
161
+ request['Content-Type'] = "application/x-www-form-urlencoded"
162
+ resp = https.request(request)
163
+ log.debug("#{resp.body}")
164
+ bcdb_response = JSON.parse(resp.body)
165
+ if bcdb_response["code"] == 5000
166
+ status = false
167
+ log.error("Authentification failed please check your credentials")
168
+ else
169
+ @token_oauth = bcdb_response['access_token']
170
+ @expires_token = Time.now.utc + bcdb_response['expires_in'].to_i
171
+ end
172
+ end
173
+ rescue => e
174
+ # This should never happen unless there's a flat out issue with the network
175
+ log.error("Error Makeing Authorization Request to BCDB. Error: #{e.message} | Backtrace: #{e.backtrace}")
176
+ sleep(2)
177
+ bcdb_authorise()
178
+ end
179
+ return status
180
+ end
181
+
182
+ def bcdb_update_schema(data, cached_keys=false)
183
+ schema_uri = URI.parse(@create_schema_url)
184
+ schema_properties = {}
185
+ data.each do |key|
186
+ log.debug("KEY #{key.inspect}")
187
+ schema_properties["#{key}"] = {
188
+ :"$id" => "/properties/#{schema_properties["#{key}"]}",
189
+ :type => "string",
190
+ :title => "The #{schema_properties["#{key}"]} Schema"
191
+ }
192
+ end
193
+ schema_data = {
194
+ :type => "object",
195
+ :"$id" => @bcdb_entity,
196
+ :"$schema" => "http://json-schema.org/draft-07/schema#",
197
+ :title => "The Root Schema",
198
+ :properties => schema_properties,
199
+ :autoId => true
200
+ }
201
+ body = JSON(schema_data)
202
+
203
+ if cached_keys
204
+ request = bcdb_url(schema_uri,'put', body)
205
+ else
206
+ request = bcdb_url(schema_uri,'post',body)
207
+ res = JSON.parse(request.body)["code"]
208
+ if res == 4009 || res == 4000
209
+ request = bcdb_url(schema_uri,'put', body)
210
+ end
211
+ end
212
+ log.debug("UPDATE SCHEMA: #{body}")
213
+
214
+ log.debug("UPDATE SCHEMA RESPONSE: #{request.body}")
215
+ return data, true
216
+ end
217
+ def bcdb_url(uri,type,body)
218
+ bcdb_request = Net::HTTP.new(uri.host,uri.port)
219
+ bcdb_request.use_ssl = uri.scheme == 'https'
220
+ case type
221
+ when 'post'
222
+ request = Net::HTTP::Post.new(uri.path)
223
+ when 'put'
224
+ request = Net::HTTP::Put.new(uri.path)
225
+ end
226
+ request.body = body
227
+ request['Content-Type'] = "application/json"
228
+ request['authorization'] = "Bearer #{@token_oauth}"
229
+ response = bcdb_request.request(request)
230
+ return response
231
+ end
232
+
233
+ def start
234
+ super
235
+ end
236
+
237
+ def shutdown
238
+ super
239
+ end
240
+
241
+ def format_url(tag, time, record)
242
+ @base_url
243
+ end
244
+
245
+ def set_body(req, tag, time, record)
246
+ if @serializer == :json
247
+ set_json_body(req, record)
248
+ elsif @serializer == :text
249
+ set_text_body(req, record)
250
+ elsif @serializer == :raw
251
+ set_raw_body(req, record)
252
+ elsif @serializer == :x_ndjson
253
+ set_bulk_body(req, record)
254
+ else
255
+ req.set_form_data(record)
256
+ end
257
+ req
258
+ end
259
+
260
+ def set_header(req, tag, time, record)
261
+ if @custom_headers
262
+ @custom_headers.each do |k,v|
263
+ req[k] = v
264
+ end
265
+ req
266
+ else
267
+ req
268
+ end
269
+ end
270
+
271
+ def compress_body(req, data)
272
+ return unless @compress_request
273
+ gz = Zlib::GzipWriter.new(StringIO.new)
274
+ gz << data
275
+
276
+ req['Content-Encoding'] = "gzip"
277
+ req.body = gz.close.string
278
+ end
279
+
280
+ def set_json_body(req, data)
281
+ bcdb_authorise()
282
+ unless @cached_keys && @keys.sort == data.keys.sort
283
+ @keys, @cached_keys = bcdb_update_schema(data.keys, @cached_keys)
284
+ end
285
+ # data = { :records => [data] } if @bulk_request
286
+ req.body = Yajl.dump(data)
287
+ req['Content-Type'] = "application/json"
288
+ compress_body(req, req.body)
289
+ end
290
+
291
+ def set_text_body(req, data)
292
+ req.body = data["message"]
293
+ req['Content-Type'] = 'text/plain'
294
+ compress_body(req, req.body)
295
+ end
296
+
297
+ def set_raw_body(req, data)
298
+ req.body = data.to_s
299
+ req['Content-Type'] = 'application/octet-stream'
300
+ compress_body(req, req.body)
301
+ end
302
+
303
+ def set_bulk_body(req, data)
304
+ bcdb_authorise()
305
+ if data.is_a? String
306
+ flat_keys = []
307
+ bcdb_data = data.split("\n").map{ |x| JSON.parse(x) }
308
+ bcdb_data.each do |data|
309
+ flat_keys = flat_keys + data.keys
310
+ end
311
+ flat_keys.uniq!
312
+ unless @cached_keys && @keys.sort == flat_keys.sort
313
+ @keys, @cached_keys = bcdb_update_schema(flat_keys, @cached_keys)
314
+ end
315
+ data = { :records => bcdb_data }
316
+ @base_url = "#{@base_url_}" + "/data/bulkAsync/" + "#{@bcdb_entity}"
317
+ else
318
+ log.debug("DATA: #{data.inspect}")
319
+ unless @cached_keys && @keys.sort == data.keys.sort
320
+ @keys, @cached_keys = bcdb_update_schema(data.keys, @cached_keys)
321
+ end
322
+ data = { :records => [data] }
323
+ end
324
+ req.body = Yajl.dump(data)
325
+ # req['Content-Type'] = 'application/x-ndjson'
326
+ req['Content-Type'] = 'application/json'
327
+ compress_body(req, req.body)
328
+ end
329
+
330
+ def create_request(tag, time, record)
331
+ url = format_url(tag, time, record)
332
+ uri = URI.parse(url)
333
+ req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.request_uri)
334
+ set_body(req, tag, time, record)
335
+ set_header(req, tag, time, record)
336
+ log.trace("CREATE REQUEST: #{req}, #{uri}")
337
+ return req, uri
338
+ end
339
+
340
+ def http_opts(uri)
341
+ opts = {
342
+ :use_ssl => uri.scheme == 'https'
343
+ }
344
+ opts[:verify_mode] = @ssl_verify_mode if opts[:use_ssl]
345
+ opts[:ca_file] = File.join(@ca_file) if File.file?(@ca_file)
346
+ opts[:cert] = OpenSSL::X509::Certificate.new(File.read(@client_cert_path)) if File.file?(@client_cert_path)
347
+ opts[:key] = OpenSSL::PKey.read(File.read(@private_key_path), @private_key_passphrase) if File.file?(@private_key_path)
348
+ opts
349
+ end
350
+
351
+ def proxies
352
+ ENV['HTTPS_PROXY'] || ENV['HTTP_PROXY'] || ENV['http_proxy'] || ENV['https_proxy']
353
+ end
354
+
355
+ def send_request(req, uri)
356
+ is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
357
+ if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
358
+ log.info('Dropped request due to rate limiting')
359
+ return
360
+ end
361
+
362
+ res = nil
363
+
364
+ begin
365
+ if @authentication == :basic
366
+ req.basic_auth(@username, @password)
367
+ elsif @authentication == :bearer
368
+ req['authorization'] = "bearer #{@token}"
369
+ elsif @authentication == :jwt
370
+ req['authorization'] = "jwt #{@token}"
371
+ elsif @authentication == :oauth
372
+ req['authorization'] = "Bearer #{@token_oauth}"
373
+ end
374
+ @last_request_time = Time.now.to_f
375
+
376
+ if proxy = proxies
377
+ proxy_uri = URI.parse(proxy)
378
+
379
+ res = Net::HTTP.start(uri.host, uri.port,
380
+ proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password,
381
+ **http_opts(uri)) {|http| http.request(req) }
382
+ else
383
+ res = Net::HTTP.start(uri.host, uri.port, **http_opts(uri)) {|http| http.request(req) }
384
+ log.debug("REQUEST BODY: #{req.body}")
385
+ log.debug("RESPONSE BODY: #{res.body}")
386
+ end
387
+ rescue => e # rescue all StandardErrors
388
+ # server didn't respond
389
+ log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{e.class}, '#{e.message}'"
390
+ raise e if @raise_on_error
391
+ else
392
+ unless res and res.is_a?(Net::HTTPSuccess)
393
+ res_summary = if res
394
+ "#{res.code} #{res.message} #{res.body}"
395
+ else
396
+ "res=nil"
397
+ end
398
+ if @recoverable_status_codes.include?(res.code.to_i)
399
+ raise RecoverableResponse, res_summary
400
+ else
401
+ log.warn "failed to #{req.method} #{uri} (#{res_summary})"
402
+ end
403
+ end #end unless
404
+ end # end begin
405
+ end # end send_request
406
+
407
+ def handle_record(tag, time, record)
408
+ if @formatter_config
409
+ record = @formatter.format(tag, time, record)
410
+ end
411
+ req, uri = create_request(tag, time, record)
412
+ send_request(req, uri)
413
+ end
414
+
415
+ def handle_records(tag, time, chunk)
416
+ req, uri = create_request(tag, time, chunk.read)
417
+ send_request(req, uri)
418
+ end
419
+
420
+ def prefer_buffered_processing
421
+ @buffered
422
+ end
423
+
424
+ def format(tag, time, record)
425
+ # For safety.
426
+ end
427
+
428
+ def split_request_format(tag, time, record)
429
+ [time, record].to_msgpack
430
+ end
431
+
432
+ def bulk_request_format(tag, time, record)
433
+ @formatter.format(tag, time, record)
434
+ end
435
+
436
+ def formatted_to_msgpack_binary?
437
+ if @bulk_request
438
+ false
439
+ else
440
+ true
441
+ end
442
+ end
443
+
444
+ def multi_workers_ready?
445
+ true
446
+ end
447
+
448
+ def process(tag, es)
449
+ log.trace("TRACE PROCESS: #{tag}, #{es}")
450
+ es.each do |time, record|
451
+ handle_record(tag, time, record)
452
+ end
453
+ end
454
+
455
+ def write(chunk)
456
+ tag = chunk.metadata.tag
457
+ @base_url = extract_placeholders(@base_url, chunk)
458
+ if @bulk_request
459
+ time = Fluent::Engine.now
460
+ handle_records(tag, time, chunk)
461
+ else
462
+ chunk.msgpack_each do |time, record|
463
+ handle_record(tag, time, record)
464
+ end
465
+ end
466
+ end
467
+ end