fluent-plugin-bcdb 0.2.0 → 0.2.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (21) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +9 -8
  3. data/CODE_OF_CONDUCT.md +74 -74
  4. data/Gemfile +4 -4
  5. data/Gemfile.lock +51 -51
  6. data/LICENSE +204 -204
  7. data/README.md +76 -78
  8. data/Rakefile +2 -2
  9. data/bin/console +14 -14
  10. data/bin/setup +8 -8
  11. data/dev/csv-conf/AB_NYC_2019.csv +49081 -0
  12. data/dev/csv-conf/fluent.conf +39 -0
  13. data/dev/fluent.conf +37 -37
  14. data/dev/http-conf/fluent.conf +37 -0
  15. data/dev/readme.md +64 -0
  16. data/fluent-plugin-bcdb.gemspec +48 -48
  17. data/lib/fluent/plugin/bcdb/version.rb +7 -7
  18. data/lib/fluent/plugin/out_bcdb.rb +467 -459
  19. metadata +20 -12
  20. data/Dockerfile +0 -28
  21. data/docker-compose.yaml +0 -38
data/dev/csv-conf/fluent.conf ADDED
@@ -0,0 +1,39 @@
1
+ <source>
2
+ @type tail
3
+ @id input1
4
+ @label @mainstream
5
+ path C:/Users/dorin.stan/Documents/workspace/0_CUSTOMERS-PEERS/Parteners/logstash-output-bcdb/AB_NYC_2019.csv
6
+ pos_file C:/Users/dorin.stan/Documents/fluentd_sincedb_path_file.pos # This is where you record file position
7
+ tag testcsv
8
+ <parse>
9
+ @type csv
10
+ delimiter (',')
11
+ keys id,name,host_id,host_name,neighbourhood_group,neighbourhood,latitude,longitude,room_type,price,minimum_nights,number_of_reviews,last_review,reviews_per_month,calculated_host_listings_count,availability_365
12
+ #time_key timestamp
13
+ </parse>
14
+ </source>
15
+
16
+ <label @mainstream>
17
+ <match **>
18
+ @type bcdb
19
+ base_url "http://10.0.11.141:32018/services/core/v1/api" # Api endpoint like https://bcdb.modex.tech/node-03/services/core/v1/api
20
+ auth_url "http://10.0.11.160:32018/services/oauth/token" # OAuth token endpoint like https://bcdb.modex.tech/oauth/token
21
+ bcdb_entity "log-csv" # default: logline
22
+ ssl_no_verify false # default: false
23
+ rate_limit_msec 0 # default: 0 = no rate limiting
24
+ raise_on_error true # default: true
25
+ recoverable_status_codes 5000, 400 # default: 503
26
+ cacert_file /etc/ssl/endpoint1.cert # default: ''
27
+ client_cert_path /path/to/client_cert.crt # default: ''
28
+ private_key_path /path/to/private_key.key # default: ''
29
+ private_key_passphrase yourpassphrase # default: ''
30
+ username admin@ibt.ro # ex. bcdb.admin@modex.tech default: ''
31
+ password 123123 # ex. admin default: '', secret: true
32
+ client_id 0x01 # default: ''. BDCB client_id
33
+ client_secret 0x000001 # default: ''. BDCB client_secret
34
+ buffered false # default: false. Switch non-buffered/buffered mode
35
+ bulk_request true # default: false. Send events as application/x-ndjson
36
+ compress_request false # default: false. Send compressed events
37
+ @log_level trace
38
+ </match>
39
+ </label>
data/dev/fluent.conf CHANGED
@@ -1,37 +1,37 @@
1
- <source>
2
- @type http
3
- @id input1
4
- @label @mainstream
5
- @log_level trace
6
- port 24224
7
- </source>
8
-
9
- <system>
10
- # equal to -qq option
11
- log_level trace
12
- </system>
13
-
14
- <label @mainstream>
15
- <match **>
16
- @type bcdb
17
- base_url "http://<your_domain_or_hostIP:port>/services/core/v1/api" # Api endpoint like https://bcdb.modex.tech/node-03/services/core/v1/api
18
- auth_url "http://<your_domain_or_hostIP:port>/token" # OAuth token endpoint like https://bcdb.modex.tech/oauth/token
19
- bcdb_entity "log" # default: logline
20
- ssl_no_verify false # default: false
21
- rate_limit_msec 100 # default: 0 = no rate limiting
22
- raise_on_error true # default: true
23
- recoverable_status_codes 5000, 400 # default: 503
24
- cacert_file /etc/ssl/endpoint1.cert # default: ''
25
- client_cert_path /path/to/client_cert.crt # default: ''
26
- private_key_path /path/to/private_key.key # default: ''
27
- private_key_passphrase yourpassphrase # default: ''
28
- username <username> # ex. bcdb.admin@modex.tech default: ''
29
- password <password> # ex. admin default: '', secret: true
30
- client_id 0x01 # default: ''. BDCB client_id
31
- client_secret 0x000001 # default: ''. BDCB client_secret
32
- buffered true # default: false. Switch non-buffered/buffered mode
33
- bulk_request true # default: false. Send events as application/x-ndjson
34
- compress_request false # default: false. Send compressed events
35
- @log_level trace
36
- </match>
37
- </label>
1
+ <source>
2
+ @type http
3
+ @id input1
4
+ @label @mainstream
5
+ @log_level trace
6
+ port 24224
7
+ </source>
8
+
9
+ <system>
10
+ # equal to -qq option
11
+ log_level trace
12
+ </system>
13
+
14
+ <label @mainstream>
15
+ <match **>
16
+ @type bcdb
17
+ base_url "http://<your_domain_or_hostIP:port>/services/core/v1/api" # Api endpoint like https://bcdb.modex.tech/node-03/services/core/v1/api
18
+ auth_url "http://<your_domain_or_hostIP:port>/token" # OAuth token endpoint like https://bcdb.modex.tech/oauth/token
19
+ bcdb_entity "log" # default: logline
20
+ ssl_no_verify false # default: false
21
+ rate_limit_msec 100 # default: 0 = no rate limiting
22
+ raise_on_error true # default: true
23
+ recoverable_status_codes 5000, 400 # default: 503
24
+ cacert_file /etc/ssl/endpoint1.cert # default: ''
25
+ client_cert_path /path/to/client_cert.crt # default: ''
26
+ private_key_path /path/to/private_key.key # default: ''
27
+ private_key_passphrase yourpassphrase # default: ''
28
+ username <username> # ex. bcdb.admin@modex.tech default: ''
29
+ password <password> # ex. admin default: '', secret: true
30
+ client_id 0x01 # default: ''. BDCB client_id
31
+ client_secret 0x000001 # default: ''. BDCB client_secret
32
+ buffered true # default: false. Switch non-buffered/buffered mode
33
+ bulk_request true # default: false. Send events as application/x-ndjson
34
+ compress_request false # default: false. Send compressed events
35
+ @log_level trace
36
+ </match>
37
+ </label>
data/dev/http-conf/fluent.conf ADDED
@@ -0,0 +1,37 @@
1
+ <source>
2
+ @type http
3
+ @id input1
4
+ @label @mainstream
5
+ @log_level trace
6
+ port 24224
7
+ </source>
8
+
9
+ <system>
10
+ # equal to -qq option
11
+ log_level trace
12
+ </system>
13
+
14
+ <label @mainstream>
15
+ <match **>
16
+ @type bcdb
17
+ base_url "http://<your_domain_or_hostIP:port>/services/core/v1/api" # Api endpoint like https://bcdb.modex.tech/node-03/services/core/v1/api
18
+ auth_url "http://<your_domain_or_hostIP:port>/token" # OAuth token endpoint like https://bcdb.modex.tech/oauth/token
19
+ bcdb_entity "log" # default: logline
20
+ ssl_no_verify false # default: false
21
+ rate_limit_msec 100 # default: 0 = no rate limiting
22
+ raise_on_error true # default: true
23
+ recoverable_status_codes 5000, 400 # default: 503
24
+ cacert_file /etc/ssl/endpoint1.cert # default: ''
25
+ client_cert_path /path/to/client_cert.crt # default: ''
26
+ private_key_path /path/to/private_key.key # default: ''
27
+ private_key_passphrase yourpassphrase # default: ''
28
+ username <username> # ex. bcdb.admin@modex.tech default: ''
29
+ password <password> # ex. admin default: '', secret: true
30
+ client_id 0x01 # default: ''. BDCB client_id
31
+ client_secret 0x000001 # default: ''. BDCB client_secret
32
+ buffered true # default: false. Switch non-buffered/buffered mode
33
+ bulk_request true # default: false. Send events as application/x-ndjson
34
+ compress_request false # default: false. Send compressed events
35
+ @log_level trace
36
+ </match>
37
+ </label>
data/dev/readme.md ADDED
@@ -0,0 +1,64 @@
1
+ # Fluentd:
2
+
3
+ ## 1. Installation steps
4
+ ---
5
+ ### **Install Ruby environment**
6
+
7
+ 1. Go to [JRuby download location]https://rubyinstaller.org/downloads/)
8
+
9
+ 1. Install from file JRuby with Dev tools as a bundle
10
+
11
+ ### **Install Fluentd**
12
+ 1. Go to [Fluentd download location]https://www.fluentd.org/download)
13
+
14
+ 2. Add Fluentd /usr/sbin folder to your PATH variable
15
+
16
+ ### **Use plugin**
17
+
18
+ 1. Build the custom plugin
19
+
20
+ ```bash
21
+ gem build fluent-plugin-bcdb
22
+ ```
23
+
24
+ 2. Install custom plugin to logstash
25
+
26
+ ```bash
27
+ gem install fluent-plugin-bcdb.gem
28
+ ```
29
+
30
+ 3. Verify installed gem plugins
31
+ ```bash
32
+ gem list
33
+ ```
34
+
35
+ 4. Edit fluent.conf with the configuration appropriate to your BCDB endpoints:
36
+ ```
37
+ <match **>
38
+ @type bcdb
39
+ base_url "http://<your_domain_or_hostIP:port>/services/core/v1/api" # Api endpoint like https://bcdb.modex.tech/node-03/services/core/v1/api
40
+ auth_url "http://<your_domain_or_hostIP:port>/token" # OAuth token endpoint like https://bcdb.modex.tech/oauth/token
41
+ bcdb_entity "log" # default: logline
42
+ ssl_no_verify false # default: false
43
+ rate_limit_msec 100 # default: 0 = no rate limiting
44
+ raise_on_error true # default: true
45
+ recoverable_status_codes 5000, 400 # default: 503
46
+ cacert_file /etc/ssl/endpoint1.cert # default: ''
47
+ client_cert_path /path/to/client_cert.crt # default: ''
48
+ private_key_path /path/to/private_key.key # default: ''
49
+ private_key_passphrase yourpassphrase # default: ''
50
+ username <username> # ex. bcdb.admin@modex.tech default: ''
51
+ password <password> # ex. admin default: '', secret: true
52
+ client_id 0x01 # default: ''. BDCB client_id
53
+ client_secret 0x000001 # default: ''. BDCB client_secret
54
+ buffered true # default: false. Switch non-buffered/buffered mode
55
+ compress_request false # default: false. Send compressed events
56
+ @log_level trace
57
+ </match>
58
+ ```
59
+
60
+ 5. Execute logstash based on logstash.conf
61
+
62
+ ```bash
63
+ fluentd -c <absolute-path>/fluent.conf
64
+ ```
data/fluent-plugin-bcdb.gemspec CHANGED
@@ -1,48 +1,48 @@
1
- #
2
- # Copyright ©2019. MODEX (Gibraltar) LIMITED
3
- #
4
- # Licensed under the Apache License, Version 2.0 (the "License");
5
- # you may not use this file except in compliance with the License.
6
- # You may obtain a copy of the License at
7
- #
8
- # http://www.apache.org/licenses/LICENSE-2.0
9
- #
10
- # Unless required by applicable law or agreed to in writing, software
11
- # distributed under the License is distributed on an "AS IS" BASIS,
12
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- # See the License for the specific language governing permissions and
14
- # limitations under the License.
15
-
16
- lib = File.expand_path("lib", __dir__)
17
- $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
18
- require "fluent/plugin/bcdb/version"
19
-
20
- Gem::Specification.new do |spec|
21
- spec.name = "fluent-plugin-bcdb"
22
- spec.version = Fluent::Plugin::Bcdb::VERSION
23
- spec.authors = ["Modex Team"]
24
- spec.email = ["support@modex.tech"]
25
-
26
- spec.summary = %q{Fluent output plugin to Modex Blockchain Database}
27
- spec.homepage = "https://github.com/messbusters/fluent-plugin-bcdb"
28
- spec.license = "APACHE"
29
-
30
- spec.metadata["homepage_uri"] = spec.homepage
31
- spec.metadata["source_code_uri"] = "https://github.com/messbusters/fluent-plugin-bcdb"
32
- spec.metadata["changelog_uri"] = "https://github.com/messbusters/fluent-plugin-bcdb"
33
-
34
- # Specify which files should be added to the gem when it is released.
35
- # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
36
- spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
37
- `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
38
- end
39
- spec.bindir = "exe"
40
- spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
41
- spec.require_paths = ["lib"]
42
-
43
- spec.add_dependency "yajl-ruby", "~> 1.0"
44
- spec.add_dependency "fluentd", [">= 0.14.22", "< 2"]
45
- spec.add_development_dependency "bundler", "~> 2.0"
46
- spec.add_development_dependency "rake", "~> 10.0"
47
- spec.add_development_dependency "test-unit", ">= 3.1.0"
48
- end
1
+ #
2
+ # Copyright ©2019. MODEX (Gibraltar) LIMITED
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ lib = File.expand_path("lib", __dir__)
17
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
18
+ require "fluent/plugin/bcdb/version"
19
+
20
+ Gem::Specification.new do |spec|
21
+ spec.name = "fluent-plugin-bcdb"
22
+ spec.version = Fluent::Plugin::Bcdb::VERSION
23
+ spec.authors = ["Modex Team"]
24
+ spec.email = ["support@modex.tech"]
25
+
26
+ spec.summary = %q{Fluent output plugin to Modex Blockchain Database}
27
+ spec.homepage = "https://github.com/modex-bcdb/fluent-plugin-bcdb"
28
+ spec.license = "APACHE"
29
+
30
+ spec.metadata["homepage_uri"] = spec.homepage
31
+ spec.metadata["source_code_uri"] = "https://github.com/modex-bcdb/fluent-plugin-bcdb"
32
+ spec.metadata["changelog_uri"] = "https://github.com/modex-bcdb/fluent-plugin-bcdb"
33
+
34
+ # Specify which files should be added to the gem when it is released.
35
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
36
+ spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
37
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
38
+ end
39
+ spec.bindir = "exe"
40
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
41
+ spec.require_paths = ["lib"]
42
+
43
+ spec.add_dependency "yajl-ruby", "~> 1.0"
44
+ spec.add_dependency "fluentd", [">= 0.14.22", "< 2"]
45
+ spec.add_development_dependency "bundler", "~> 2.0"
46
+ spec.add_development_dependency "rake", "~> 13.0"
47
+ spec.add_development_dependency "test-unit", ">= 3.1.0"
48
+ end
data/lib/fluent/plugin/bcdb/version.rb CHANGED
@@ -1,7 +1,7 @@
1
- module Fluent
2
- module Plugin
3
- module Bcdb
4
- VERSION = "0.2.0"
5
- end
6
- end
7
- end
1
+ module Fluent
2
+ module Plugin
3
+ module Bcdb
4
+ VERSION = "0.2.1"
5
+ end
6
+ end
7
+ end
data/lib/fluent/plugin/out_bcdb.rb CHANGED
@@ -1,459 +1,467 @@
1
- #
2
- # Copyright ©2019. MODEX (Gibraltar) LIMITED
3
- #
4
- # Licensed under the Apache License, Version 2.0 (the "License");
5
- # you may not use this file except in compliance with the License.
6
- # You may obtain a copy of the License at
7
- #
8
- # http://www.apache.org/licenses/LICENSE-2.0
9
- #
10
- # Unless required by applicable law or agreed to in writing, software
11
- # distributed under the License is distributed on an "AS IS" BASIS,
12
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- # See the License for the specific language governing permissions and
14
- # limitations under the License.
15
-
16
- require 'net/http'
17
- require 'uri'
18
- require 'yajl'
19
- require 'fluent/plugin/output'
20
- require "fluent/plugin/bcdb/version"
21
- require 'tempfile'
22
- require 'openssl'
23
- require 'zlib'
24
-
25
- class BcdbOut < Fluent::Plugin::Output
26
- Fluent::Plugin.register_output('bcdb', self)
27
-
28
- class RecoverableResponse < StandardError; end
29
-
30
- helpers :compat_parameters, :formatter
31
-
32
- DEFAULT_BUFFER_TYPE = "memory"
33
- DEFAULT_FORMATTER = "json"
34
-
35
- def initialize
36
- super
37
- end
38
-
39
- # BCDB data endpoint
40
- config_param :base_url, :string
41
-
42
- # BCDB Auth endpoint
43
- config_param :auth_url, :string
44
-
45
- # BCDB database entity model name
46
- config_param :bcdb_entity, :string, :default => 'loglines'
47
-
48
- # Set Net::HTTP.verify_mode to `OpenSSL::SSL::VERIFY_NONE`
49
- config_param :ssl_no_verify, :bool, :default => false
50
-
51
- # HTTP method
52
- config_param :http_method, :enum, list: [:get, :put, :post, :delete], :default => :post
53
-
54
- # form | json | text | raw
55
- config_param :serializer, :enum, list: [:json, :form, :text, :raw], :default => :json
56
-
57
- # Simple rate limiting: ignore any records within `rate_limit_msec`
58
- # since the last one.
59
- config_param :rate_limit_msec, :integer, :default => 0
60
-
61
- # Raise errors that were rescued during HTTP requests?
62
- config_param :raise_on_error, :bool, :default => true
63
-
64
- # Specify recoverable error codes
65
- config_param :recoverable_status_codes, :array, value_type: :integer, default: [503]
66
-
67
- # ca file to use for https request
68
- config_param :cacert_file, :string, :default => ''
69
-
70
- # specify client sertificate
71
- config_param :client_cert_path, :string, :default => ''
72
-
73
- # specify private key path
74
- config_param :private_key_path, :string, :default => ''
75
-
76
- # specify private key passphrase
77
- config_param :private_key_passphrase, :string, :default => '', :secret => true
78
-
79
- # custom headers
80
- config_param :custom_headers, :hash, :default => nil
81
-
82
- # 'none' | 'basic' | 'jwt' | 'bearer'
83
- config_param :authentication, :enum, list: [:none, :basic, :jwt, :bearer, :oauth], :default => :oauth
84
- config_param :username, :string, :default => ''
85
- config_param :password, :string, :default => '', :secret => true
86
- config_param :client_id, :string, :default => ''
87
- config_param :client_secret, :string, :default => '', :secret => true
88
- config_param :grant_type, :enum, list: [:password, :authorization_code], :default => :password
89
- config_param :token, :string, :default => ''
90
- # Switch non-buffered/buffered plugin
91
- config_param :buffered, :bool, :default => false
92
- config_param :bulk_request, :bool, :default => false
93
- # Compress with gzip except for form serializer
94
- config_param :compress_request, :bool, :default => false
95
-
96
- config_section :buffer do
97
- config_set_default :@type, DEFAULT_BUFFER_TYPE
98
- config_set_default :chunk_keys, ['tag']
99
- end
100
-
101
- config_section :format do
102
- config_set_default :@type, DEFAULT_FORMATTER
103
- end
104
-
105
- def configure(conf)
106
- compat_parameters_convert(conf, :buffer, :formatter)
107
- super
108
- @create_schema_url = "#{@base_url}" + "/catalog/_JsonSchema/" + "#{@bcdb_entity}"
109
- if @bulk_request
110
- @base_url = "#{@base_url}" + "/data/bulk/" + "#{@bcdb_entity}"
111
- else
112
- @base_url = "#{@base_url}" + "/data/" + "#{@bcdb_entity}"
113
- end
114
-
115
- bcdb_authorise() if @authentication == :oauth
116
-
117
- @ssl_verify_mode = if @ssl_no_verify
118
- OpenSSL::SSL::VERIFY_NONE
119
- else
120
- OpenSSL::SSL::VERIFY_PEER
121
- end
122
-
123
- @ca_file = @cacert_file
124
- @last_request_time = nil
125
- raise Fluent::ConfigError, "'tag' in chunk_keys is required." if !@chunk_key_tag && @buffered
126
-
127
- if @formatter_config = conf.elements('format').first
128
- @formatter = formatter_create
129
- end
130
-
131
- if @bulk_request
132
- class << self
133
- alias_method :format, :bulk_request_format
134
- end
135
- @formatter = formatter_create(type: :json)
136
- @serializer = :x_ndjson # secret settings for bulk_request
137
- else
138
- class << self
139
- alias_method :format, :split_request_format
140
- end
141
- end
142
- end
143
-
144
- def bcdb_authorise()
145
- auth_uri = URI.parse(@auth_url)
146
- auth_data = {
147
- :username => @username,
148
- :password => @password,
149
- :client_id => @client_id,
150
- :client_secret => @client_secret,
151
- :grant_type => @grant_type
152
- }
153
- status = true
154
- unless @token_oauth || (@expires_token && Time.now.utc > @expires_token)
155
- https= Net::HTTP.new(auth_uri.host,auth_uri.port)
156
- https.use_ssl = auth_uri.scheme == 'https'
157
-
158
- request = Net::HTTP::Post.new(auth_uri.path)
159
- request.set_form_data(auth_data)
160
- request['Content-Type'] = "application/x-www-form-urlencoded"
161
- resp = https.request(request)
162
- log.debug("#{resp.body}")
163
- bcdb_response = JSON.parse(resp.body)
164
- if bcdb_response["code"] == 5000
165
- status = false
166
- log.error("Authentification failed please check your credentials")
167
- else
168
- @token_oauth = bcdb_response['access_token']
169
- @expires_token = Time.now.utc + bcdb_response['expires_in'].to_i
170
- end
171
- end
172
- return status
173
- end
174
-
175
- def bcdb_update_schema(data, cached_keys=false)
176
- schema_uri = URI.parse(@create_schema_url)
177
- schema_properties = {}
178
- data.each do |key|
179
- log.debug("KEY #{key.inspect}")
180
- schema_properties["#{key}"] = {
181
- :"$id" => "/properties/#{schema_properties["#{key}"]}",
182
- :type => "string",
183
- :title => "The #{schema_properties["#{key}"]} Schema"
184
- }
185
- end
186
- schema_data = {
187
- :type => "object",
188
- :"$id" => @bcdb_entity,
189
- :"$schema" => "http://json-schema.org/draft-07/schema#",
190
- :title => "The Root Schema",
191
- :properties => schema_properties,
192
- :autoId => true
193
- }
194
- body = JSON(schema_data)
195
-
196
- if cached_keys
197
- request = bcdb_url(schema_uri,'put', body)
198
- else
199
- request = bcdb_url(schema_uri,'post',body)
200
- if JSON.parse(request.body)["code"] == 4009
201
- request = bcdb_url(schema_uri,'put', body)
202
- end
203
- end
204
- log.debug("UPDATE SCHEMA: #{body}")
205
-
206
- log.debug("UPDATE SCHEMA RESPONSE: #{request.body}")
207
- return data, true
208
- end
209
- def bcdb_url(uri,type,body)
210
- bcdb_request = Net::HTTP.new(uri.host,uri.port)
211
- bcdb_request.use_ssl = uri.scheme == 'https'
212
- case type
213
- when 'post'
214
- request = Net::HTTP::Post.new(uri.path)
215
- when 'put'
216
- request = Net::HTTP::Put.new(uri.path)
217
- end
218
- request.body = body
219
- request['Content-Type'] = "application/json"
220
- request['authorization'] = "Bearer #{@token_oauth}"
221
- response = bcdb_request.request(request)
222
- return response
223
- end
224
-
225
- def start
226
- super
227
- end
228
-
229
- def shutdown
230
- super
231
- end
232
-
233
- def format_url(tag, time, record)
234
- @base_url
235
- end
236
-
237
- def set_body(req, tag, time, record)
238
- if @serializer == :json
239
- set_json_body(req, record)
240
- elsif @serializer == :text
241
- set_text_body(req, record)
242
- elsif @serializer == :raw
243
- set_raw_body(req, record)
244
- elsif @serializer == :x_ndjson
245
- set_bulk_body(req, record)
246
- else
247
- req.set_form_data(record)
248
- end
249
- req
250
- end
251
-
252
- def set_header(req, tag, time, record)
253
- if @custom_headers
254
- @custom_headers.each do |k,v|
255
- req[k] = v
256
- end
257
- req
258
- else
259
- req
260
- end
261
- end
262
-
263
- def compress_body(req, data)
264
- return unless @compress_request
265
- gz = Zlib::GzipWriter.new(StringIO.new)
266
- gz << data
267
-
268
- req['Content-Encoding'] = "gzip"
269
- req.body = gz.close.string
270
- end
271
-
272
- def set_json_body(req, data)
273
- bcdb_authorise()
274
- unless @cached_keys && @keys.sort == data.keys.sort
275
- @keys, @cached_keys = bcdb_update_schema(data.keys, @cached_keys)
276
- end
277
- # data = { :records => [data] } if @bulk_request
278
- req.body = Yajl.dump(data)
279
- req['Content-Type'] = "application/json"
280
- compress_body(req, req.body)
281
- end
282
-
283
- def set_text_body(req, data)
284
- req.body = data["message"]
285
- req['Content-Type'] = 'text/plain'
286
- compress_body(req, req.body)
287
- end
288
-
289
- def set_raw_body(req, data)
290
- req.body = data.to_s
291
- req['Content-Type'] = 'application/octet-stream'
292
- compress_body(req, req.body)
293
- end
294
-
295
- def set_bulk_body(req, data)
296
- bcdb_authorise()
297
- if data.is_a? String
298
- flat_keys = []
299
- bcdb_data = data.split("\n").map{ |x| JSON.parse(x) }
300
- bcdb_data.each do |data|
301
- flat_keys = flat_keys + data.keys
302
- end
303
- flat_keys.uniq!
304
- unless @cached_keys && @keys.sort == flat_keys.sort
305
- @keys, @cached_keys = bcdb_update_schema(flat_keys, @cached_keys)
306
- end
307
- data = { :records => bcdb_data }
308
- @base_url = "#{@base_url_}" + "/data/bulk/" + "#{@bcdb_entity}"
309
- else
310
- log.debug("DATA: #{data.inspect}")
311
- unless @cached_keys && @keys.sort == data.keys.sort
312
- @keys, @cached_keys = bcdb_update_schema(data.keys, @cached_keys)
313
- end
314
- data = { :records => [data] }
315
- end
316
- req.body = Yajl.dump(data)
317
- # req['Content-Type'] = 'application/x-ndjson'
318
- req['Content-Type'] = 'application/json'
319
- compress_body(req, req.body)
320
- end
321
-
322
- def create_request(tag, time, record)
323
- url = format_url(tag, time, record)
324
- uri = URI.parse(url)
325
- req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.request_uri)
326
- set_body(req, tag, time, record)
327
- set_header(req, tag, time, record)
328
- log.trace("CREATE REQUEST: #{req}, #{uri}")
329
- return req, uri
330
- end
331
-
332
- def http_opts(uri)
333
- opts = {
334
- :use_ssl => uri.scheme == 'https'
335
- }
336
- opts[:verify_mode] = @ssl_verify_mode if opts[:use_ssl]
337
- opts[:ca_file] = File.join(@ca_file) if File.file?(@ca_file)
338
- opts[:cert] = OpenSSL::X509::Certificate.new(File.read(@client_cert_path)) if File.file?(@client_cert_path)
339
- opts[:key] = OpenSSL::PKey.read(File.read(@private_key_path), @private_key_passphrase) if File.file?(@private_key_path)
340
- opts
341
- end
342
-
343
- def proxies
344
- ENV['HTTPS_PROXY'] || ENV['HTTP_PROXY'] || ENV['http_proxy'] || ENV['https_proxy']
345
- end
346
-
347
- def send_request(req, uri)
348
- is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
349
- if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
350
- log.info('Dropped request due to rate limiting')
351
- return
352
- end
353
-
354
- res = nil
355
-
356
- begin
357
- if @authentication == :basic
358
- req.basic_auth(@username, @password)
359
- elsif @authentication == :bearer
360
- req['authorization'] = "bearer #{@token}"
361
- elsif @authentication == :jwt
362
- req['authorization'] = "jwt #{@token}"
363
- elsif @authentication == :oauth
364
- req['authorization'] = "Bearer #{@token_oauth}"
365
- end
366
- @last_request_time = Time.now.to_f
367
-
368
- if proxy = proxies
369
- proxy_uri = URI.parse(proxy)
370
-
371
- res = Net::HTTP.start(uri.host, uri.port,
372
- proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password,
373
- **http_opts(uri)) {|http| http.request(req) }
374
- else
375
- res = Net::HTTP.start(uri.host, uri.port, **http_opts(uri)) {|http| http.request(req) }
376
- log.debug("REQUEST BODY: #{req.body}")
377
- log.debug("RESPONSE BODY: #{res.body}")
378
- end
379
- rescue => e # rescue all StandardErrors
380
- # server didn't respond
381
- log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{e.class}, '#{e.message}'"
382
- raise e if @raise_on_error
383
- else
384
- unless res and res.is_a?(Net::HTTPSuccess)
385
- res_summary = if res
386
- "#{res.code} #{res.message} #{res.body}"
387
- else
388
- "res=nil"
389
- end
390
- if @recoverable_status_codes.include?(res.code.to_i)
391
- raise RecoverableResponse, res_summary
392
- else
393
- log.warn "failed to #{req.method} #{uri} (#{res_summary})"
394
- end
395
- end #end unless
396
- end # end begin
397
- end # end send_request
398
-
399
- def handle_record(tag, time, record)
400
- if @formatter_config
401
- record = @formatter.format(tag, time, record)
402
- end
403
- req, uri = create_request(tag, time, record)
404
- send_request(req, uri)
405
- end
406
-
407
- def handle_records(tag, time, chunk)
408
- req, uri = create_request(tag, time, chunk.read)
409
- send_request(req, uri)
410
- end
411
-
412
- def prefer_buffered_processing
413
- @buffered
414
- end
415
-
416
- def format(tag, time, record)
417
- # For safety.
418
- end
419
-
420
- def split_request_format(tag, time, record)
421
- [time, record].to_msgpack
422
- end
423
-
424
- def bulk_request_format(tag, time, record)
425
- @formatter.format(tag, time, record)
426
- end
427
-
428
- def formatted_to_msgpack_binary?
429
- if @bulk_request
430
- false
431
- else
432
- true
433
- end
434
- end
435
-
436
- def multi_workers_ready?
437
- true
438
- end
439
-
440
- def process(tag, es)
441
- log.trace("TRACE PROCESS: #{tag}, #{es}")
442
- es.each do |time, record|
443
- handle_record(tag, time, record)
444
- end
445
- end
446
-
447
- def write(chunk)
448
- tag = chunk.metadata.tag
449
- @base_url = extract_placeholders(@base_url, chunk)
450
- if @bulk_request
451
- time = Fluent::Engine.now
452
- handle_records(tag, time, chunk)
453
- else
454
- chunk.msgpack_each do |time, record|
455
- handle_record(tag, time, record)
456
- end
457
- end
458
- end
459
- end
1
+ #
2
+ # Copyright ©2019. MODEX (Gibraltar) LIMITED
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ require 'net/http'
17
+ require 'uri'
18
+ require 'yajl'
19
+ require 'fluent/plugin/output'
20
+ require "fluent/plugin/bcdb/version"
21
+ require 'tempfile'
22
+ require 'openssl'
23
+ require 'zlib'
24
+
25
+ class BcdbOut < Fluent::Plugin::Output
26
+ Fluent::Plugin.register_output('bcdb', self)
27
+
28
+ class RecoverableResponse < StandardError; end
29
+
30
+ helpers :compat_parameters, :formatter
31
+
32
+ DEFAULT_BUFFER_TYPE = "memory"
33
+ DEFAULT_FORMATTER = "json"
34
+
35
+ def initialize
36
+ super
37
+ end
38
+
39
+ # BCDB data endpoint
40
+ config_param :base_url, :string
41
+
42
+ # BCDB Auth endpoint
43
+ config_param :auth_url, :string
44
+
45
+ # BCDB database entity model name
46
+ config_param :bcdb_entity, :string, :default => 'loglines'
47
+
48
+ # Set Net::HTTP.verify_mode to `OpenSSL::SSL::VERIFY_NONE`
49
+ config_param :ssl_no_verify, :bool, :default => false
50
+
51
+ # HTTP method
52
+ config_param :http_method, :enum, list: [:get, :put, :post, :delete], :default => :post
53
+
54
+ # form | json | text | raw
55
+ config_param :serializer, :enum, list: [:json, :form, :text, :raw], :default => :json
56
+
57
+ # Simple rate limiting: ignore any records within `rate_limit_msec`
58
+ # since the last one.
59
+ config_param :rate_limit_msec, :integer, :default => 0
60
+
61
+ # Raise errors that were rescued during HTTP requests?
62
+ config_param :raise_on_error, :bool, :default => true
63
+
64
+ # Specify recoverable error codes
65
+ config_param :recoverable_status_codes, :array, value_type: :integer, default: [503]
66
+
67
+ # ca file to use for https request
68
+ config_param :cacert_file, :string, :default => ''
69
+
70
+ # specify client sertificate
71
+ config_param :client_cert_path, :string, :default => ''
72
+
73
+ # specify private key path
74
+ config_param :private_key_path, :string, :default => ''
75
+
76
+ # specify private key passphrase
77
+ config_param :private_key_passphrase, :string, :default => '', :secret => true
78
+
79
+ # custom headers
80
+ config_param :custom_headers, :hash, :default => nil
81
+
82
+ # 'none' | 'basic' | 'jwt' | 'bearer'
83
+ config_param :authentication, :enum, list: [:none, :basic, :jwt, :bearer, :oauth], :default => :oauth
84
+ config_param :username, :string, :default => ''
85
+ config_param :password, :string, :default => '', :secret => true
86
+ config_param :client_id, :string, :default => ''
87
+ config_param :client_secret, :string, :default => '', :secret => true
88
+ config_param :grant_type, :enum, list: [:password, :authorization_code], :default => :password
89
+ config_param :token, :string, :default => ''
90
+ # Switch non-buffered/buffered plugin
91
+ config_param :buffered, :bool, :default => false
92
+ config_param :bulk_request, :bool, :default => false
93
+ # Compress with gzip except for form serializer
94
+ config_param :compress_request, :bool, :default => false
95
+
96
+ config_section :buffer do
97
+ config_set_default :@type, DEFAULT_BUFFER_TYPE
98
+ config_set_default :chunk_keys, ['tag']
99
+ end
100
+
101
+ config_section :format do
102
+ config_set_default :@type, DEFAULT_FORMATTER
103
+ end
104
+
105
+ def configure(conf)
106
+ compat_parameters_convert(conf, :buffer, :formatter)
107
+ super
108
+ @create_schema_url = "#{@base_url}" + "/data/catalog/_JsonSchema/" + "#{@bcdb_entity}"
109
+ if @bulk_request
110
+ @base_url = "#{@base_url}" + "/data/bulkAsync/" + "#{@bcdb_entity}"
111
+ else
112
+ @base_url = "#{@base_url}" + "/data/" + "#{@bcdb_entity}"
113
+ end
114
+
115
+ bcdb_authorise() if @authentication == :oauth
116
+
117
+ @ssl_verify_mode = if @ssl_no_verify
118
+ OpenSSL::SSL::VERIFY_NONE
119
+ else
120
+ OpenSSL::SSL::VERIFY_PEER
121
+ end
122
+
123
+ @ca_file = @cacert_file
124
+ @last_request_time = nil
125
+ raise Fluent::ConfigError, "'tag' in chunk_keys is required." if !@chunk_key_tag && @buffered
126
+
127
+ if @formatter_config = conf.elements('format').first
128
+ @formatter = formatter_create
129
+ end
130
+
131
+ if @bulk_request
132
+ class << self
133
+ alias_method :format, :bulk_request_format
134
+ end
135
+ @formatter = formatter_create(type: :json)
136
+ @serializer = :x_ndjson # secret settings for bulk_request
137
+ else
138
+ class << self
139
+ alias_method :format, :split_request_format
140
+ end
141
+ end
142
+ end
143
+
144
+ def bcdb_authorise()
145
+ auth_uri = URI.parse(@auth_url)
146
+ auth_data = {
147
+ :username => @username,
148
+ :password => @password,
149
+ :client_id => @client_id,
150
+ :client_secret => @client_secret,
151
+ :grant_type => @grant_type
152
+ }
153
+ status = true
154
+ begin
155
+ unless (@token_oauth && (@expires_token && Time.now.utc > @expires_token))
156
+ https= Net::HTTP.new(auth_uri.host,auth_uri.port)
157
+ https.use_ssl = auth_uri.scheme == 'https'
158
+
159
+ request = Net::HTTP::Post.new(auth_uri.path)
160
+ request.set_form_data(auth_data)
161
+ request['Content-Type'] = "application/x-www-form-urlencoded"
162
+ resp = https.request(request)
163
+ log.debug("#{resp.body}")
164
+ bcdb_response = JSON.parse(resp.body)
165
+ if bcdb_response["code"] == 5000
166
+ status = false
167
+ log.error("Authentification failed please check your credentials")
168
+ else
169
+ @token_oauth = bcdb_response['access_token']
170
+ @expires_token = Time.now.utc + bcdb_response['expires_in'].to_i
171
+ end
172
+ end
173
+ rescue => e
174
+ # This should never happen unless there's a flat out issue with the network
175
+ log.error("Error Makeing Authorization Request to BCDB. Error: #{e.message} | Backtrace: #{e.backtrace}")
176
+ sleep(2)
177
+ bcdb_authorise()
178
+ end
179
+ return status
180
+ end
181
+
182
+ def bcdb_update_schema(data, cached_keys=false)
183
+ schema_uri = URI.parse(@create_schema_url)
184
+ schema_properties = {}
185
+ data.each do |key|
186
+ log.debug("KEY #{key.inspect}")
187
+ schema_properties["#{key}"] = {
188
+ :"$id" => "/properties/#{schema_properties["#{key}"]}",
189
+ :type => "string",
190
+ :title => "The #{schema_properties["#{key}"]} Schema"
191
+ }
192
+ end
193
+ schema_data = {
194
+ :type => "object",
195
+ :"$id" => @bcdb_entity,
196
+ :"$schema" => "http://json-schema.org/draft-07/schema#",
197
+ :title => "The Root Schema",
198
+ :properties => schema_properties,
199
+ :autoId => true
200
+ }
201
+ body = JSON(schema_data)
202
+
203
+ if cached_keys
204
+ request = bcdb_url(schema_uri,'put', body)
205
+ else
206
+ request = bcdb_url(schema_uri,'post',body)
207
+ res = JSON.parse(request.body)["code"]
208
+ if res == 4009 || res == 4000
209
+ request = bcdb_url(schema_uri,'put', body)
210
+ end
211
+ end
212
+ log.debug("UPDATE SCHEMA: #{body}")
213
+
214
+ log.debug("UPDATE SCHEMA RESPONSE: #{request.body}")
215
+ return data, true
216
+ end
217
+ def bcdb_url(uri,type,body)
218
+ bcdb_request = Net::HTTP.new(uri.host,uri.port)
219
+ bcdb_request.use_ssl = uri.scheme == 'https'
220
+ case type
221
+ when 'post'
222
+ request = Net::HTTP::Post.new(uri.path)
223
+ when 'put'
224
+ request = Net::HTTP::Put.new(uri.path)
225
+ end
226
+ request.body = body
227
+ request['Content-Type'] = "application/json"
228
+ request['authorization'] = "Bearer #{@token_oauth}"
229
+ response = bcdb_request.request(request)
230
+ return response
231
+ end
232
+
233
+ def start
234
+ super
235
+ end
236
+
237
+ def shutdown
238
+ super
239
+ end
240
+
241
+ def format_url(tag, time, record)
242
+ @base_url
243
+ end
244
+
245
+ def set_body(req, tag, time, record)
246
+ if @serializer == :json
247
+ set_json_body(req, record)
248
+ elsif @serializer == :text
249
+ set_text_body(req, record)
250
+ elsif @serializer == :raw
251
+ set_raw_body(req, record)
252
+ elsif @serializer == :x_ndjson
253
+ set_bulk_body(req, record)
254
+ else
255
+ req.set_form_data(record)
256
+ end
257
+ req
258
+ end
259
+
260
+ def set_header(req, tag, time, record)
261
+ if @custom_headers
262
+ @custom_headers.each do |k,v|
263
+ req[k] = v
264
+ end
265
+ req
266
+ else
267
+ req
268
+ end
269
+ end
270
+
271
+ def compress_body(req, data)
272
+ return unless @compress_request
273
+ gz = Zlib::GzipWriter.new(StringIO.new)
274
+ gz << data
275
+
276
+ req['Content-Encoding'] = "gzip"
277
+ req.body = gz.close.string
278
+ end
279
+
280
+ def set_json_body(req, data)
281
+ bcdb_authorise()
282
+ unless @cached_keys && @keys.sort == data.keys.sort
283
+ @keys, @cached_keys = bcdb_update_schema(data.keys, @cached_keys)
284
+ end
285
+ # data = { :records => [data] } if @bulk_request
286
+ req.body = Yajl.dump(data)
287
+ req['Content-Type'] = "application/json"
288
+ compress_body(req, req.body)
289
+ end
290
+
291
+ def set_text_body(req, data)
292
+ req.body = data["message"]
293
+ req['Content-Type'] = 'text/plain'
294
+ compress_body(req, req.body)
295
+ end
296
+
297
+ def set_raw_body(req, data)
298
+ req.body = data.to_s
299
+ req['Content-Type'] = 'application/octet-stream'
300
+ compress_body(req, req.body)
301
+ end
302
+
303
+ def set_bulk_body(req, data)
304
+ bcdb_authorise()
305
+ if data.is_a? String
306
+ flat_keys = []
307
+ bcdb_data = data.split("\n").map{ |x| JSON.parse(x) }
308
+ bcdb_data.each do |data|
309
+ flat_keys = flat_keys + data.keys
310
+ end
311
+ flat_keys.uniq!
312
+ unless @cached_keys && @keys.sort == flat_keys.sort
313
+ @keys, @cached_keys = bcdb_update_schema(flat_keys, @cached_keys)
314
+ end
315
+ data = { :records => bcdb_data }
316
+ @base_url = "#{@base_url_}" + "/data/bulkAsync/" + "#{@bcdb_entity}"
317
+ else
318
+ log.debug("DATA: #{data.inspect}")
319
+ unless @cached_keys && @keys.sort == data.keys.sort
320
+ @keys, @cached_keys = bcdb_update_schema(data.keys, @cached_keys)
321
+ end
322
+ data = { :records => [data] }
323
+ end
324
+ req.body = Yajl.dump(data)
325
+ # req['Content-Type'] = 'application/x-ndjson'
326
+ req['Content-Type'] = 'application/json'
327
+ compress_body(req, req.body)
328
+ end
329
+
330
+ def create_request(tag, time, record)
331
+ url = format_url(tag, time, record)
332
+ uri = URI.parse(url)
333
+ req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.request_uri)
334
+ set_body(req, tag, time, record)
335
+ set_header(req, tag, time, record)
336
+ log.trace("CREATE REQUEST: #{req}, #{uri}")
337
+ return req, uri
338
+ end
339
+
340
+ def http_opts(uri)
341
+ opts = {
342
+ :use_ssl => uri.scheme == 'https'
343
+ }
344
+ opts[:verify_mode] = @ssl_verify_mode if opts[:use_ssl]
345
+ opts[:ca_file] = File.join(@ca_file) if File.file?(@ca_file)
346
+ opts[:cert] = OpenSSL::X509::Certificate.new(File.read(@client_cert_path)) if File.file?(@client_cert_path)
347
+ opts[:key] = OpenSSL::PKey.read(File.read(@private_key_path), @private_key_passphrase) if File.file?(@private_key_path)
348
+ opts
349
+ end
350
+
351
+ def proxies
352
+ ENV['HTTPS_PROXY'] || ENV['HTTP_PROXY'] || ENV['http_proxy'] || ENV['https_proxy']
353
+ end
354
+
355
+ def send_request(req, uri)
356
+ is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
357
+ if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
358
+ log.info('Dropped request due to rate limiting')
359
+ return
360
+ end
361
+
362
+ res = nil
363
+
364
+ begin
365
+ if @authentication == :basic
366
+ req.basic_auth(@username, @password)
367
+ elsif @authentication == :bearer
368
+ req['authorization'] = "bearer #{@token}"
369
+ elsif @authentication == :jwt
370
+ req['authorization'] = "jwt #{@token}"
371
+ elsif @authentication == :oauth
372
+ req['authorization'] = "Bearer #{@token_oauth}"
373
+ end
374
+ @last_request_time = Time.now.to_f
375
+
376
+ if proxy = proxies
377
+ proxy_uri = URI.parse(proxy)
378
+
379
+ res = Net::HTTP.start(uri.host, uri.port,
380
+ proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password,
381
+ **http_opts(uri)) {|http| http.request(req) }
382
+ else
383
+ res = Net::HTTP.start(uri.host, uri.port, **http_opts(uri)) {|http| http.request(req) }
384
+ log.debug("REQUEST BODY: #{req.body}")
385
+ log.debug("RESPONSE BODY: #{res.body}")
386
+ end
387
+ rescue => e # rescue all StandardErrors
388
+ # server didn't respond
389
+ log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{e.class}, '#{e.message}'"
390
+ raise e if @raise_on_error
391
+ else
392
+ unless res and res.is_a?(Net::HTTPSuccess)
393
+ res_summary = if res
394
+ "#{res.code} #{res.message} #{res.body}"
395
+ else
396
+ "res=nil"
397
+ end
398
+ if @recoverable_status_codes.include?(res.code.to_i)
399
+ raise RecoverableResponse, res_summary
400
+ else
401
+ log.warn "failed to #{req.method} #{uri} (#{res_summary})"
402
+ end
403
+ end #end unless
404
+ end # end begin
405
+ end # end send_request
406
+
407
+ def handle_record(tag, time, record)
408
+ if @formatter_config
409
+ record = @formatter.format(tag, time, record)
410
+ end
411
+ req, uri = create_request(tag, time, record)
412
+ send_request(req, uri)
413
+ end
414
+
415
+ def handle_records(tag, time, chunk)
416
+ req, uri = create_request(tag, time, chunk.read)
417
+ send_request(req, uri)
418
+ end
419
+
420
+ def prefer_buffered_processing
421
+ @buffered
422
+ end
423
+
424
+ def format(tag, time, record)
425
+ # For safety.
426
+ end
427
+
428
+ def split_request_format(tag, time, record)
429
+ [time, record].to_msgpack
430
+ end
431
+
432
+ def bulk_request_format(tag, time, record)
433
+ @formatter.format(tag, time, record)
434
+ end
435
+
436
+ def formatted_to_msgpack_binary?
437
+ if @bulk_request
438
+ false
439
+ else
440
+ true
441
+ end
442
+ end
443
+
444
+ def multi_workers_ready?
445
+ true
446
+ end
447
+
448
+ def process(tag, es)
449
+ log.trace("TRACE PROCESS: #{tag}, #{es}")
450
+ es.each do |time, record|
451
+ handle_record(tag, time, record)
452
+ end
453
+ end
454
+
455
+ def write(chunk)
456
+ tag = chunk.metadata.tag
457
+ @base_url = extract_placeholders(@base_url, chunk)
458
+ if @bulk_request
459
+ time = Fluent::Engine.now
460
+ handle_records(tag, time, chunk)
461
+ else
462
+ chunk.msgpack_each do |time, record|
463
+ handle_record(tag, time, record)
464
+ end
465
+ end
466
+ end
467
+ end