fluent-plugin-datadog 0.12.0 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +0 -1
- data/README.md +28 -7
- data/fluent-plugin-datadog.gemspec +9 -8
- data/lib/fluent/plugin/out_datadog.rb +48 -20
- data/lib/fluent/plugin/version.rb +5 -0
- metadata +43 -28
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 804db6997735d59d4e5cf785c13ac0743462462576931220dbc6b335048cf258
|
4
|
+
data.tar.gz: c128237af27db094d3670979dc71d70f77687d93ad16266fb1831624f70479c1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4e7b4b62f2ef22ebd0fd479cdf272acd4f972ae89466f5da499fbfe3fefef1f9c8e0864f081d1b2e6796f3a9dfde1d4c8b255be8ed899b9457c0a9a774b2d815
|
7
|
+
data.tar.gz: bb3c0f83647dd926ad24dee88728718e2c62cc20661e6da77131a472d7c9b93ceac26f00854c23834be438b3be459c3c489869eeacfc96b69da8d2b480f35987
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -1,7 +1,6 @@
|
|
1
1
|
# Fluentd output plugin for Datadog
|
2
2
|
|
3
|
-
|
4
|
-
streams logs directly to Datadog - so no need to use a log shipper
|
3
|
+
This output plugin allows sending logs directly from Fluentd to Datadog - so you don't have to use a separate log shipper
|
5
4
|
if you don't wan't to.
|
6
5
|
|
7
6
|
## Pre-requirements
|
@@ -24,7 +23,7 @@ If you installed the td-agent instead
|
|
24
23
|
|
25
24
|
To match events and send them to Datadog, simply add the following code to your configuration file.
|
26
25
|
|
27
|
-
|
26
|
+
HTTP example:
|
28
27
|
|
29
28
|
```xml
|
30
29
|
# Match events tagged with "datadog.**" and
|
@@ -40,9 +39,13 @@ TCP example:
|
|
40
39
|
tag_key 'tag'
|
41
40
|
|
42
41
|
# Optional parameters
|
43
|
-
dd_source '<INTEGRATION_NAME>'
|
44
|
-
dd_tags '<KEY1:
|
42
|
+
dd_source '<INTEGRATION_NAME>'
|
43
|
+
dd_tags '<KEY1:VALUE1>,<KEY2:VALUE2>'
|
45
44
|
dd_sourcecategory '<MY_SOURCE_CATEGORY>'
|
45
|
+
|
46
|
+
# Optional http proxy
|
47
|
+
http_proxy 'http://my-proxy.example'
|
48
|
+
|
46
49
|
<buffer>
|
47
50
|
@type memory
|
48
51
|
flush_thread_count 4
|
@@ -51,7 +54,6 @@ TCP example:
|
|
51
54
|
chunk_limit_records 500
|
52
55
|
</buffer>
|
53
56
|
|
54
|
-
|
55
57
|
</match>
|
56
58
|
```
|
57
59
|
|
@@ -98,7 +100,8 @@ As fluent-plugin-datadog is an output_buffer, you can set all output_buffer prop
|
|
98
100
|
| **dd_hostname** | Used by Datadog to identify the host submitting the logs. | `hostname -f` |
|
99
101
|
| **service** | Used by Datadog to correlate between logs, traces and metrics. | nil |
|
100
102
|
| **port** | Proxy port when logs are not directly forwarded to Datadog and ssl is not used | 80 |
|
101
|
-
| **host** | Proxy endpoint when logs are not directly forwarded to Datadog | intake.logs.datadoghq.com |
|
103
|
+
| **host** | Proxy endpoint when logs are not directly forwarded to Datadog | http-intake.logs.datadoghq.com |
|
104
|
+
| **http_proxy** | HTTP proxy, only takes effect if HTTP forwarding is enabled (`use_http`). Defaults to `HTTP_PROXY`/`http_proxy` env vars. | nil |
|
102
105
|
|
103
106
|
### Docker and Kubernetes tags
|
104
107
|
|
@@ -123,6 +126,24 @@ Configuration example:
|
|
123
126
|
</filter>
|
124
127
|
```
|
125
128
|
|
129
|
+
### Encoding
|
130
|
+
|
131
|
+
Datadog's API expects log messages to be encoded in UTF-8.
|
132
|
+
If some of your logs are encoded with a different encoding, we recommend using the [`record_modifier` filter plugin](https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding)
|
133
|
+
to encode these logs to UTF-8.
|
134
|
+
|
135
|
+
Configuration example:
|
136
|
+
|
137
|
+
```
|
138
|
+
# Change encoding of logs tagged with "datadog.**"
|
139
|
+
<filter datadog.**>
|
140
|
+
@type record_modifier
|
141
|
+
|
142
|
+
# change the encoding from the '<SOURCE_ENCODING>' of your logs to 'utf-8'
|
143
|
+
char_encoding <SOURCE_ENCODING>:utf-8
|
144
|
+
</filter>
|
145
|
+
```
|
146
|
+
|
126
147
|
## Build
|
127
148
|
|
128
149
|
To build a new version of this plugin and push it to RubyGems:
|
@@ -7,27 +7,28 @@
|
|
7
7
|
lib = File.expand_path('../lib', __FILE__)
|
8
8
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
9
9
|
|
10
|
+
require "fluent/plugin/version.rb"
|
11
|
+
|
10
12
|
Gem::Specification.new do |spec|
|
11
13
|
spec.name = "fluent-plugin-datadog"
|
12
|
-
spec.version =
|
14
|
+
spec.version = DatadogFluentPlugin::VERSION
|
13
15
|
spec.authors = ["Datadog Solutions Team"]
|
14
16
|
spec.email = ["support@datadoghq.com"]
|
15
17
|
spec.summary = "Datadog output plugin for Fluent event collector"
|
16
18
|
spec.homepage = "http://datadoghq.com"
|
17
|
-
spec.license = "Apache
|
19
|
+
spec.license = "Apache-2.0"
|
18
20
|
|
19
|
-
spec.files = [".gitignore", "Gemfile", "LICENSE", "README.md", "Rakefile", "fluent-plugin-datadog.gemspec", "lib/fluent/plugin/out_datadog.rb"]
|
21
|
+
spec.files = [".gitignore", "Gemfile", "LICENSE", "README.md", "Rakefile", "fluent-plugin-datadog.gemspec", "lib/fluent/plugin/version.rb", "lib/fluent/plugin/out_datadog.rb"]
|
20
22
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
21
23
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
22
24
|
spec.require_paths = ["lib"]
|
23
25
|
|
24
|
-
spec.add_development_dependency "bundler", "~> 1.5"
|
25
26
|
spec.add_runtime_dependency "fluentd", [">= 1", "< 2"]
|
26
|
-
spec.add_runtime_dependency "net-http-persistent", '~>
|
27
|
+
spec.add_runtime_dependency "net-http-persistent", '~> 3.1'
|
27
28
|
|
29
|
+
spec.add_development_dependency "bundler", "~> 2.1"
|
28
30
|
spec.add_development_dependency "test-unit", '~> 3.1'
|
29
|
-
spec.add_development_dependency "rake"
|
31
|
+
spec.add_development_dependency "rake", "~> 12.0"
|
30
32
|
spec.add_development_dependency "yajl-ruby", "~> 1.2"
|
31
|
-
|
32
|
-
|
33
|
+
spec.add_development_dependency 'webmock', "~> 3.6.0"
|
33
34
|
end
|
@@ -9,6 +9,8 @@ require "yajl"
|
|
9
9
|
require "zlib"
|
10
10
|
require "fluent/plugin/output"
|
11
11
|
|
12
|
+
require_relative "version"
|
13
|
+
|
12
14
|
class Fluent::DatadogOutput < Fluent::Plugin::Output
|
13
15
|
class RetryableError < StandardError;
|
14
16
|
end
|
@@ -49,12 +51,14 @@ class Fluent::DatadogOutput < Fluent::Plugin::Output
|
|
49
51
|
config_param :use_compression, :bool, :default => true
|
50
52
|
config_param :compression_level, :integer, :default => 6
|
51
53
|
config_param :no_ssl_validation, :bool, :default => false
|
54
|
+
config_param :http_proxy, :string, :default => nil
|
55
|
+
config_param :force_v1_routes, :bool, :default => false
|
52
56
|
|
53
57
|
# Format settings
|
54
58
|
config_param :use_json, :bool, :default => true
|
55
59
|
|
56
60
|
# API Settings
|
57
|
-
config_param :api_key, :string
|
61
|
+
config_param :api_key, :string, secret: true
|
58
62
|
|
59
63
|
config_section :buffer do
|
60
64
|
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
@@ -88,7 +92,7 @@ class Fluent::DatadogOutput < Fluent::Plugin::Output
|
|
88
92
|
|
89
93
|
def start
|
90
94
|
super
|
91
|
-
@client = new_client(log, @api_key, @use_http, @use_ssl, @no_ssl_validation, @host, @ssl_port, @port, @use_compression)
|
95
|
+
@client = new_client(log, @api_key, @use_http, @use_ssl, @no_ssl_validation, @host, @ssl_port, @port, @http_proxy, @use_compression, @force_v1_routes)
|
92
96
|
end
|
93
97
|
|
94
98
|
def shutdown
|
@@ -122,18 +126,22 @@ class Fluent::DatadogOutput < Fluent::Plugin::Output
|
|
122
126
|
# NOTE! This method is called by internal thread, not Fluentd's main thread.
|
123
127
|
# 'chunk' is a buffer chunk that includes multiple formatted events.
|
124
128
|
def write(chunk)
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
129
|
+
begin
|
130
|
+
if @use_http
|
131
|
+
events = Array.new
|
132
|
+
chunk.msgpack_each do |record|
|
133
|
+
next if record.empty?
|
134
|
+
events.push record[0]
|
135
|
+
end
|
136
|
+
process_http_events(events, @use_compression, @compression_level, @max_retries, @max_backoff, DD_MAX_BATCH_LENGTH, DD_MAX_BATCH_SIZE)
|
137
|
+
else
|
138
|
+
chunk.msgpack_each do |record|
|
139
|
+
next if record.empty?
|
140
|
+
process_tcp_event(record[0], @max_retries, @max_backoff, DD_MAX_BATCH_SIZE)
|
141
|
+
end
|
136
142
|
end
|
143
|
+
rescue Exception => e
|
144
|
+
log.error("Uncaught processing exception in datadog forwarder #{e.message}")
|
137
145
|
end
|
138
146
|
end
|
139
147
|
|
@@ -256,9 +264,9 @@ class Fluent::DatadogOutput < Fluent::Plugin::Output
|
|
256
264
|
end
|
257
265
|
|
258
266
|
# Build a new transport client
|
259
|
-
def new_client(logger, api_key, use_http, use_ssl, no_ssl_validation, host, ssl_port, port, use_compression)
|
267
|
+
def new_client(logger, api_key, use_http, use_ssl, no_ssl_validation, host, ssl_port, port, http_proxy, use_compression, force_v1_routes)
|
260
268
|
if use_http
|
261
|
-
DatadogHTTPClient.new logger, use_ssl, no_ssl_validation, host, ssl_port, port, use_compression, api_key
|
269
|
+
DatadogHTTPClient.new logger, use_ssl, no_ssl_validation, host, ssl_port, port, http_proxy, use_compression, api_key, force_v1_routes
|
262
270
|
else
|
263
271
|
DatadogTCPClient.new logger, use_ssl, no_ssl_validation, host, ssl_port, port
|
264
272
|
end
|
@@ -296,18 +304,37 @@ class Fluent::DatadogOutput < Fluent::Plugin::Output
|
|
296
304
|
require 'net/http'
|
297
305
|
require 'net/http/persistent'
|
298
306
|
|
299
|
-
def initialize(logger, use_ssl, no_ssl_validation, host, ssl_port, port, use_compression, api_key)
|
307
|
+
def initialize(logger, use_ssl, no_ssl_validation, host, ssl_port, port, http_proxy, use_compression, api_key, force_v1_routes = false)
|
300
308
|
@logger = logger
|
301
309
|
protocol = use_ssl ? "https" : "http"
|
302
310
|
port = use_ssl ? ssl_port : port
|
303
|
-
|
304
|
-
|
305
|
-
|
311
|
+
if force_v1_routes
|
312
|
+
@uri = URI("#{protocol}://#{host}:#{port.to_s}/v1/input/#{api_key}")
|
313
|
+
else
|
314
|
+
@uri = URI("#{protocol}://#{host}:#{port.to_s}/api/v2/logs")
|
315
|
+
end
|
316
|
+
proxy_uri = :ENV
|
317
|
+
if http_proxy
|
318
|
+
proxy_uri = URI.parse(http_proxy)
|
319
|
+
elsif ENV['HTTP_PROXY'] || ENV['http_proxy']
|
320
|
+
logger.info("Using HTTP proxy defined in `HTTP_PROXY`/`http_proxy` env vars")
|
321
|
+
end
|
322
|
+
logger.info("Starting HTTP connection to #{protocol}://#{host}:#{port.to_s} with compression " + (use_compression ? "enabled" : "disabled") + (force_v1_routes ? " using v1 routes" : " using v2 routes"))
|
323
|
+
@client = Net::HTTP::Persistent.new name: "fluent-plugin-datadog-logcollector", proxy: proxy_uri
|
306
324
|
@client.verify_mode = OpenSSL::SSL::VERIFY_NONE if no_ssl_validation
|
325
|
+
unless force_v1_routes
|
326
|
+
@client.override_headers["DD-API-KEY"] = api_key
|
327
|
+
@client.override_headers["DD-EVP-ORIGIN"] = "fluent"
|
328
|
+
@client.override_headers["DD-EVP-ORIGIN-VERSION"] = DatadogFluentPlugin::VERSION
|
329
|
+
end
|
307
330
|
@client.override_headers["Content-Type"] = "application/json"
|
308
331
|
if use_compression
|
309
332
|
@client.override_headers["Content-Encoding"] = "gzip"
|
310
333
|
end
|
334
|
+
if !@client.proxy_uri.nil?
|
335
|
+
# Log the proxy settings as resolved by the HTTP client
|
336
|
+
logger.info("Using HTTP proxy #{@client.proxy_uri.scheme}://#{@client.proxy_uri.host}:#{@client.proxy_uri.port} username: #{@client.proxy_uri.user ? "set" : "unset"}, password: #{@client.proxy_uri.password ? "set" : "unset"}")
|
337
|
+
end
|
311
338
|
end
|
312
339
|
|
313
340
|
def send(payload)
|
@@ -315,7 +342,8 @@ class Fluent::DatadogOutput < Fluent::Plugin::Output
|
|
315
342
|
request.body = payload
|
316
343
|
response = @client.request @uri, request
|
317
344
|
res_code = response.code.to_i
|
318
|
-
|
345
|
+
# on a backend error or on an http 429, retry with backoff
|
346
|
+
if res_code >= 500 || res_code == 429
|
319
347
|
raise RetryableError.new "Unable to send payload: #{res_code} #{response.message}"
|
320
348
|
end
|
321
349
|
if res_code >= 400
|
metadata
CHANGED
@@ -1,29 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-datadog
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.14.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Datadog Solutions Team
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-01-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
-
- !ruby/object:Gem::Dependency
|
14
|
-
name: bundler
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - "~>"
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '1.5'
|
20
|
-
type: :development
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
-
requirements:
|
24
|
-
- - "~>"
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: '1.5'
|
27
13
|
- !ruby/object:Gem::Dependency
|
28
14
|
name: fluentd
|
29
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -50,14 +36,28 @@ dependencies:
|
|
50
36
|
requirements:
|
51
37
|
- - "~>"
|
52
38
|
- !ruby/object:Gem::Version
|
53
|
-
version: '
|
39
|
+
version: '3.1'
|
54
40
|
type: :runtime
|
55
41
|
prerelease: false
|
56
42
|
version_requirements: !ruby/object:Gem::Requirement
|
57
43
|
requirements:
|
58
44
|
- - "~>"
|
59
45
|
- !ruby/object:Gem::Version
|
60
|
-
version: '
|
46
|
+
version: '3.1'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: bundler
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - "~>"
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '2.1'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - "~>"
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '2.1'
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
62
|
name: test-unit
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
@@ -76,16 +76,16 @@ dependencies:
|
|
76
76
|
name: rake
|
77
77
|
requirement: !ruby/object:Gem::Requirement
|
78
78
|
requirements:
|
79
|
-
- - "
|
79
|
+
- - "~>"
|
80
80
|
- !ruby/object:Gem::Version
|
81
|
-
version: '0'
|
81
|
+
version: '12.0'
|
82
82
|
type: :development
|
83
83
|
prerelease: false
|
84
84
|
version_requirements: !ruby/object:Gem::Requirement
|
85
85
|
requirements:
|
86
|
-
- - "
|
86
|
+
- - "~>"
|
87
87
|
- !ruby/object:Gem::Version
|
88
|
-
version: '0'
|
88
|
+
version: '12.0'
|
89
89
|
- !ruby/object:Gem::Dependency
|
90
90
|
name: yajl-ruby
|
91
91
|
requirement: !ruby/object:Gem::Requirement
|
@@ -100,7 +100,21 @@ dependencies:
|
|
100
100
|
- - "~>"
|
101
101
|
- !ruby/object:Gem::Version
|
102
102
|
version: '1.2'
|
103
|
-
|
103
|
+
- !ruby/object:Gem::Dependency
|
104
|
+
name: webmock
|
105
|
+
requirement: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - "~>"
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: 3.6.0
|
110
|
+
type: :development
|
111
|
+
prerelease: false
|
112
|
+
version_requirements: !ruby/object:Gem::Requirement
|
113
|
+
requirements:
|
114
|
+
- - "~>"
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: 3.6.0
|
117
|
+
description:
|
104
118
|
email:
|
105
119
|
- support@datadoghq.com
|
106
120
|
executables: []
|
@@ -114,11 +128,12 @@ files:
|
|
114
128
|
- Rakefile
|
115
129
|
- fluent-plugin-datadog.gemspec
|
116
130
|
- lib/fluent/plugin/out_datadog.rb
|
131
|
+
- lib/fluent/plugin/version.rb
|
117
132
|
homepage: http://datadoghq.com
|
118
133
|
licenses:
|
119
|
-
- Apache
|
134
|
+
- Apache-2.0
|
120
135
|
metadata: {}
|
121
|
-
post_install_message:
|
136
|
+
post_install_message:
|
122
137
|
rdoc_options: []
|
123
138
|
require_paths:
|
124
139
|
- lib
|
@@ -133,9 +148,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
133
148
|
- !ruby/object:Gem::Version
|
134
149
|
version: '0'
|
135
150
|
requirements: []
|
136
|
-
rubyforge_project:
|
151
|
+
rubyforge_project:
|
137
152
|
rubygems_version: 2.7.10
|
138
|
-
signing_key:
|
153
|
+
signing_key:
|
139
154
|
specification_version: 4
|
140
155
|
summary: Datadog output plugin for Fluent event collector
|
141
156
|
test_files: []
|