logstash-output-timber 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/README.md +1 -1
- data/docs/index.asciidoc +20 -4
- data/lib/logstash/outputs/timber.rb +62 -5
- data/logstash-output-timber.gemspec +1 -1
- data/spec/outputs/timber_spec.rb +52 -13
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e6c3387a121c8546ddba33fc547511708b71c28082d251fa7d5f7d7521e67790
|
4
|
+
data.tar.gz: 054f3d3296cd151bfba7c4e978a4bd213ed28c82e5813f111380c0452c28c3eb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 61531e2d2edf5258e9de03ccfc28f8ccb2e57158b68dd14285510c67eb87b146455733b8b9691e3ac64737b69f496b16fee1a35654c3b18b52d393477748d9b0
|
7
|
+
data.tar.gz: 7199379917b57e6a10935e8f24bf66643f472af01995b23bff74d09c362537c2bf928421e93740113fa31ce8ed09e765dd9fc401e399a40478a2244b65ef14fd
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
## 1.0.3
|
2
|
+
- Intelligently map logstash events to the Timber.io log event JSON schema
|
3
|
+
(https://github.com/timberio/log-event-json-schema).
|
4
|
+
|
5
|
+
## 1.0.2
|
6
|
+
- Add hostname to the payload being sent to Timber.
|
7
|
+
|
1
8
|
## 1.0.1
|
2
9
|
- Drop the @version field since it is not useful to external systems
|
3
10
|
|
data/README.md
CHANGED
data/docs/index.asciidoc
CHANGED
@@ -20,9 +20,16 @@ include::{include_path}/plugin_header.asciidoc[]
|
|
20
20
|
|
21
21
|
==== Description
|
22
22
|
|
23
|
-
This output sends structured events to the Timber.io logging service.
|
24
|
-
|
25
|
-
|
23
|
+
This output sends structured events to the https://timber.io[Timber.io logging service].
|
24
|
+
Timber is a cloud-based logging service designed for developers, providing easy features
|
25
|
+
out of the box that make you more productive.
|
26
|
+
https://timber.io/docs/app/console/tail-a-user[Tail users],
|
27
|
+
https://timber.io/docs/app/console/trace-http-requests[trace requests],
|
28
|
+
https://timber.io/docs/app/console/inspect-http-requests[inspect HTTP parameters],
|
29
|
+
and https://timber.io/docs/app/console/searching[search] on rich structured data without
|
30
|
+
sacrificing readability.
|
31
|
+
|
32
|
+
Internally, it's a highly efficient HTTP transport that uses batching and retries for
|
26
33
|
fast and reliable delivery.
|
27
34
|
|
28
35
|
This output will execute up to 'pool_max' requests in parallel for performance.
|
@@ -63,7 +70,7 @@ output plugins.
|
|
63
70
|
|
64
71
|
|
65
72
|
[id="plugins-{type}s-{plugin}-api_key"]
|
66
|
-
===== `
|
73
|
+
===== `api_key`
|
67
74
|
|
68
75
|
* Value type is <<string,string>>
|
69
76
|
* There is no default value for this setting.
|
@@ -72,6 +79,15 @@ Your Timber.io API key. You can obtain your API by creating an app in the
|
|
72
79
|
[Timber console](https://app.timber.io).
|
73
80
|
|
74
81
|
|
82
|
+
[id="plugins-{type}s-{plugin}-cacert"]
|
83
|
+
===== `cacert`
|
84
|
+
|
85
|
+
* Value type is <<path,path>>
|
86
|
+
* There is no default value for this setting.
|
87
|
+
|
88
|
+
If you need to use a custom X.509 CA (.pem certs) specify the path to that here.
|
89
|
+
|
90
|
+
|
75
91
|
[id="plugins-{type}s-{plugin}-client_cert"]
|
76
92
|
===== `client_cert`
|
77
93
|
|
@@ -10,11 +10,16 @@ require "logstash/json"
|
|
10
10
|
require "logstash/outputs/base"
|
11
11
|
require "logstash/outputs/timber/http_client"
|
12
12
|
|
13
|
+
# This is an output class that intelligently forwards logstash events to the Timber.io service.
|
14
|
+
#
|
15
|
+
# For a comprehensive overview around how this works and the various configuration options,
|
16
|
+
# please see: https://timber.io/docs/platforms/logstash
|
13
17
|
class LogStash::Outputs::Timber < LogStash::Outputs::Base
|
14
18
|
include HttpClient
|
15
19
|
|
16
20
|
VERSION = "1.0.2".freeze
|
17
21
|
CONTENT_TYPE = "application/json".freeze
|
22
|
+
JSON_SCHEMA = "https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json".freeze
|
18
23
|
MAX_ATTEMPTS = 3
|
19
24
|
METHOD = :post.freeze
|
20
25
|
RETRYABLE_MANTICORE_EXCEPTIONS = [
|
@@ -127,12 +132,64 @@ class LogStash::Outputs::Timber < LogStash::Outputs::Base
|
|
127
132
|
end
|
128
133
|
end
|
129
134
|
|
135
|
+
# This method takes a `Logstash::Event` object and converts it into a hash
|
136
|
+
# that is acceptable by the Timber API. Each event is converted into a JSON
|
137
|
+
# document that conforms to the Timber log event JSON schema:
|
138
|
+
#
|
139
|
+
# https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json
|
140
|
+
#
|
141
|
+
# This realized by the following steps:
|
142
|
+
#
|
143
|
+
# 1. Timber will look for specific keys and map them to the appropriate keys as defined
|
144
|
+
# in our log event JSON schema. Specifically `@timestamp`, `host`, and `message`.
|
145
|
+
# 2. If a `timber` key is present it _must_ be a hash that conforms to the Timber log event
|
146
|
+
# JSON schema. This hash will be merged in before being sent to Timber.
|
147
|
+
# 3. All other root level keys will be treated as generic JSON and will be made available
|
148
|
+
# in Timber as they would kibana, etc.
|
130
149
|
def event_hash(e)
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
150
|
+
timber_hash = {"$schema" => JSON_SCHEMA}
|
151
|
+
event_hash = e.to_hash
|
152
|
+
|
153
|
+
# Delete unused logstash specific attributes
|
154
|
+
event_hash.delete("@version")
|
155
|
+
|
156
|
+
# Map the timber key first since we merge in values
|
157
|
+
# later.
|
158
|
+
timber = event_hash.delete("timber")
|
159
|
+
if timber.is_a?(Hash)
|
160
|
+
timber_hash.merge!(timber)
|
161
|
+
end
|
162
|
+
|
163
|
+
# Map the timestamp
|
164
|
+
timestamp = event_hash.delete("@timestamp")
|
165
|
+
|
166
|
+
if timestamp
|
167
|
+
timber_hash["dt"] ||= timestamp.utc.to_iso8601
|
168
|
+
end
|
169
|
+
|
170
|
+
# Map the host
|
171
|
+
host = event_hash.delete("host")
|
172
|
+
|
173
|
+
if host
|
174
|
+
timber_hash["context"] ||= {}
|
175
|
+
timber_hash["context"]["system"] ||= {}
|
176
|
+
timber_hash["context"]["system"]["hostname"] ||= host
|
177
|
+
end
|
178
|
+
|
179
|
+
# Map the message
|
180
|
+
message = event_hash.delete("message")
|
181
|
+
|
182
|
+
if message
|
183
|
+
timber_hash["message"] ||= message
|
184
|
+
end
|
185
|
+
|
186
|
+
# Move everything else to meta, merging to preseve previous meta values.
|
187
|
+
if event_hash != {}
|
188
|
+
timber_hash["meta"] ||= {}
|
189
|
+
timber_hash["meta"].merge!(event_hash)
|
190
|
+
end
|
191
|
+
|
192
|
+
timber_hash
|
136
193
|
end
|
137
194
|
|
138
195
|
def retryable_exception?(e)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-timber'
|
3
|
-
s.version = "1.0.
|
3
|
+
s.version = "1.0.3"
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "This output send events to the Timber.io logging service"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/outputs/timber_spec.rb
CHANGED
@@ -145,23 +145,62 @@ describe LogStash::Outputs::Timber do
|
|
145
145
|
|
146
146
|
body_event = parsed_body.first
|
147
147
|
timestamp_iso8601 = event.get("@timestamp").to_iso8601
|
148
|
-
|
148
|
+
expected_payload = {"$schema"=>"https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json", "dt"=>timestamp_iso8601, "message"=>"hi"}
|
149
|
+
expect(body_event).to eq(expected_payload)
|
149
150
|
end
|
151
|
+
end
|
150
152
|
|
151
|
-
|
152
|
-
|
153
|
-
event = LogStash::Event.new({"
|
154
|
-
|
155
|
-
|
156
|
-
|
153
|
+
describe "#event_hash" do
|
154
|
+
it "merges the timber key" do
|
155
|
+
event = LogStash::Event.new({"message" => "my message", "timber" => {"context" => {"system" => {"pid" => 123}}}})
|
156
|
+
hash = output.send(:event_hash, event)
|
157
|
+
|
158
|
+
dt = event.get("@timestamp").utc.to_iso8601
|
159
|
+
expect(hash).to eq({
|
160
|
+
"$schema" => "https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json",
|
161
|
+
"context" => {"system" => {"pid" => 123}},
|
162
|
+
"message" => "my message",
|
163
|
+
"dt" => dt
|
164
|
+
})
|
165
|
+
end
|
157
166
|
|
158
|
-
|
159
|
-
|
160
|
-
|
167
|
+
it "moves host" do
|
168
|
+
event = LogStash::Event.new({"message" => "my message", "host" => "local.myhost.com"})
|
169
|
+
hash = output.send(:event_hash, event)
|
170
|
+
|
171
|
+
dt = event.get("@timestamp").utc.to_iso8601
|
172
|
+
expect(hash).to eq({
|
173
|
+
"$schema" => "https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json",
|
174
|
+
"message" => "my message",
|
175
|
+
"dt" => dt,
|
176
|
+
"context" => {"system" => {"hostname" => "local.myhost.com"}}
|
177
|
+
})
|
178
|
+
end
|
161
179
|
|
162
|
-
|
163
|
-
|
164
|
-
|
180
|
+
it "moves everything else to meta" do
|
181
|
+
event = LogStash::Event.new({"message" => "my message", "key" => "val"})
|
182
|
+
hash = output.send(:event_hash, event)
|
183
|
+
|
184
|
+
dt = event.get("@timestamp").utc.to_iso8601
|
185
|
+
expect(hash).to eq({
|
186
|
+
"$schema" => "https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json",
|
187
|
+
"message" => "my message",
|
188
|
+
"dt" => dt,
|
189
|
+
"meta" => {"key" => "val"}
|
190
|
+
})
|
191
|
+
end
|
192
|
+
|
193
|
+
it "handles bother timber and host" do
|
194
|
+
event = LogStash::Event.new({"message" => "my message", "host" => "local.myhost.com", "timber" => {"context" => {"system" => {"pid" => 123}}}})
|
195
|
+
hash = output.send(:event_hash, event)
|
196
|
+
|
197
|
+
dt = event.get("@timestamp").utc.to_iso8601
|
198
|
+
expect(hash).to eq({
|
199
|
+
"$schema" => "https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json",
|
200
|
+
"message" => "my message",
|
201
|
+
"dt" => dt,
|
202
|
+
"context" => {"system" => {"hostname" => "local.myhost.com", "pid" => 123}}
|
203
|
+
})
|
165
204
|
end
|
166
205
|
end
|
167
206
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-timber
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-09-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|