logstash-output-timber 1.0.2 → 1.0.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 41db80ea8dc6e0c7655f0fcc3c84101be85c8241de366ef07493dedea222b67d
4
- data.tar.gz: 1c0c6e41505a4a1b4b090663d29480aee491eceb58dce58c9af5e0c47418817d
3
+ metadata.gz: e6c3387a121c8546ddba33fc547511708b71c28082d251fa7d5f7d7521e67790
4
+ data.tar.gz: 054f3d3296cd151bfba7c4e978a4bd213ed28c82e5813f111380c0452c28c3eb
5
5
  SHA512:
6
- metadata.gz: 49fb12cb9a25669e2d270ce7b315222f9a5a5988aa51c826dfc85821ed6b116bff18dcf1740e49c3acf03e910b897bd31cd6b2d62f645d5a1be65f7006d2b3a5
7
- data.tar.gz: 58c94bb43555676e76e7d11bedf0df71277c2f47f6832cfa6ec5b2e7610a15270750d7fec0672a6a661bd5cd0f87fb2a6bf33142d626e504ad66cdf430878b5b
6
+ metadata.gz: 61531e2d2edf5258e9de03ccfc28f8ccb2e57158b68dd14285510c67eb87b146455733b8b9691e3ac64737b69f496b16fee1a35654c3b18b52d393477748d9b0
7
+ data.tar.gz: 7199379917b57e6a10935e8f24bf66643f472af01995b23bff74d09c362537c2bf928421e93740113fa31ce8ed09e765dd9fc401e399a40478a2244b65ef14fd
@@ -1,3 +1,10 @@
1
+ ## 1.0.3
2
+ - Intelligently map logstash events to the Timber.io log event JSON schema
3
+ (https://github.com/timberio/log-event-json-schema).
4
+
5
+ ## 1.0.2
6
+ - Add hostname to the payload being sent to Timber.
7
+
1
8
  ## 1.0.1
2
9
  - Drop the @version field since it is not useful to external systems
3
10
 
data/README.md CHANGED
@@ -1,4 +1,4 @@
1
- # Logstash Plugin
1
+ # 🌲 Timber.io Logstash Output Plugin
2
2
 
3
3
  [![Travis Build Status](https://travis-ci.org/timberio/logstash-output-timber.svg)](https://travis-ci.org/timberio/logstash-output-timber)
4
4
 
@@ -20,9 +20,16 @@ include::{include_path}/plugin_header.asciidoc[]
20
20
 
21
21
  ==== Description
22
22
 
23
- This output sends structured events to the Timber.io logging service.
24
-
25
- It's a highly efficient HTTP transport that uses batching and retries for
23
+ This output sends structured events to the https://timber.io[Timber.io logging service].
24
+ Timber is a cloud-based logging service designed for developers, providing easy features
25
+ out of the box that make you more productive.
26
+ https://timber.io/docs/app/console/tail-a-user[Tail users],
27
+ https://timber.io/docs/app/console/trace-http-requests[trace requests],
28
+ https://timber.io/docs/app/console/inspect-http-requests[inspect HTTP parameters],
29
+ and https://timber.io/docs/app/console/searching[search] on rich structured data without
30
+ sacrificing readability.
31
+
32
+ Internally, it's a highly efficient HTTP transport that uses batching and retries for
26
33
  fast and reliable delivery.
27
34
 
28
35
  This output will execute up to 'pool_max' requests in parallel for performance.
@@ -63,7 +70,7 @@ output plugins.
63
70
   
64
71
 
65
72
  [id="plugins-{type}s-{plugin}-api_key"]
66
- ===== `cacert`
73
+ ===== `api_key`
67
74
 
68
75
  * Value type is <<string,string>>
69
76
  * There is no default value for this setting.
@@ -72,6 +79,15 @@ Your Timber.io API key. You can obtain your API by creating an app in the
72
79
  [Timber console](https://app.timber.io).
73
80
 
74
81
 
82
+ [id="plugins-{type}s-{plugin}-cacert"]
83
+ ===== `cacert`
84
+
85
+ * Value type is <<path,path>>
86
+ * There is no default value for this setting.
87
+
88
+ If you need to use a custom X.509 CA (.pem certs) specify the path to that here.
89
+
90
+
75
91
  [id="plugins-{type}s-{plugin}-client_cert"]
76
92
  ===== `client_cert`
77
93
 
@@ -10,11 +10,16 @@ require "logstash/json"
10
10
  require "logstash/outputs/base"
11
11
  require "logstash/outputs/timber/http_client"
12
12
 
13
+ # This is an output class that intelligently forwards logstash events to the Timber.io service.
14
+ #
15
+ # For a comprehensive overview around how this works and the various configuration options,
16
+ # please see: https://timber.io/docs/platforms/logstash
13
17
  class LogStash::Outputs::Timber < LogStash::Outputs::Base
14
18
  include HttpClient
15
19
 
16
20
  VERSION = "1.0.2".freeze
17
21
  CONTENT_TYPE = "application/json".freeze
22
+ JSON_SCHEMA = "https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json".freeze
18
23
  MAX_ATTEMPTS = 3
19
24
  METHOD = :post.freeze
20
25
  RETRYABLE_MANTICORE_EXCEPTIONS = [
@@ -127,12 +132,64 @@ class LogStash::Outputs::Timber < LogStash::Outputs::Base
127
132
  end
128
133
  end
129
134
 
135
+ # This method takes a `Logstash::Event` object and converts it into a hash
136
+ # that is acceptable by the Timber API. Each event is converted into a JSON
137
+ # document that conforms to the Timber log event JSON schema:
138
+ #
139
+ # https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json
140
+ #
141
+ # This realized by the following steps:
142
+ #
143
+ # 1. Timber will look for specific keys and map them to the appropriate keys as defined
144
+ # in our log event JSON schema. Specifically `@timestamp`, `host`, and `message`.
145
+ # 2. If a `timber` key is present it _must_ be a hash that conforms to the Timber log event
146
+ # JSON schema. This hash will be merged in before being sent to Timber.
147
+ # 3. All other root level keys will be treated as generic JSON and will be made available
148
+ # in Timber as they would kibana, etc.
130
149
  def event_hash(e)
131
- hash = e.to_hash
132
- hash.delete("@version")
133
- dt = hash.delete("@timestamp")
134
- hash["dt"] = dt if !hash.has_key?("dt")
135
- hash
150
+ timber_hash = {"$schema" => JSON_SCHEMA}
151
+ event_hash = e.to_hash
152
+
153
+ # Delete unused logstash specific attributes
154
+ event_hash.delete("@version")
155
+
156
+ # Map the timber key first since we merge in values
157
+ # later.
158
+ timber = event_hash.delete("timber")
159
+ if timber.is_a?(Hash)
160
+ timber_hash.merge!(timber)
161
+ end
162
+
163
+ # Map the timestamp
164
+ timestamp = event_hash.delete("@timestamp")
165
+
166
+ if timestamp
167
+ timber_hash["dt"] ||= timestamp.utc.to_iso8601
168
+ end
169
+
170
+ # Map the host
171
+ host = event_hash.delete("host")
172
+
173
+ if host
174
+ timber_hash["context"] ||= {}
175
+ timber_hash["context"]["system"] ||= {}
176
+ timber_hash["context"]["system"]["hostname"] ||= host
177
+ end
178
+
179
+ # Map the message
180
+ message = event_hash.delete("message")
181
+
182
+ if message
183
+ timber_hash["message"] ||= message
184
+ end
185
+
186
+ # Move everything else to meta, merging to preseve previous meta values.
187
+ if event_hash != {}
188
+ timber_hash["meta"] ||= {}
189
+ timber_hash["meta"].merge!(event_hash)
190
+ end
191
+
192
+ timber_hash
136
193
  end
137
194
 
138
195
  def retryable_exception?(e)
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-timber'
3
- s.version = "1.0.2"
3
+ s.version = "1.0.3"
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "This output send events to the Timber.io logging service"
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -145,23 +145,62 @@ describe LogStash::Outputs::Timber do
145
145
 
146
146
  body_event = parsed_body.first
147
147
  timestamp_iso8601 = event.get("@timestamp").to_iso8601
148
- expect(body_event).to eq({"dt"=>timestamp_iso8601, "message"=>"hi"})
148
+ expected_payload = {"$schema"=>"https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json", "dt"=>timestamp_iso8601, "message"=>"hi"}
149
+ expect(body_event).to eq(expected_payload)
149
150
  end
151
+ end
150
152
 
151
- it "allows payloads conformed to the timber JSON schema" do
152
- output.url = "http://localhost:#{port}/good"
153
- event = LogStash::Event.new({"$schema" => "https://raw.githubusercontent.com/timberio/log-event-json-schema/v2.4.2/schema.json", "message" => "my message", "context" => {"http" => {"path" => "/path"}}})
154
- result = output.send(:send_events, [event], 1)
155
- expect(result).to eq(true)
156
- expect(requests.length).to eq(1)
153
+ describe "#event_hash" do
154
+ it "merges the timber key" do
155
+ event = LogStash::Event.new({"message" => "my message", "timber" => {"context" => {"system" => {"pid" => 123}}}})
156
+ hash = output.send(:event_hash, event)
157
+
158
+ dt = event.get("@timestamp").utc.to_iso8601
159
+ expect(hash).to eq({
160
+ "$schema" => "https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json",
161
+ "context" => {"system" => {"pid" => 123}},
162
+ "message" => "my message",
163
+ "dt" => dt
164
+ })
165
+ end
157
166
 
158
- request = requests.first
159
- parsed_body = JSON.parse!(request.body.read)
160
- expect(parsed_body.length).to eq(1)
167
+ it "moves host" do
168
+ event = LogStash::Event.new({"message" => "my message", "host" => "local.myhost.com"})
169
+ hash = output.send(:event_hash, event)
170
+
171
+ dt = event.get("@timestamp").utc.to_iso8601
172
+ expect(hash).to eq({
173
+ "$schema" => "https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json",
174
+ "message" => "my message",
175
+ "dt" => dt,
176
+ "context" => {"system" => {"hostname" => "local.myhost.com"}}
177
+ })
178
+ end
161
179
 
162
- body_event = parsed_body.first
163
- timestamp_iso8601 = event.get("@timestamp").to_iso8601
164
- expect(body_event).to eq({"$schema"=>"https://raw.githubusercontent.com/timberio/log-event-json-schema/v2.4.2/schema.json", "context"=>{"http"=>{"path"=>"/path"}}, "message"=>"my message", "dt"=>timestamp_iso8601})
180
+ it "moves everything else to meta" do
181
+ event = LogStash::Event.new({"message" => "my message", "key" => "val"})
182
+ hash = output.send(:event_hash, event)
183
+
184
+ dt = event.get("@timestamp").utc.to_iso8601
185
+ expect(hash).to eq({
186
+ "$schema" => "https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json",
187
+ "message" => "my message",
188
+ "dt" => dt,
189
+ "meta" => {"key" => "val"}
190
+ })
191
+ end
192
+
193
+ it "handles bother timber and host" do
194
+ event = LogStash::Event.new({"message" => "my message", "host" => "local.myhost.com", "timber" => {"context" => {"system" => {"pid" => 123}}}})
195
+ hash = output.send(:event_hash, event)
196
+
197
+ dt = event.get("@timestamp").utc.to_iso8601
198
+ expect(hash).to eq({
199
+ "$schema" => "https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json",
200
+ "message" => "my message",
201
+ "dt" => dt,
202
+ "context" => {"system" => {"hostname" => "local.myhost.com", "pid" => 123}}
203
+ })
165
204
  end
166
205
  end
167
206
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-timber
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.2
4
+ version: 1.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-07-16 00:00:00.000000000 Z
11
+ date: 2017-09-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement