logstash-output-timber 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 41db80ea8dc6e0c7655f0fcc3c84101be85c8241de366ef07493dedea222b67d
-  data.tar.gz: 1c0c6e41505a4a1b4b090663d29480aee491eceb58dce58c9af5e0c47418817d
+  metadata.gz: e6c3387a121c8546ddba33fc547511708b71c28082d251fa7d5f7d7521e67790
+  data.tar.gz: 054f3d3296cd151bfba7c4e978a4bd213ed28c82e5813f111380c0452c28c3eb
 SHA512:
-  metadata.gz: 49fb12cb9a25669e2d270ce7b315222f9a5a5988aa51c826dfc85821ed6b116bff18dcf1740e49c3acf03e910b897bd31cd6b2d62f645d5a1be65f7006d2b3a5
-  data.tar.gz: 58c94bb43555676e76e7d11bedf0df71277c2f47f6832cfa6ec5b2e7610a15270750d7fec0672a6a661bd5cd0f87fb2a6bf33142d626e504ad66cdf430878b5b
+  metadata.gz: 61531e2d2edf5258e9de03ccfc28f8ccb2e57158b68dd14285510c67eb87b146455733b8b9691e3ac64737b69f496b16fee1a35654c3b18b52d393477748d9b0
+  data.tar.gz: 7199379917b57e6a10935e8f24bf66643f472af01995b23bff74d09c362537c2bf928421e93740113fa31ce8ed09e765dd9fc401e399a40478a2244b65ef14fd

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 1.0.3
+ - Intelligently map logstash events to the Timber.io log event JSON schema
+   (https://github.com/timberio/log-event-json-schema).
+
+## 1.0.2
+ - Add hostname to the payload being sent to Timber.
+
 ## 1.0.1
  - Drop the @version field since it is not useful to external systems
 

data/README.md

@@ -1,4 +1,4 @@
-# Logstash Plugin
+# 🌲 Timber.io Logstash Output Plugin
 
 [![Travis Build Status](https://travis-ci.org/timberio/logstash-output-timber.svg)](https://travis-ci.org/timberio/logstash-output-timber)
 

data/docs/index.asciidoc

@@ -20,9 +20,16 @@ include::{include_path}/plugin_header.asciidoc[]
 
 ==== Description
 
-This output sends structured events to the Timber.io logging service.
-
-It's a highly efficient HTTP transport that uses batching and retries for
+This output sends structured events to the https://timber.io[Timber.io logging service].
+Timber is a cloud-based logging service designed for developers, providing easy features
+out of the box that make you more productive.
+https://timber.io/docs/app/console/tail-a-user[Tail users],
+https://timber.io/docs/app/console/trace-http-requests[trace requests],
+https://timber.io/docs/app/console/inspect-http-requests[inspect HTTP parameters],
+and https://timber.io/docs/app/console/searching[search] on rich structured data without
+sacrificing readability.
+
+Internally, it's a highly efficient HTTP transport that uses batching and retries for
 fast and reliable delivery.
 
 This output will execute up to 'pool_max' requests in parallel for performance.
@@ -63,7 +70,7 @@ output plugins.
  
 
 [id="plugins-{type}s-{plugin}-api_key"]
-===== `cacert`
+===== `api_key`
 
   * Value type is <<string,string>>
   * There is no default value for this setting.
@@ -72,6 +79,15 @@ Your Timber.io API key. You can obtain your API by creating an app in the
 [Timber console](https://app.timber.io).
 
 
+[id="plugins-{type}s-{plugin}-cacert"]
+===== `cacert`
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+If you need to use a custom X.509 CA (.pem certs) specify the path to that here.
+
+
 [id="plugins-{type}s-{plugin}-client_cert"]
 ===== `client_cert`
 

data/lib/logstash/outputs/timber.rb

@@ -10,11 +10,16 @@ require "logstash/json"
 require "logstash/outputs/base"
 require "logstash/outputs/timber/http_client"
 
+# This is an output class that intelligently forwards logstash events to the Timber.io service.
+#
+# For a comprehensive overview around how this works and the various configuration options,
+# please see: https://timber.io/docs/platforms/logstash
 class LogStash::Outputs::Timber < LogStash::Outputs::Base
   include HttpClient
 
   VERSION = "1.0.2".freeze
   CONTENT_TYPE = "application/json".freeze
+  JSON_SCHEMA = "https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json".freeze
   MAX_ATTEMPTS = 3
   METHOD = :post.freeze
   RETRYABLE_MANTICORE_EXCEPTIONS = [
@@ -127,12 +132,64 @@ class LogStash::Outputs::Timber < LogStash::Outputs::Base
       end
     end
 
+    # This method takes a `Logstash::Event` object and converts it into a hash
+    # that is acceptable by the Timber API. Each event is converted into a JSON
+    # document that conforms to the Timber log event JSON schema:
+    #
+    # https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json
+    #
+    # This realized by the following steps:
+    #
+    # 1. Timber will look for specific keys and map them to the appropriate keys as defined
+    #    in our log event JSON schema. Specifically `@timestamp`, `host`, and `message`.
+    # 2. If a `timber` key is present it _must_ be a hash that conforms to the Timber log event
+    #    JSON schema. This hash will be merged in before being sent to Timber.
+    # 3. All other root level keys will be treated as generic JSON and will be made available
+    #    in Timber as they would kibana, etc.
     def event_hash(e)
-      hash = e.to_hash
-      hash.delete("@version")
-      dt = hash.delete("@timestamp")
-      hash["dt"] = dt if !hash.has_key?("dt")
-      hash
+      timber_hash = {"$schema" => JSON_SCHEMA}
+      event_hash = e.to_hash
+
+      # Delete unused logstash specific attributes
+      event_hash.delete("@version")
+
+      # Map the timber key first since we merge in values
+      # later.
+      timber = event_hash.delete("timber")
+      if timber.is_a?(Hash)
+        timber_hash.merge!(timber)
+      end
+
+      # Map the timestamp
+      timestamp = event_hash.delete("@timestamp")
+
+      if timestamp
+        timber_hash["dt"] ||= timestamp.utc.to_iso8601
+      end
+
+      # Map the host
+      host = event_hash.delete("host")
+
+      if host
+        timber_hash["context"] ||= {}
+        timber_hash["context"]["system"] ||= {}
+        timber_hash["context"]["system"]["hostname"] ||= host
+      end
+
+      # Map the message
+      message = event_hash.delete("message")
+
+      if message
+        timber_hash["message"] ||= message
+      end
+
+      # Move everything else to meta, merging to preseve previous meta values.
+      if event_hash != {}
+        timber_hash["meta"] ||= {}
+        timber_hash["meta"].merge!(event_hash)
+      end
+
+      timber_hash
     end
 
     def retryable_exception?(e)

data/logstash-output-timber.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-timber'
-  s.version         = "1.0.2"
+  s.version         = "1.0.3"
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This output send events to the Timber.io logging service"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/timber_spec.rb

@@ -145,23 +145,62 @@ describe LogStash::Outputs::Timber do
 
       body_event = parsed_body.first
       timestamp_iso8601 = event.get("@timestamp").to_iso8601
-      expect(body_event).to eq({"dt"=>timestamp_iso8601, "message"=>"hi"})
+      expected_payload = {"$schema"=>"https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json", "dt"=>timestamp_iso8601, "message"=>"hi"}
+      expect(body_event).to eq(expected_payload)
     end
+  end
 
-    it "allows payloads conformed to the timber JSON schema" do
-      output.url = "http://localhost:#{port}/good"
-      event = LogStash::Event.new({"$schema" => "https://raw.githubusercontent.com/timberio/log-event-json-schema/v2.4.2/schema.json", "message" => "my message", "context" => {"http" => {"path" => "/path"}}})
-      result = output.send(:send_events, [event], 1)
-      expect(result).to eq(true)
-      expect(requests.length).to eq(1)
+  describe "#event_hash" do
+    it "merges the timber key" do
+      event = LogStash::Event.new({"message" => "my message", "timber" => {"context" => {"system" => {"pid" => 123}}}})
+      hash = output.send(:event_hash, event)
+
+      dt = event.get("@timestamp").utc.to_iso8601
+      expect(hash).to eq({
+        "$schema" => "https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json",
+        "context" => {"system" => {"pid" => 123}},
+        "message" => "my message",
+        "dt" => dt
+      })
+    end
 
-      request = requests.first
-      parsed_body = JSON.parse!(request.body.read)
-      expect(parsed_body.length).to eq(1)
+    it "moves host" do
+      event = LogStash::Event.new({"message" => "my message", "host" => "local.myhost.com"})
+      hash = output.send(:event_hash, event)
+
+      dt = event.get("@timestamp").utc.to_iso8601
+      expect(hash).to eq({
+        "$schema" => "https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json",
+        "message" => "my message",
+        "dt" => dt,
+        "context" => {"system" => {"hostname" => "local.myhost.com"}}
+      })
+    end
 
-      body_event = parsed_body.first
-      timestamp_iso8601 = event.get("@timestamp").to_iso8601
-      expect(body_event).to eq({"$schema"=>"https://raw.githubusercontent.com/timberio/log-event-json-schema/v2.4.2/schema.json", "context"=>{"http"=>{"path"=>"/path"}}, "message"=>"my message", "dt"=>timestamp_iso8601})
+    it "moves everything else to meta" do
+      event = LogStash::Event.new({"message" => "my message", "key" => "val"})
+      hash = output.send(:event_hash, event)
+
+      dt = event.get("@timestamp").utc.to_iso8601
+      expect(hash).to eq({
+        "$schema" => "https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json",
+        "message" => "my message",
+        "dt" => dt,
+        "meta" => {"key" => "val"}
+      })
+    end
+
+    it "handles bother timber and host" do
+      event = LogStash::Event.new({"message" => "my message", "host" => "local.myhost.com", "timber" => {"context" => {"system" => {"pid" => 123}}}})
+      hash = output.send(:event_hash, event)
+
+      dt = event.get("@timestamp").utc.to_iso8601
+      expect(hash).to eq({
+        "$schema" => "https://raw.githubusercontent.com/timberio/log-event-json-schema/v3.1.1/schema.json",
+        "message" => "my message",
+        "dt" => dt,
+        "context" => {"system" => {"hostname" => "local.myhost.com", "pid" => 123}}
+      })
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-timber
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-07-16 00:00:00.000000000 Z
+date: 2017-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement