logstash-codec-cloudwatch_logs 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c93c61a0f6290abeaa6435db7b5abd7eaea0e431
-  data.tar.gz: 2f24162de1251bad7eeeb8dfcaf0bc40de815dab
+  metadata.gz: 1fe34dd5f76336b9bcd3bfefb5d741c4a0cf60e3
+  data.tar.gz: c47f73f3e7332ccf6956ae23d0b4daa58a710929
 SHA512:
-  metadata.gz: a942cf6850c35d2675bbad4304168157134a5c0883cc2b1350f4e2516ee199575a8f3be1da913f17804a45c1242f2510d27a0a7f30b6e9120d69af739e1dc8fc
-  data.tar.gz: 75b69c66dde11669ee7b97d05e1473f3e2873a5fcd17708392b76575e18f74f636bac0741726fe310132444d9f1b632e26cd1f2626616937733c15a8dd8c43b6
+  metadata.gz: 6da618f7c472d645e72a1a3796a63fc40e334d12e41b4aefb756c27a8d3f59309255f6016a2f343c706feb355caae6899ec7c4797fc449d411dedc1f965f0cf3
+  data.tar.gz: 02e9f55c7eb968e1d99b6142d78509e124030481c86166b9026b945d94ede0749bcb09061655a87498e08a4b671f0f6a24abf4c3a16f2183b3b918b95ee005b8

data/README.md

@@ -2,10 +2,20 @@
 
 [![Travis Build Status](https://travis-ci.org/threadwaste/logstash-codec-cloudwatch_logs.svg)](https://travis-ci.org/threadwaste/logstash-codec-cloudwatch_logs)
 
-Parse [CloudWatch Logs subscriptions](http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/Subscriptions.html#DestinationKinesisExample) sent to Kinesis.
+Parse [CloudWatch Logs subscriptions](http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/Subscriptions.html#DestinationKinesisExample) into individual events.
+
+## Installation
+
+This plugin can be installed by Logstash's plugin tool.
+
+```
+bin/logstash-plugin install logstash-codec-cloudwatch_logs
+```
 
 ## Usage
 
+At its simplest:
+
 ```
 input {
   kinesis {
@@ -14,3 +24,67 @@ input {
   }
 }
 ```
+
+### Event Format
+
+The CloudWatch Logs codec breaks each multi-event subscription record into
+individual events. It does this by iterating over the `logEvents` field, and
+merging each event with all other top-level fields. The codec drops the
+`logEvents` field from the final event.
+
+For example, given a subscription record:
+
+```
+{
+    "owner": "123456789012",
+    "logGroup": "Example",
+    "logStream": "Example1",
+    "subscriptionFilters": [
+        "RootAccess"
+    ],
+    "messageType": "DATA_MESSAGE",
+    "logEvents": [
+        {
+            "id": "1",
+            "timestamp": 1478014822000,
+            "message": "event1"
+        },
+        {
+            "id": "2",
+            "timestamp": 1478014825000,
+            "message": "event2"
+        }
+    ]
+}
+```
+
+...this codec would yield two individual events:
+
+```
+[
+  {
+      "owner": "123456789012",
+      "logGroup": "Example",
+      "logStream": "Example1",
+      "subscriptionFilters": [
+          "RootAccess"
+      ],
+      "messageType": "DATA_MESSAGE",
+      "id": "1",
+      "timestamp": 1478014822000,
+      "message": "event1"
+  },
+  {
+      "owner": "123456789012",
+      "logGroup": "Example",
+      "logStream": "Example1",
+      "subscriptionFilters": [
+          "RootAccess"
+      ],
+      "messageType": "DATA_MESSAGE",
+      "id": "2",
+      "timestamp": 1478014825000,
+      "message": "event2"
+  }
+]
+```

data/lib/logstash/codecs/cloudwatch_logs.rb

@@ -1,5 +1,7 @@
 # encoding: utf-8
-require "logstash/codecs/base"
+require 'logstash/codecs/base'
+require 'logstash/timestamp'
+require 'logstash/event'
 require 'logstash/json'
 require 'zlib'
 
@@ -8,11 +10,15 @@ require 'zlib'
 class LogStash::Codecs::CloudWatchLogs < LogStash::Codecs::Base
   config_name "cloudwatch_logs"
 
+  # Disable the gzip decompression phase if your events have already
+  # been decompressed by the input processor.
+  config :decompress, :validate => :boolean, :default => true
+
   public
   def register; end
 
   def decode(data, &block)
-    data = decompress(StringIO.new(data))
+    data = decompress(StringIO.new(data)) if @decompress
     parse(LogStash::Json.load(data), &block)
   end
 
@@ -25,11 +31,13 @@ class LogStash::Codecs::CloudWatchLogs < LogStash::Codecs::Base
   end
 
   def parse(json, &block)
-    base = json.reject { |k,_| k == "logEvents" }.freeze
-    events = json["logEvents"]
+    events = json.delete("logEvents")
+    json.freeze
 
     events.each do |event|
-      yield LogStash::Event.new(base.merge(event))
+      epochmillis = event.delete("timestamp").to_i
+      event[LogStash::Event::TIMESTAMP] = LogStash::Timestamp.at(epochmillis / 1000, (epochmillis % 1000) * 1000)
+      yield LogStash::Event.new(json.merge(event))
     end
   end
 end

data/logstash-codec-cloudwatch_logs.gemspec

@@ -1,12 +1,12 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-codec-cloudwatch_logs'
-  s.version       = '0.0.1'
+  s.version       = '0.0.2'
   s.licenses      = ['Apache License (2.0)']
   s.summary       = "Parse CloudWatch Logs subscription data"
   s.description   = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors       = ["Anthony M."]
   s.email         = 'tony@threadwaste.com'
-  s.homepage      = "https://github.com/threadwaste/logstash-codec-cloudwatchlogs"
+  s.homepage      = "https://github.com/threadwaste/logstash-codec-cloudwatch_logs"
   s.require_paths = ["lib"]
 
   # Files
@@ -19,7 +19,11 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
+  s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
+  s.add_runtime_dependency 'cabin', '~> 0.6'
 
   s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
+  s.add_development_dependency 'logstash-codec-json'
+  s.add_development_dependency 'tins', '1.6'
+ 
 end

data/spec/codecs/cloudwatch_logs_spec.rb

@@ -33,14 +33,14 @@ describe LogStash::Codecs::CloudWatchLogs do
       expect(events.size).to eq 3
 
       events.each do |event|
-        expect(event['owner']).to eq '123456789012'
-        expect(event['logGroup']).to eq 'CloudTrail'
-        expect(event['logStream']).to eq '123456789012_CloudTrail_us-east-1'
-        expect(event['subscriptionFilters']).to eq ['RootAccess']
-        expect(event['messageType']).to eq 'DATA_MESSAGE'
+        expect(event.get('owner')).to eq '123456789012'
+        expect(event.get('logGroup')).to eq 'CloudTrail'
+        expect(event.get('logStream')).to eq '123456789012_CloudTrail_us-east-1'
+        expect(event.get('subscriptionFilters')).to eq ['RootAccess']
+        expect(event.get('messageType')).to eq 'DATA_MESSAGE'
       end
 
-      messages = events.map { |e| e["message"] }
+      messages = events.map { |e| e.get("message") }
       expect(messages).to eq ["first", "second", "third"]
     end
   end

metadata

@@ -1,35 +1,49 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-cloudwatch_logs
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Anthony M.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-01 00:00:00.000000000 Z
+date: 2016-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.0
-    - - "<"
+        version: '1.60'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: 3.0.0
-  name: logstash-core
+        version: '2.99'
+  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.0
-    - - "<"
+        version: '1.60'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  name: cabin
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -44,6 +58,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.16
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  name: tins
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '1.6'
 description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: tony@threadwaste.com
 executables: []
@@ -56,7 +98,7 @@ files:
 - lib/logstash/codecs/cloudwatch_logs.rb
 - logstash-codec-cloudwatch_logs.gemspec
 - spec/codecs/cloudwatch_logs_spec.rb
-homepage: https://github.com/threadwaste/logstash-codec-cloudwatchlogs
+homepage: https://github.com/threadwaste/logstash-codec-cloudwatch_logs
 licenses:
 - Apache License (2.0)
 metadata: