logstash-codec-cloudwatch_logs 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c93c61a0f6290abeaa6435db7b5abd7eaea0e431
+  data.tar.gz: 2f24162de1251bad7eeeb8dfcaf0bc40de815dab
+SHA512:
+  metadata.gz: a942cf6850c35d2675bbad4304168157134a5c0883cc2b1350f4e2516ee199575a8f3be1da913f17804a45c1242f2510d27a0a7f30b6e9120d69af739e1dc8fc
+  data.tar.gz: 75b69c66dde11669ee7b97d05e1473f3e2873a5fcd17708392b76575e18f74f636bac0741726fe310132444d9f1b632e26cd1f2626616937733c15a8dd8c43b6

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,14 @@
+Copyright (c) 2015 Anthony M.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+

data/README.md

@@ -0,0 +1,16 @@
+# Logstash Cloudwatch Logs Codec
+
+[![Travis Build Status](https://travis-ci.org/threadwaste/logstash-codec-cloudwatch_logs.svg)](https://travis-ci.org/threadwaste/logstash-codec-cloudwatch_logs)
+
+Parse [CloudWatch Logs subscriptions](http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/Subscriptions.html#DestinationKinesisExample) sent to Kinesis.
+
+## Usage
+
+```
+input {
+  kinesis {
+    kinesis_stream_name => "stream"
+    codec => cloudwatch_logs
+  }
+}
+```

data/lib/logstash/codecs/cloudwatch_logs.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+require 'logstash/json'
+require 'zlib'
+
+
+# Parse CloudWatch Logs
+class LogStash::Codecs::CloudWatchLogs < LogStash::Codecs::Base
+  config_name "cloudwatch_logs"
+
+  public
+  def register; end
+
+  def decode(data, &block)
+    data = decompress(StringIO.new(data))
+    parse(LogStash::Json.load(data), &block)
+  end
+
+  private
+  def decompress(data)
+    gz = Zlib::GzipReader.new(data)
+    gz.read
+  rescue Zlib::Error, Zlib::GzipFile::Error => e
+    @logger.error("Error decompressing CloudWatch Logs data: #{e}")
+  end
+
+  def parse(json, &block)
+    base = json.reject { |k,_| k == "logEvents" }.freeze
+    events = json["logEvents"]
+
+    events.each do |event|
+      yield LogStash::Event.new(base.merge(event))
+    end
+  end
+end

data/logstash-codec-cloudwatch_logs.gemspec

@@ -0,0 +1,25 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-codec-cloudwatch_logs'
+  s.version       = '0.0.1'
+  s.licenses      = ['Apache License (2.0)']
+  s.summary       = "Parse CloudWatch Logs subscription data"
+  s.description   = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors       = ["Anthony M."]
+  s.email         = 'tony@threadwaste.com'
+  s.homepage      = "https://github.com/threadwaste/logstash-codec-cloudwatchlogs"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
+
+  s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
+end

data/spec/codecs/cloudwatch_logs_spec.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/codecs/cloudwatch_logs"
+
+describe LogStash::Codecs::CloudWatchLogs do
+  let!(:raw_data) do
+    data = StringIO.new
+    data << '{'
+    data << '"owner":"123456789012",'
+    data << '"logGroup":"CloudTrail",'
+    data << '"logStream":"123456789012_CloudTrail_us-east-1",'
+    data << '"subscriptionFilters":["RootAccess"],"messageType":"DATA_MESSAGE","logEvents":[{"id":"31953106606966983378809025079804211143289615424298221568","timestamp":1432826855000,"message":"first"},{"id":"31953106606966983378809025079804211143289615424298221569","timestamp":1432826855000,"message":"second"},{"id":"31953106606966983378809025079804211143289615424298221570","timestamp":1432826855000,"message":"third"}]}'
+
+    data.rewind
+    data
+  end
+
+  describe '#decode' do
+    it 'decompresses and parses CloudWatch Logs data' do
+      events = []
+
+      zipped = StringIO.new('', 'r+b')
+      zipper = Zlib::GzipWriter.new(zipped)
+      zipper.write(raw_data.read)
+      zipper.finish
+
+      zipped.rewind
+
+      subject.decode(zipped.string) do |event|
+        events << event
+      end
+
+      expect(events.size).to eq 3
+
+      events.each do |event|
+        expect(event['owner']).to eq '123456789012'
+        expect(event['logGroup']).to eq 'CloudTrail'
+        expect(event['logStream']).to eq '123456789012_CloudTrail_us-east-1'
+        expect(event['subscriptionFilters']).to eq ['RootAccess']
+        expect(event['messageType']).to eq 'DATA_MESSAGE'
+      end
+
+      messages = events.map { |e| e["message"] }
+      expect(messages).to eq ["first", "second", "third"]
+    end
+  end
+end

metadata

@@ -0,0 +1,86 @@
+--- !ruby/object:Gem::Specification
+name: logstash-codec-cloudwatch_logs
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Anthony M.
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-11-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.16
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.16
+description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+email: tony@threadwaste.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/codecs/cloudwatch_logs.rb
+- logstash-codec-cloudwatch_logs.gemspec
+- spec/codecs/cloudwatch_logs_spec.rb
+homepage: https://github.com/threadwaste/logstash-codec-cloudwatchlogs
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: codec
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: Parse CloudWatch Logs subscription data
+test_files:
+- spec/codecs/cloudwatch_logs_spec.rb