logstash-codec-cloudfront 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1acb89596fe3f204d4c0ee4f57a5df136adaba53
+  data.tar.gz: 6a46f6f11903cb279301941b0fd84b729121ff15
+SHA512:
+  metadata.gz: ab1fc16e7e90aa297b450e4567e32c6ec7b89091d8fc688df0b5ba6dd29786fd6fc0a0996c8987cb674e618914c9d85de5f52253e901bedb3f7dd64583cb55e8
+  data.tar.gz: c7d3d3fc1aa056dbb3cb241738ae0a0f2225d774398b74d6927600aaf82ce84bf8ef22068ae597fd802b9787ead78d026dc7d28b42fe66c8317306b28a154933

data/.gitignore

@@ -0,0 +1,3 @@
+*.gem
+Gemfile.lock
+.bundle

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -0,0 +1,7 @@
+@files=[]
+
+task :default do
+  system("rake -T")
+end
+
+require "logstash/devutils/rake"

data/lib/logstash/codecs/cloudfront.rb

@@ -0,0 +1,85 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+require "logstash/codecs/plain"
+require "logstash/json"
+
+# This codec will read cloudfront encoded content
+class LogStash::Codecs::Cloudfront < LogStash::Codecs::Base
+  config_name "cloudfront"
+
+  milestone 3
+
+  # The character encoding used in this codec. Examples include "UTF-8" and
+  # "CP1252"
+  #
+  # JSON requires valid UTF-8 strings, but in some cases, software that
+  # emits JSON does so in another encoding (nxlog, for example). In
+  # weird cases like this, you can set the charset setting to the
+  # actual encoding of the text and logstash will convert it for you.
+  #
+  # For nxlog users, you'll want to set this to "CP1252"
+  config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
+
+  public
+  def initialize(params={})
+    super(params)
+    @converter = LogStash::Util::Charset.new(@charset)
+    @converter.logger = @logger
+  end
+
+  public
+  def decode(data)
+    begin
+      @gzip = Zlib::GzipReader.new(data)
+
+      metadata = extract_metadata(@gzip)
+
+      @logger.debug("Cloudfront: Extracting metadata", :metadata => metadata)
+
+      @gzip.each_line do |line|
+        yield create_event(line, metadata)
+      end
+
+    rescue Zlib::Error, Zlib::GzipFile::Error=> e
+      file = data.is_a?(String) ? data : data.class
+
+      @logger.error("Cloudfront codec: We cannot uncompress the gzip file", :filename => file)
+      raise e
+    end
+  end # def decode
+
+  public
+  def create_event(line, metadata)
+    event = LogStash::Event.new("message" => @converter.convert(line))
+    event["cloudfront_version"] = metadata["cloudfront_version"]
+    event["cloudfront_fields"] = metadata["cloudfront_fields"]
+    event
+  end
+
+
+  def extract_metadata(io)
+    version = extract_version(io.gets)
+    fields = extract_fields(io.gets)
+
+    return {
+      "cloudfront_version" => version,
+      "cloudfront_fields" => fields,
+    }
+  end
+
+
+  def extract_version(line)
+    if /^#Version: .+/.match(line)
+      junk, version = line.strip().split(/#Version: (.+)/)
+      version unless version.nil?
+    end
+  end
+
+
+  def extract_fields(line)
+    if /^#Fields: .+/.match(line)
+      junk, format = line.strip().split(/#Fields: (.+)/)
+      format unless format.nil?
+    end
+  end
+end # class LogStash::Codecs::Cloudfront

data/logstash-codec-cloudfront.gemspec

@@ -0,0 +1,28 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-codec-cloudfront'
+  s.version         = '0.1.1'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "This codec may be used to decode (via inputs) cloudfront gziped files"
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Elasticsearch"]
+  s.email           = 'richard.pijnenburg@elasticsearch.com'
+  s.homepage        = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
+
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-codec-plain'
+end
+

data/spec/codecs/cloudfront_spec.rb

@@ -0,0 +1,92 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/codecs/cloudfront"
+require "logstash/errors"
+require "stringio"
+require "zlib"
+
+def compress_with_gzip(io)
+  compressed = StringIO.new('', 'r+b')
+
+  gzip = Zlib::GzipWriter.new(compressed)
+  gzip.write(io.read)
+  gzip.finish
+
+  compressed.rewind
+
+  compressed
+end
+
+describe LogStash::Codecs::Cloudfront do
+  let!(:uncompressed_cloudfront_log) do
+    # Using format from
+    # http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/AccessLogs.html
+    str = StringIO.new
+
+    str << "#Version: 1.0\n"
+    str << "#Fields: date time x-edge-location c-ip x-event sc-bytes x-cf-status x-cf-client-id cs-uri-stem cs-uri-query c-referrer x-page-url​  c-user-agent x-sname x-sname-query x-file-ext x-sid\n"
+    str << "2010-03-12   23:51:20   SEA4   192.0.2.147   connect   2014   OK   bfd8a98bee0840d9b871b7f6ade9908f   rtmp://shqshne4jdp4b6.cloudfront.net/cfx/st​  key=value   http://player.longtailvideo.com/player.swf   http://www.longtailvideo.com/support/jw-player-setup-wizard?example=204   LNX%2010,0,32,18   -   -   -   -\n"
+    str << "2010-03-12   23:51:21   SEA4   192.0.2.222   play   3914   OK   bfd8a98bee0840d9b871b7f6ade9908f   rtmp://shqshne4jdp4b6.cloudfront.net/cfx/st​  key=value   http://player.longtailvideo.com/player.swf   http://www.longtailvideo.com/support/jw-player-setup-wizard?example=204   LNX%2010,0,32,18   myvideo   p=2&q=4   flv   1\n"
+
+    str.rewind
+    str
+  end
+
+  describe "#decode" do
+    it "should create events from a gzip file" do
+      events = []
+
+      subject.decode(compress_with_gzip(uncompressed_cloudfront_log)) do |event|
+        events << event
+      end
+
+      expect(events.size).to eq(2)
+    end
+
+    it 'should extract the metadata of the file' do
+      events = []
+
+      subject.decode(compress_with_gzip(uncompressed_cloudfront_log)) do |event|
+        events << event
+      end
+
+      expect(events.first["cloudfront_version"]).to eq("1.0")
+      expect(events.first["cloudfront_fields"]).to eq("date time x-edge-location c-ip x-event sc-bytes x-cf-status x-cf-client-id cs-uri-stem cs-uri-query c-referrer x-page-url​  c-user-agent x-sname x-sname-query x-file-ext x-sid")
+    end
+  end
+
+  describe "#extract_version" do
+    it "returns the version from a matched string" do
+      line = "#Version: 1.0"
+
+      expect(subject.extract_version(line)).to eq("1.0")
+    end
+
+    it "doesn't return anything if version isnt matched" do
+      line = "Bleh my string"
+      expect(subject.extract_version(line)).to eq(nil)
+    end
+
+    it "doesn't match if #Version is not at the beginning of the string" do
+      line = "2010-03-12   23:53:44   SEA4   192.0.2.4   stop   323914   OK   bfd8a98bee0840d9b871b7f6ade9908f #Version: 1.0 Bleh blah"
+      expect(subject.extract_version(line)).to eq(nil)
+    end
+  end
+
+  describe "#extract_fields" do
+    it "return a string with all the fields" do
+      line = "#Fields: date time x-edge-location c-ip x-event sc-bytes x-cf-status x-cf-client-id cs-uri-stem cs-uri-query c-referrer x-page-url​  c-user-agent x-sname x-sname-query x-file-ext x-sid"
+      expect(subject.extract_fields(line)).to eq("date time x-edge-location c-ip x-event sc-bytes x-cf-status x-cf-client-id cs-uri-stem cs-uri-query c-referrer x-page-url​  c-user-agent x-sname x-sname-query x-file-ext x-sid")
+    end
+
+    it "doesn't return anything if we can the fields list" do
+      line = "Bleh my string"
+      expect(subject.extract_fields(line)).to eq(nil)
+    end
+
+    it "doesnt match if #Fields: is not at the beginning of the string" do
+      line = "2010-03-12   23:53:44   SEA4   192.0.2.4   stop   323914   OK   bfd8a98bee0840d9b871b7f6ade9908f #Fields: 1.0 Bleh blah"
+      expect(subject.extract_fields(line)).to eq(nil)
+    end
+  end
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: logstash-codec-cloudfront
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Elasticsearch
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-11-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: richard.pijnenburg@elasticsearch.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- Rakefile
+- lib/logstash/codecs/cloudfront.rb
+- logstash-codec-cloudfront.gemspec
+- spec/codecs/cloudfront_spec.rb
+homepage: http://www.elasticsearch.org/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: codec
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.4
+signing_key:
+specification_version: 4
+summary: This codec may be used to decode (via inputs) cloudfront gziped files
+test_files:
+- spec/codecs/cloudfront_spec.rb