gelfd2 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8b3499cd8199e37e4da3a8ff9ca40f532e15369f
+  data.tar.gz: acf0f142d4f915d3f2381be5c96f8b71abf58d70
+SHA512:
+  metadata.gz: be9ca319feb6f9dce7bfb779ecb20af6391e33e4fa110c09a661ecbb6792f8519c2f88563e08e1e827e98d7ca9e33c2cb89a8316dfb41af7b1532d376327623c
+  data.tar.gz: 6f42adedef1b33599c89f27a1bae7e9e183de63055fa7188cfc13a88f2eaceaa91049e349c7a7d61d2ff58ced68315a9c6eb0a74b0f86fec9dfa020247acef75

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.rvmrc

data/.travis.yml

@@ -0,0 +1,8 @@
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.3.1
+  - rbx
+  - jruby
+
+script: "bundle exec rake test"

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,14 @@
+Copyright 2011-2013 John E. Vincent and contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+

data/README.md

@@ -0,0 +1,8 @@
+# gelfd2
+[![Build Status](https://travis-ci.org/ptqa/gelfd2.svg?branch=master)](https://travis-ci.org/ptqa/gelfd2)
+
+
+This is a fork of gelfd (see https://github.com/lusis/gelfd). Gelf is a standalone implementation of GELF (Graylog2 Extended Log Format). This is used by the graylog2 server as an extended format for handling log messages.
+
+## Rationale
+This was originally concieved as a way to provide a graylog2-compatible input for logstash.

data/Rakefile

@@ -0,0 +1,8 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList['test/ts_*.rb']
+  t.verbose = true
+end

data/bin/gelfd

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'socket'
+require 'gelfd2'
+
+server = UDPSocket.new
+puts "Starting up"
+server.bind(nil, 11211)
+trap("INT") { puts "Shutting down"; exit }
+
+loop do
+  data, addr = server.recvfrom(8192)
+  begin
+    res = Gelfd2::Parser.parse(data)
+  rescue Exception => e
+    puts e.message
+  end
+  puts res unless res.nil?
+end

data/gelfd2.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "gelfd2/version"
+
+Gem::Specification.new do |s|
+  s.name        = "gelfd2"
+  s.version     = Gelfd2::VERSION
+  s.authors     = ["John E. Vincent", "ptQa"]
+  s.homepage    = "https://github.com/ptqa/gelfd2"
+  s.summary     = %q{Pure ruby gelf server and decoding library}
+  s.description = %q{Standalone implementation of the Graylog Extended Log Format}
+
+  s.rubyforge_project = "gelfd2"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  s.add_development_dependency "json", "~> 1.8.3"
+  s.add_development_dependency "rake", "~> 11.2.2"
+  s.add_development_dependency "test-unit", "~> 3.2.1"
+  # s.add_runtime_dependency "rest-client"
+end

data/lib/gelfd2/chunked_parser.rb

@@ -0,0 +1,48 @@
+module Gelfd2
+  class ChunkedParser
+    @@chunk_map = Hash.new {|hash,key| hash[key] = {:total_chunks => 0, :chunks => {} } }
+
+    attr_accessor :message_id, :max_chunks, :decoded_data, :chunks, :seen
+
+    def self.parse(data)
+      msg_id = self.parse_chunk(data)
+      if @@chunk_map[msg_id][:chunks].size == @@chunk_map[msg_id][:total_chunks]
+        assemble_chunks(msg_id)
+      end
+    end
+
+    def self.assemble_chunks(msg_id)
+      buff = ''
+      chunks = @@chunk_map[msg_id][:chunks]
+      chunks.keys.sort.each do |k|
+        buff += chunks[k]
+      end
+      begin
+        # TODO
+        # This has a chance for an DoS
+        # you can send a chunked message as a chunked message
+        t = Parser.parse(buff.clone)
+        @@chunk_map.delete(msg_id)
+        t
+      rescue Exception => e
+        "Exception: #{e.message}"
+      end
+    end
+
+    private
+    def self.parse_chunk(data)
+      header = data[0..1]
+      raise NotChunkedDataError, "This doesn't look like a Chunked GELF message!" if header != CHUNKED_MAGIC
+      begin
+        msg_id = data[2..9].unpack('C*').join
+        seq_number, total_number = data[10].ord, data[11].ord
+        zlib_chunk = data[12..-1]
+        raise TooManyChunksError, "#{total_number} greater than #{MAX_CHUNKS}" if total_number > MAX_CHUNKS
+        @@chunk_map[msg_id][:total_chunks] = total_number.to_i
+        @@chunk_map[msg_id][:chunks].merge!({seq_number.to_i => zlib_chunk})
+        msg_id
+      end
+    end
+
+  end
+end

data/lib/gelfd2/exceptions.rb

@@ -0,0 +1,8 @@
+module Gelfd2
+  class NotChunkedDataError < StandardError; end
+  class DuplicateChunkError < StandardError; end
+  class TooManyChunksError < StandardError; end
+  class UnknownHeaderError < StandardError; end
+  class DecodeError < StandardError; end
+  class NotYetImplementedError < StandardError; end
+end

data/lib/gelfd2/gzip_parser.rb

@@ -0,0 +1,16 @@
+require 'zlib'
+module Gelfd2
+  class GzipParser
+
+    def self.parse(data)
+      begin
+        t = Zlib::GzipReader.new(StringIO.new(data))
+        t.read
+      #raise NotYetImplementedError, "GZip decoding is not yet implemented"
+      rescue Exception => e
+        raise DecodeError, "Failed to decode data: #{e}"
+      end
+    end
+
+  end
+end

data/lib/gelfd2/parser.rb

@@ -0,0 +1,22 @@
+module Gelfd2
+  class Parser
+
+    def self.parse(data)
+      header = data[0..1]
+      case header
+      when ZLIB_MAGIC
+        ZlibParser.parse(data)
+      when CHUNKED_MAGIC
+        ChunkedParser.parse(data)
+      when GZIP_MAGIC
+        GzipParser.parse(data)
+      else
+        # by default assume the payload to be "raw, uncompressed" GELF, parsing will fail if it's malformed.
+        data
+      end
+    end
+
+  end
+end
+
+

data/lib/gelfd2/version.rb

@@ -0,0 +1,3 @@
+module Gelfd2
+  VERSION = "0.3.0"
+end

data/lib/gelfd2/zlib_parser.rb

@@ -0,0 +1,15 @@
+require 'zlib'
+module Gelfd2
+  class ZlibParser
+
+    def self.parse(data)
+      begin
+        t = Zlib::Inflate.inflate(data)
+        t
+      rescue Exception => e
+        raise DecodeError, "Failed to decode data: #{e}"
+      end
+    end
+
+  end
+end

data/lib/gelfd2.rb

@@ -0,0 +1,17 @@
+require "gelfd2/version"
+
+module Gelfd2
+  CHUNKED_MAGIC = [0x1e,0x0f].pack('C*').freeze
+  ZLIB_MAGIC = [0x78,0x9c].pack('C*').freeze
+  GZIP_MAGIC = [0x1f,0x8b].pack('C*').freeze
+  UNCOMPRESSED_MAGIC = [0x1f,0x3c].pack('C*').freeze
+  HEADER_LENGTH = 12
+  DATA_LENGTH = 8192 - HEADER_LENGTH
+  MAX_CHUNKS = 128
+end
+
+require File.join(File.dirname(__FILE__), 'gelfd2', 'exceptions')
+require File.join(File.dirname(__FILE__), 'gelfd2', 'zlib_parser')
+require File.join(File.dirname(__FILE__), 'gelfd2', 'gzip_parser')
+require File.join(File.dirname(__FILE__), 'gelfd2', 'chunked_parser')
+require File.join(File.dirname(__FILE__), 'gelfd2', 'parser')

data/test/fixtures/unchunked.uc

@@ -0,0 +1 @@
+{"this":"is","my":"boomstick"}

data/test/tc_chunked.rb

@@ -0,0 +1,17 @@
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'gelfd2'
+require 'json'
+require 'test/unit'
+
+class TestChhunkedGelf < Test::Unit::TestCase
+  JSON_MESSAGE = '{"version":"1.0","host":"somehost","level":"debug","facility":"myapp","short_message":"boom","full_message":"something failed horribly","file":"myapp.rb","timestamp":1315539095.041,"line":105}'
+  FIXTURE_PATH = File.expand_path(File.join(File.dirname(__FILE__), 'fixtures'))
+  def test_chunked_message
+    files = Dir.glob("#{FIXTURE_PATH}/*.chunk")
+    files.each do |file|
+      data = File.open("#{file}", "rb") {|f| f.read}
+      @t = Gelfd2::Parser.parse(data)
+    end
+    assert_equal(JSON_MESSAGE, @t)
+  end
+end

data/test/tc_unchunked.rb

@@ -0,0 +1,14 @@
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'gelfd2'
+require 'json'
+require 'test/unit'
+
+class TestUnchunkedGelf < Test::Unit::TestCase
+  FIXTURE_PATH = File.expand_path(File.join(File.dirname(__FILE__), 'fixtures'))
+  
+  def test_unchunked_message
+    data = File.open("#{FIXTURE_PATH}/unchunked.zl", "rb") {|f| f.read}
+    t = Gelfd2::Parser.parse(data)
+    assert_equal('{"this":"is","my":"boomstick"}', t)
+  end
+end

data/test/tc_uncompressed.rb

@@ -0,0 +1,14 @@
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'gelfd2'
+require 'json'
+require 'test/unit'
+
+class TestUncompressedGelf < Test::Unit::TestCase
+  FIXTURE_PATH = File.expand_path(File.join(File.dirname(__FILE__), 'fixtures'))
+
+  def test_uncompressed_message
+    data = File.open("#{FIXTURE_PATH}/unchunked.uc", "rb") {|f| f.read}
+    t = Gelfd2::Parser.parse(data)
+    assert_equal('{"this":"is","my":"boomstick"}', t)
+  end
+end

data/test/ts_all.rb

@@ -0,0 +1,5 @@
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "test")))
+require 'test/unit'
+require 'tc_unchunked'
+require 'tc_uncompressed'
+require 'tc_chunked'

metadata

@@ -0,0 +1,145 @@
+--- !ruby/object:Gem::Specification
+name: gelfd2
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- John E. Vincent
+- ptQa
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-09-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.3
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 11.2.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 11.2.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.1
+description: Standalone implementation of the Graylog Extended Log Format
+email: 
+executables:
+- gelfd
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/gelfd
+- gelfd2.gemspec
+- lib/gelfd2.rb
+- lib/gelfd2/chunked_parser.rb
+- lib/gelfd2/exceptions.rb
+- lib/gelfd2/gzip_parser.rb
+- lib/gelfd2/parser.rb
+- lib/gelfd2/version.rb
+- lib/gelfd2/zlib_parser.rb
+- test/fixtures/chunked_10_37.chunk
+- test/fixtures/chunked_11_37.chunk
+- test/fixtures/chunked_12_37.chunk
+- test/fixtures/chunked_13_37.chunk
+- test/fixtures/chunked_14_37.chunk
+- test/fixtures/chunked_15_37.chunk
+- test/fixtures/chunked_16_37.chunk
+- test/fixtures/chunked_17_37.chunk
+- test/fixtures/chunked_18_37.chunk
+- test/fixtures/chunked_19_37.chunk
+- test/fixtures/chunked_1_37.chunk
+- test/fixtures/chunked_20_37.chunk
+- test/fixtures/chunked_21_37.chunk
+- test/fixtures/chunked_22_37.chunk
+- test/fixtures/chunked_23_37.chunk
+- test/fixtures/chunked_24_37.chunk
+- test/fixtures/chunked_25_37.chunk
+- test/fixtures/chunked_26_37.chunk
+- test/fixtures/chunked_27_37.chunk
+- test/fixtures/chunked_28_37.chunk
+- test/fixtures/chunked_29_37.chunk
+- test/fixtures/chunked_2_37.chunk
+- test/fixtures/chunked_30_37.chunk
+- test/fixtures/chunked_31_37.chunk
+- test/fixtures/chunked_32_37.chunk
+- test/fixtures/chunked_33_37.chunk
+- test/fixtures/chunked_34_37.chunk
+- test/fixtures/chunked_35_37.chunk
+- test/fixtures/chunked_36_37.chunk
+- test/fixtures/chunked_37_37.chunk
+- test/fixtures/chunked_3_37.chunk
+- test/fixtures/chunked_4_37.chunk
+- test/fixtures/chunked_5_37.chunk
+- test/fixtures/chunked_6_37.chunk
+- test/fixtures/chunked_7_37.chunk
+- test/fixtures/chunked_8_37.chunk
+- test/fixtures/chunked_9_37.chunk
+- test/fixtures/unchunked.uc
+- test/fixtures/unchunked.zl
+- test/tc_chunked.rb
+- test/tc_unchunked.rb
+- test/tc_uncompressed.rb
+- test/ts_all.rb
+homepage: https://github.com/ptqa/gelfd2
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: gelfd2
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: Pure ruby gelf server and decoding library
+test_files: []
+has_rdoc: