logstash-codec-lineeof 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 098a38756136c401d608c5b490615ccbec1d97b9
+  data.tar.gz: f6494f73ab63c8748e1a01439b205c6296daa779
+SHA512:
+  metadata.gz: 8e72f6ef34e30231afbae0e52e15831e1aca92787f90b2fca4474226811285dc2156804bcc7f5726f60fd6d93c67ee6f250d926c8214d69d7fad9310d31c8057
+  data.tar.gz: f328da1975307c4517ab8b9eb6dfe3c9f742e5951a89b001ba4349a2c53aedb0c55d31621be37e6bcdf1445c5cc5fc15692788e9484e476b3c6985da201a0a46

data/.gitignore

@@ -0,0 +1,3 @@
+Gemfile.lock
+.ruby-version
+*.gem

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -0,0 +1,7 @@
+@files=[]
+
+task :default do
+  system("rake -T")
+end
+
+require "logstash/devutils/rake"

data/lib/logstash/codecs/lineeof.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+require "logstash/util/charset"
+
+# Line-oriented text data with EOF(end of file) recognition.
+#
+# Decoding behavior: Only whole line events will be emitted.
+#
+# Encoding behavior: Each event will be emitted with a trailing newline.
+class LogStash::Codecs::LineEOF < LogStash::Codecs::Base
+  config_name "lineeof"
+
+  # Set the desired text format for encoding.
+  config :format, :validate => :string
+
+  # The character encoding used in this input. Examples include "UTF-8"
+  # and "cp1252"
+  #
+  # This setting is useful if your log files are in Latin-1 (aka cp1252)
+  # or in another character set other than UTF-8.
+  #
+  # This only affects "plain" format logs since json is UTF-8 already.
+  config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
+
+  public
+  def register
+    require "logstash/util/buftok"
+    @buffer = FileWatch::BufferedTokenizer.new
+    @converter = LogStash::Util::Charset.new(@charset)
+    @converter.logger = @logger
+  end
+
+  public
+  def decode(data)
+    lines = @buffer.extract(data)
+    lines.each_with_index do |line, index|
+      event = LogStash::Event.new("message" => @converter.convert(line))
+      #if end of file is reached?
+      event.tag("eof") if index == lines.size - 1 && data.length < 32768
+      yield event
+    end
+  end # def decode
+
+  public
+  def flush(&block)
+    remainder = @buffer.flush
+    if !remainder.empty?
+      block.call(LogStash::Event.new({"message" => remainder}))
+    end
+  end
+
+  public
+  def encode(data)
+    if data.is_a? LogStash::Event and @format
+      @on_event.call(data.sprintf(@format) + "\n")
+    else
+      @on_event.call(data.to_s + "\n")
+    end
+  end # def encode
+
+end # class LogStash::Codecs::Plain

data/logstash-codec-lineeof.gemspec

@@ -0,0 +1,27 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-codec-lineeof'
+  s.version         = '0.1.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Line-oriented text data with EOF(end of file) recognition."
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Signify"]
+  s.email           = 'dietmar@signifydata.com'
+  s.homepage        = "http://www.signifydata.com"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core', '>= 1.4.0', '< 2.0.0'
+
+  s.add_development_dependency 'logstash-devutils'
+end
+

data/spec/codecs/lineeof_spec.rb

@@ -0,0 +1,67 @@
+# encoding: utf-8
+
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/codecs/line"
+require "logstash/event"
+
+describe LogStash::Codecs::Line do
+  subject do
+    next LogStash::Codecs::Line.new
+  end
+
+  context "#encode" do
+    let (:event) {LogStash::Event.new({"message" => "hello world", "host" => "test"})}
+
+    it "should return a default date formatted line" do
+      expect(subject).to receive(:on_event).once.and_call_original
+      subject.on_event do |e, d|
+        insist {d} == event.to_s + "\n"
+      end
+      subject.encode(event)
+    end
+
+    it "should respect the supplied format" do
+      format = "%{host}"
+      subject.format = format
+      expect(subject).to receive(:on_event).once.and_call_original
+      subject.on_event do |e, d|
+        insist {d} == event.sprintf(format) + "\n"
+      end
+      subject.encode(event)
+    end
+  end
+
+  context "#decode" do
+    it "should return an event from an ascii string" do
+      decoded = false
+      subject.decode("hello world\n") do |e|
+        decoded = true
+        insist { e.is_a?(LogStash::Event) }
+        insist { e["message"] } == "hello world"
+      end
+      insist { decoded } == true
+    end
+
+    it "should return an event from a valid utf-8 string" do
+      subject.decode("München\n") do |e|
+        insist { e.is_a?(LogStash::Event) }
+        insist { e["message"] } == "München"
+      end
+    end
+  end
+
+  context "#flush" do
+    it "should convert charsets" do
+      garbage = [0xD0].pack("C")
+      subject.decode(garbage) do |e|
+        fail "Should not get here."
+      end
+      count = 0
+      subject.flush do |event|
+        count += 1
+        insist { event["message"].encoding } == Encoding::UTF_8
+      end
+      insist { count } == 1
+    end
+  end
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: logstash-codec-lineeof
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Signify
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-08-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: dietmar@signifydata.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- Rakefile
+- lib/logstash/codecs/lineeof.rb
+- logstash-codec-lineeof.gemspec
+- spec/codecs/lineeof_spec.rb
+homepage: http://www.signifydata.com
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: codec
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Line-oriented text data with EOF(end of file) recognition.
+test_files:
+- spec/codecs/lineeof_spec.rb