logstash-codec-compress_spooler 0.1.6-java → 0.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a68d715f2b9ec925fa606d33dd531b41cc27a1a2
-  data.tar.gz: dce17b823d17bb711742923d124d9ff8cd61851f
+  metadata.gz: 27a52170cd8af6d61263363855f549a4c385aa8f
+  data.tar.gz: 7858feb1f93f7b899a29232d9475a4a1f6ffbdfc
 SHA512:
-  metadata.gz: 39013a5d6c29d8623e8b02aaa9a2bdf93f5f30f911072be727c3defe4e1f554f2de1e1f7b789f6581d5558be6493418561d98e26821a13807b27a093ef0125c3
-  data.tar.gz: c6bebd2f0d572c101c6c32fc85bc1f4d9d50ef62efe02130fbe93fce9d0ff7667f2d6f74f4127f40aad82da4ea7a0ecad00809cc0b2bb460186331b57d72c910
+  metadata.gz: 58be29508e461f884e0dce405b3d6935d6d4c8c359233b43f359589a7195847668d34ebd56635b9f05e1559573bdba00032da163eecc5628275f740808602949
+  data.tar.gz: 4aa77b6f4806fb2143ce511c52f4eee57e5e3f1bacc5883c3cd43f12d93140fecd8ad6c8942e9a813e50c34f90dcd3edfb7f891158a8a492c6dcc04bd06e7ea3

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+# 0.2.0
+
+* Refactoring the Json and Timestamp usage
+* Make usage of mri/jruby dependencies in each platform
+* Add support for tests spec

data/CONTRIBUTORS

@@ -8,6 +8,7 @@ Contributors:
 * Pier-Hugues Pellerin (ph)
 * Richard Pijnenburg (electrical)
 * avleen
+* Pere Urbon-Bayes (purbon)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/Gemfile

@@ -1,2 +1,2 @@
 source 'https://rubygems.org'
-gemspec
+gemspec

data/NOTICE.TXT

@@ -0,0 +1,5 @@
+Elasticsearch
+Copyright 2012-2015 Elasticsearch
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

data/README.md

@@ -13,7 +13,7 @@ Logstash provides infrastructure to automatically generate documentation for thi
 
 ## Need Help?
 
-Need help? Try #logstash on freenode IRC or the logstash-users@googlegroups.com mailing list.
+Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
 
 ## Developing
 

data/lib/logstash/codecs/compress_spooler.rb

@@ -7,43 +7,50 @@ class LogStash::Codecs::CompressSpooler < LogStash::Codecs::Base
   config :compress_level, :validate => :number, :default => 6
 
   public
+
   def register
     require "msgpack"
     require "zlib"
     @buffer = []
   end
 
-  public
   def decode(data)
-    z = Zlib::Inflate.new
-    data = MessagePack.unpack(z.inflate(data))
-    z.finish
-    z.close
-    data.each do |event|
-      event = LogStash::Event.new(event)
-      event["@timestamp"] = Time.at(event["@timestamp"]).utc if event["@timestamp"].is_a? Float
-      yield event
+    decompress(data).each do |event|
+      yield(LogStash::Event.new(event))
     end
   end # def decode
 
-  public
-  def encode(data)
+  def encode(event)
     if @buffer.length >= @spool_size
-      z = Zlib::Deflate.new(@compress_level)
-      @on_event.call data, z.deflate(MessagePack.pack(@buffer), Zlib::FINISH)
-      z.close
+      @on_event.call(compress(@buffer, @compress_level))
       @buffer.clear
     else
-      data["@timestamp"] = data["@timestamp"].to_f
-      @buffer << data.to_hash
+      # use normalize to make sure returned Hash is pure Ruby for
+      # MessagePack#pack which relies on pure Ruby object recognition
+      @buffer << LogStash::Util.normalize(event.to_hash).merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)
     end
   end # def encode
 
-  public
   def teardown
-    if !@buffer.nil? and @buffer.length > 0
-      @on_event.call LogStash::Event.new, @buffer
-    end
+    return if @buffer.empty?
+    @on_event.call(compress(@buffer, @compress_level))
     @buffer.clear
   end
+
+  private
+
+  def compress(data, level)
+    z = Zlib::Deflate.new(level)
+    result = z.deflate(MessagePack.pack(data), Zlib::FINISH)
+    z.close
+    result
+  end
+
+  def decompress(data)
+    z = Zlib::Inflate.new
+    result = MessagePack.unpack(z.inflate(data))
+    z.finish
+    z.close
+    result
+  end
 end # class LogStash::Codecs::CompressSpooler

data/logstash-codec-compress_spooler.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-compress_spooler'
-  s.version         = '0.1.6'
+  s.version         = '0.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "codec that processes compressed spooled data"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -21,12 +21,12 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
+  s.add_development_dependency 'logstash-devutils'
 
   if RUBY_PLATFORM == 'java'
     s.platform = RUBY_PLATFORM
+    s.add_runtime_dependency "msgpack-jruby" #(Apache 2.0 license)
+  else
+    s.add_runtime_dependency "msgpack" #(Apache 2.0 license)
   end
-  s.add_runtime_dependency 'msgpack-jruby'
-
-  s.add_development_dependency 'logstash-devutils'
 end
-

data/spec/codecs/compress_spooler_spec.rb

@@ -1 +1,103 @@
 require "logstash/devutils/rspec/spec_helper"
+require "logstash/codecs/compress_spooler"
+
+describe LogStash::Codecs::CompressSpooler do
+
+  it "should work" do
+    expect(true).to be true
+  end
+
+  context "#decode" do
+    it "should return an event from msgpack data" do
+      codec = LogStash::Codecs::CompressSpooler.new
+      events = [{"foo" => "bar", "baz" => {"bah" => ["a", "b", "c"]}, "@timestamp" => "2014-05-30T02:52:17.929Z"}]
+
+      z = Zlib::Deflate.new
+      data = z.deflate(MessagePack.pack(events), Zlib::FINISH)
+      z.close
+
+      codec.decode(data) do |event|
+        insist { event.is_a? LogStash::Event }
+        insist { event["foo"] } == events.first["foo"]
+        insist { event["baz"] } == events.first["baz"]
+        insist { event["bah"] } == events.first["bah"]
+        insist { event["@timestamp"].to_iso8601 } == events.first["@timestamp"]
+      end
+    end
+  end
+
+  context "#encode" do
+
+    it "should return compressed data from pure ruby hash" do
+      codec = LogStash::Codecs::CompressSpooler.new("spool_size" => 1)
+      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}, "@timestamp" => "2014-05-30T02:52:17.929Z"}
+      event = LogStash::Event.new(data)
+      results = []
+      codec.on_event{|data| results << data}
+
+      # spool_size is one so encode twice to trigger encoding on the 2nd one
+      codec.encode(event)
+      codec.encode(event)
+
+      insist { results.size } == 1
+      z = Zlib::Inflate.new
+      events = MessagePack.unpack(z.inflate(results.first))
+      z.finish
+      z.close
+
+      insist { events.first["foo"] } == data["foo"]
+      insist { events.first["baz"] } == data["baz"]
+      insist { events.first["bah"] } == data["bah"]
+      insist { events.first["@timestamp"] } == "2014-05-30T02:52:17.929Z"
+      insist { events.first["@timestamp"] } == event["@timestamp"].to_iso8601
+    end
+
+    it "should return compressed data from deserialized json with normalization" do
+      codec = LogStash::Codecs::CompressSpooler.new("spool_size" => 1)
+      data = LogStash::Json.load('{"foo": "bar", "baz": {"bah": ["a","b","c"]}, "@timestamp": "2014-05-30T02:52:17.929Z"}')
+      event = LogStash::Event.new(data)
+      results = []
+      codec.on_event{|data| results << data}
+
+      # spool_size is one so encode twice to trigger encoding on the 2nd one
+      codec.encode(event)
+      codec.encode(event)
+
+      insist { results.size } == 1
+      z = Zlib::Inflate.new
+      events = MessagePack.unpack(z.inflate(results.first))
+      z.finish
+      z.close
+
+      insist { events.first["foo"] } == data["foo"]
+      insist { events.first["baz"] } == data["baz"]
+      insist { events.first["bah"] } == data["bah"]
+      insist { events.first["@timestamp"] } == "2014-05-30T02:52:17.929Z"
+      insist { events.first["@timestamp"] } == event["@timestamp"].to_iso8601
+    end
+
+    it "should support teardown and flush pending data" do
+      codec = LogStash::Codecs::CompressSpooler.new("spool_size" => 1)
+      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}, "@timestamp" => "2014-05-30T02:52:17.929Z"}
+      event = LogStash::Event.new(data)
+      results = []
+      codec.on_event{|data| results << data}
+
+      codec.encode(event)
+      codec.teardown
+
+      insist { results.size } == 1
+      z = Zlib::Inflate.new
+      events = MessagePack.unpack(z.inflate(results.first))
+      z.finish
+      z.close
+
+      insist { events.first["foo"] } == data["foo"]
+      insist { events.first["baz"] } == data["baz"]
+      insist { events.first["bah"] } == data["bah"]
+      insist { events.first["@timestamp"] } == "2014-05-30T02:52:17.929Z"
+      insist { events.first["@timestamp"] } == event["@timestamp"].to_iso8601
+    end
+  end
+
+end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-compress_spooler
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.2.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-04-20 00:00:00.000000000 Z
+date: 2015-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,34 +28,36 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: msgpack-jruby
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: msgpack-jruby
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -65,9 +65,11 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
+- NOTICE.TXT
 - README.md
 - Rakefile
 - lib/logstash/codecs/compress_spooler.rb