logstash-output-chronix 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 809b920a229b6a68d3d15a66c006255f7130fe55
-  data.tar.gz: 243a71edf32944d0e2d11c4b481b3382c2600efb
+  metadata.gz: 79a2fa0b8712acee630806c6bacc9ff585a71555
+  data.tar.gz: c8d381ff95b73cf7bd954f825536ba4451aab845
 SHA512:
-  metadata.gz: 7051227f0486a1de7944c917fa967e043e6e45419376a35b4a0cf85bc87344465e1771c4607271d1910c297d9bdf32a73f75112354f93f0d2a1177d5b3082195
-  data.tar.gz: 0387b6a2ee030d7ee24eadceaa57400c956e729e9f7b35b1364257813aa705acb4af6b33ece4b5fa5431c16e5eedbab5769be2cb93e81a9d88eaf1866974913a
+  metadata.gz: 2a8f28c532b41215fc056821685eb0e3cfe2f313ff5c2bf0f51dee466529345dc50d86a381769a30c9ff2c19308f909b2e2f3dcea4ae109dc8a81d11825983f7
+  data.tar.gz: 25b29e31b85bdbddbcfa8c20fa0c71f3bfe1e844cff6c3c173f25fe5ee38565ad766ac6fe5d222a10599a41ccb3ff4507bcf6be94641a1a35ba29b7f9168133c

data/README.md

@@ -2,13 +2,22 @@
 
 [![Build Status](https://travis-ci.org/ChronixDB/chronix.logstash.svg?branch=master)](https://travis-ci.org/ChronixDB/chronix.logstash)
 
-This is a plugin for [Logstash](https://github.com/elastic/logstash) to write time series to [Chronix](https://github.com/ChronixDB) released under the Apache 2.0 license.
+This is a plugin for [Logstash](https://github.com/elastic/logstash) to write time series to [Chronix](https://github.com/ChronixDB) released under the [Apache 2.0 License](LICENSE).
 
 ## Install the plugin
 
-There are two options to install the plugin. (check one the official [Logstash Repos](https://github.com/logstash-plugins/logstash-output-example#2-running-your-unpublished-plugin-in-logstash) for reference)
+There are two options to install the plugin. Install via gem install or manual install (check also the [Logstash Repos](https://github.com/logstash-plugins/logstash-output-example#2-running-your-unpublished-plugin-in-logstash) for reference).
 
-### Install in a local Logstash Clone:
+### Install via gem install
+Just type 
+```sh
+gem install logstash-output-chronix
+```
+to install the plugin. See the [configuration-section](#configuration) for configuration options.
+
+### Install manually
+
+#### Install in a local Logstash Clone
 - Edit Logstash `Gemfile` and add the local plugin path, for example:
 ```ruby
 gem "logstash-output-chronix", :path => "/path/to/logstash-output-chronix"
@@ -22,7 +31,7 @@ bin/plugin install --no-verify
 bin/logstash -e your_config.conf
 ```
 
-### Run in an installed Logstash
+#### Run in an installed Logstash
 
 You can use the same method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
 
@@ -54,6 +63,7 @@ chronix {
   host => "192.168.0.1"       # default is 'localhost'
   port => "8983"              # default is '8983'
   path => "/solr/chronix/"    # default is '/solr/chronix/'
+  threshold => 10             # default is 10 (every delta with (delta - prev_delta) < threshold will be nulled)
   flush_size => 100           # default is '100' (Number of events to queue up before writing to Solr)
   idle_flush_time => 30       # default is '30'  (Amount of time since the last flush before a flush is done)
 }

data/lib/logstash/outputs/chronix.rb

@@ -12,6 +12,7 @@ require "rubygems"
 require "stud/buffer"
 require "zlib"
 require_relative "proto/Point.rb"
+require_relative "proto/StracePoint.rb"
 
 class LogStash::Outputs::Chronix < LogStash::Outputs::Base
   include Stud::Buffer
@@ -27,6 +28,9 @@ class LogStash::Outputs::Chronix < LogStash::Outputs::Base
   # path to chronix, default: /solr/chronix/
   config :path, :validate => :string, :default => "/solr/chronix/"
 
+  # threshold for delta-calculation, every (delta - prev_delta) < threshold will be nulled
+  config :threshold, :validate => :number, :default => 10
+
   # Number of events to queue up before writing to Solr
   config :flush_size, :validate => :number, :default => 100
 
@@ -59,10 +63,27 @@ class LogStash::Outputs::Chronix < LogStash::Outputs::Base
 
   public
   def flush(events, close=false)
+    pointHash = createPointHash(events)
+
+    documents = []
+
+    # iterate through pointHash and create a solr document
+    pointHash.each { |metric, phash|
+      documents << createSolrDocument(metric, phash)
+    }
+
+    # send to chronix
+    @solr.add documents
+    @solr.update :data => '<commit/>'
+  end #def flush
+
+  # this method iterates through all events and creates a hash with different lists of points sorted by metric
+  def createPointHash(events)
     pointHash = Hash.new
 
     # add each event to our hash, sorted by metrics as key
     events.each do |event|
+
       eventData = event.to_hash()
 
       # format the timestamp to unix format
@@ -71,27 +92,37 @@ class LogStash::Outputs::Chronix < LogStash::Outputs::Base
 
       # if there is no list for the current metric -> create a new one
       if pointHash[metric] == nil
-        pointHash[metric] = {"startTime" => timestamp, "endTime" => timestamp, "points" => Chronix::Points.new}
+        pointHash[metric] = {"startTime" => timestamp, "lastTimestamp" => 0, "points" => Chronix::Points.new, "prevDelta" => 0, "timeSinceLastDelta" => 0, "lastStoredDate" => timestamp}
       end
 
-      pointHash[metric]["endTime"] = timestamp
+      if pointHash[metric]["lastTimestamp"] == 0
+        delta = 0
+      else
+        delta = timestamp - pointHash[metric]["lastTimestamp"]
+      end
 
-      # insert the current point in our list
-      pointHash[metric]["points"].p << createChronixPoint(timestamp, eventData["value"])
+      if (almostEquals(delta, pointHash[metric]["prevDelta"]) && noDrift(timestamp, pointHash[metric]["lastStoredDate"], pointHash[metric]["timeSinceLastDelta"]))
+        # insert the current point in our list
+        pointHash[metric]["points"].p << createChronixPoint(0, eventData["value"], eventData["chronix_type"])
 
-    end #end do
-    
-    documents = []
+        pointHash[metric]["timeSinceLastDelta"] += 1
 
-    # iterate through pointHash and create a solr document
-    pointHash.each { |metric, phash|
-      documents << createSolrDocument(metric, phash)
-    }
+      else
+        # insert the current point in our list
+        pointHash[metric]["points"].p << createChronixPoint(delta, eventData["value"], eventData["chronix_type"])
 
-    # send to chronix
-    @solr.add documents
-    @solr.update :data => '<commit/>'
-  end #def flush
+        pointHash[metric]["timeSinceLastDelta"] = 1
+        pointHash[metric]["lastStoredDate"] = timestamp
+      end
+
+      # save current timestamp as lastTimestamp and the previousOffset
+      pointHash[metric]["lastTimestamp"] = timestamp
+      pointHash[metric]["prevDelta"] = delta
+
+    end #end do
+
+    return pointHash
+  end
 
   # this method zips and base64 encodes the list of points
   def zipAndEncode(points)
@@ -109,12 +140,32 @@ class LogStash::Outputs::Chronix < LogStash::Outputs::Base
     return Base64.strict_encode64(data)
   end
 
-  def createChronixPoint(timestamp, value)
-    return Chronix::Point.new( :t => timestamp, :v => value )
+  def createChronixPoint(delta, value, type = "")
+    if type == "strace"
+      return Chronix::StracePoint.new( :t => delta, :v => value )
+    else
+      return Chronix::Point.new( :t => delta, :v => value )
+    end
   end
 
   def createSolrDocument(metric, phash)
-    return { :metric => metric, :start => phash["startTime"], :end => phash["endTime"], :data => zipAndEncode(phash["points"]) }
+    endTime = phash["lastTimestamp"] # maybe use startTime + delta here?!
+    return { :metric => metric, :start => phash["startTime"], :end => endTime, :data => zipAndEncode(phash["points"]), :threshold => @threshold }
+  end
+
+  # checks if two offsets are almost equals
+  def almostEquals(delta, prevDelta)
+    diff = (delta - prevDelta).abs
+
+    return (diff <= @threshold)
+  end
+
+  # checks if there is a drift
+  def noDrift(timestamp, lastStoredDate, timeSinceLastDelta)
+    calcMaxOffset = @threshold * timeSinceLastDelta
+    drift = lastStoredDate + calcMaxOffset - timestamp.to_i
+
+    return (drift <= (@threshold / 2))
   end
 
 end # class LogStash::Outputs::Chronix

data/lib/logstash/outputs/proto/StracePoint.proto

@@ -0,0 +1,15 @@
+syntax = "proto3";
+option optimize_for = SPEED;
+
+//Our point
+message StracePoint {
+    //The date as int64 (java long)
+    int64 t = 1;
+    string v = 2;
+}
+
+//The data of a time series is a list of points
+message StracePoints {
+    //The list of points
+    repeated StracePoint p = 1;
+}

data/lib/logstash/outputs/proto/StracePoint.rb

@@ -0,0 +1,14 @@
+# NOT GENERATED
+
+require 'protobuf/message'
+
+module Chronix
+  class StracePoint < ::Protobuf::Message
+    optional :int64, :t, 1
+    required :string, :v, 2
+  end
+
+  class StracePoints < ::Protobuf::Message
+    repeated ::Chronix::StracePoint, :p, 1
+  end
+end

data/logstash-output-chronix.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-chronix'
-  s.version         = "0.1.0"
+  s.version         = "0.1.1"
   s.licenses = ["Apache License (2.0)"]
   s.summary = "This output stores your logs in chronix"
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -23,5 +23,5 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "stud"
   s.add_runtime_dependency "rsolr"
   s.add_runtime_dependency "protobuf"
-  s.add_development_dependency "logstash-devutils"
+  s.add_development_dependency "logstash-devutils", ">= 0.0.19", "< 0.0.20"
 end

data/spec/outputs/chronix_spec.rb

@@ -36,11 +36,13 @@ describe LogStash::Outputs::Chronix do
   end
 
   context "test basic functions zip and encode, createPoint, createDocument" do
-    subject { LogStash::Outputs::Chronix.new( "flush_size" => 3, "idle_flush_time" => 10 ) }
+    # no flushing of the buffer needed, that's why we use 3 as flush_size here
+    subject { LogStash::Outputs::Chronix.new( "threshold" => 10, "flush_size" => 3, "idle_flush_time" => 10 ) }
 
     let(:ttimestamp) { "1459353272" }
     let(:tmetric) { "test1" }
     let(:tvalue) { "10.5" }
+    let(:svalue) { "string" }
     let(:events) { [LogStash::Event.new("metric" => tmetric, "value" => tvalue)] }
 
     it "should return a Chronix::Point" do
@@ -48,6 +50,12 @@ describe LogStash::Outputs::Chronix do
       expectedResult = Chronix::Point.new( :t => ttimestamp, :v => tvalue )
       expect(point).to eq(expectedResult)
     end
+
+    it "should return a Chronix::StracePoint" do
+      point = subject.createChronixPoint(0, svalue, "strace")    
+      expectedResult = Chronix::StracePoint.new( :t => 0, :v => svalue )
+      expect(point).to eq(expectedResult)
+    end
  
     it "should return a zipped and base64 encoded string containing the data" do
       points = Chronix::Points.new
@@ -56,12 +64,20 @@ describe LogStash::Outputs::Chronix do
       expect(subject.zipAndEncode(points)).to eq(expectedResult)
     end
 
+    it "should create a correct point hash" do
+      points = Chronix::Points.new
+      points.p << subject.createChronixPoint(0, tvalue)
+      phash = {tmetric => {"startTime" => ttimestamp.to_i, "lastTimestamp" => ttimestamp.to_i, "points" => points, "prevDelta" => 0, "timeSinceLastDelta" => 1, "lastStoredDate" => ttimestamp.to_i}}
+      events = [LogStash::Event.new("metric" => tmetric, "value" => tvalue, "@timestamp" => "2016-03-30T15:54:32.172Z")]
+      expect(subject.createPointHash(events)).to eq(phash)
+    end
+
     it "should create a valid document" do
       points = Chronix::Points.new
       points.p << subject.createChronixPoint(ttimestamp, tvalue)
-      phash = {"startTime" => ttimestamp, "endTime" => ttimestamp, "points" => points}
+      phash = {"startTime" => ttimestamp, "lastTimestamp" => ttimestamp, "points" => points}
       document = subject.createSolrDocument(tmetric, phash)
-      sampleDoc = { :metric => tmetric, :start => phash["startTime"], :end => phash["endTime"], :data => "H4sIAAAAAAAA/+Pi59jx9v12VkEGMFB1AACWVOXHEQAAAA==" }
+      sampleDoc = { :metric => tmetric, :start => phash["startTime"], :end => phash["lastTimestamp"], :data => "H4sIAAAAAAAA/+Pi59jx9v12VkEGMFB1AACWVOXHEQAAAA==", :threshold => 10 }
       expect(document).to eq(sampleDoc)
     end
 
@@ -71,20 +87,85 @@ describe LogStash::Outputs::Chronix do
     end
   end
 
+  context "test delta calculation" do
+    # no flushing of the buffer needed, that's why we use 3 as flush_size here
+    subject { LogStash::Outputs::Chronix.new( "threshold" => 10, "flush_size" => 3, "idle_flush_time" => 10 ) }
+
+    let(:tmetric) { "test1" }
+    let(:tvalue) { "10.5" }
+    let(:events) { [LogStash::Event.new("metric" => tmetric, "value" => tvalue)] }
+    
+    p_ev = []
+    p_ev << LogStash::Event.new("metric" => "test1", "value" => "1.0", "@timestamp" => "2016-05-10T15:00:10.000Z")
+    p_ev << LogStash::Event.new("metric" => "test1", "value" => "2.0", "@timestamp" => "2016-05-10T15:00:20.000Z")
+    p_ev << LogStash::Event.new("metric" => "test1", "value" => "3.0", "@timestamp" => "2016-05-10T15:00:30.000Z")
+    p_ev << LogStash::Event.new("metric" => "test1", "value" => "4.0", "@timestamp" => "2016-05-10T15:00:39.000Z")
+    p_ev << LogStash::Event.new("metric" => "test1", "value" => "5.0", "@timestamp" => "2016-05-10T15:00:48.000Z")
+    p_ev << LogStash::Event.new("metric" => "test1", "value" => "6.0", "@timestamp" => "2016-05-10T15:00:57.000Z")
+    p_ev << LogStash::Event.new("metric" => "test1", "value" => "7.0", "@timestamp" => "2016-05-10T15:01:06.000Z")
+    p_ev << LogStash::Event.new("metric" => "test1", "value" => "8.0", "@timestamp" => "2016-05-10T15:01:15.000Z")
+    p_ev << LogStash::Event.new("metric" => "test1", "value" => "9.0", "@timestamp" => "2016-05-10T15:01:24.000Z")
+    p_ev << LogStash::Event.new("metric" => "test1", "value" => "10.0", "@timestamp" => "2016-05-10T15:01:33.000Z")
+    p_ev << LogStash::Event.new("metric" => "test1", "value" => "11.0", "@timestamp" => "2016-05-10T15:01:42.000Z")
+    p_ev << LogStash::Event.new("metric" => "test1", "value" => "12.0", "@timestamp" => "2016-05-10T15:01:51.000Z")
+    p_ev << LogStash::Event.new("metric" => "test1", "value" => "13.0", "@timestamp" => "2016-05-10T15:02:00.000Z")
+    p_ev << LogStash::Event.new("metric" => "test1", "value" => "14.0", "@timestamp" => "2016-05-10T15:02:09.000Z")
+    p_ev << LogStash::Event.new("metric" => "test1", "value" => "15.0", "@timestamp" => "2016-05-10T15:02:18.000Z")
+
+    it "delta should not be almost equals" do
+      expect(subject.almostEquals(21, 10)).to be false
+    end
+
+    it "delta should be almost equals" do
+      expect(subject.almostEquals(-18, -10)).to be true
+    end
+
+    it "should have no drift" do
+      expect(subject.noDrift(10, 5, 1)).to be true
+    end
+
+    it "should have a drift" do
+      expect(subject.noDrift(10, 5, 2)).to be false
+    end
+
+    it "should return a point hash with the correct timestamps aka delta" do
+      pointHash = subject.createPointHash(p_ev)
+
+      points = Chronix::Points.new
+      points.p << subject.createChronixPoint(0, "1.0")
+      points.p << subject.createChronixPoint(0, "2.0")
+      points.p << subject.createChronixPoint(0, "3.0")
+      points.p << subject.createChronixPoint(0, "4.0")
+      points.p << subject.createChronixPoint(0, "5.0")
+      points.p << subject.createChronixPoint(0, "6.0")
+      points.p << subject.createChronixPoint(0, "7.0")
+      points.p << subject.createChronixPoint(0, "8.0")
+      points.p << subject.createChronixPoint(9, "9.0")
+      points.p << subject.createChronixPoint(0, "10.0")
+      points.p << subject.createChronixPoint(0, "11.0")
+      points.p << subject.createChronixPoint(0, "12.0")
+      points.p << subject.createChronixPoint(0, "13.0")
+      points.p << subject.createChronixPoint(0, "14.0")
+      points.p << subject.createChronixPoint(9, "15.0")
+      
+      expect(pointHash["test1"]["points"]).to eq(points)
+    end
+  end
+
   # these events are needed for the next two test-contexts
-  e1 = LogStash::Event.new("metric" => "test1", "value" => "1.5")
-  e2 = LogStash::Event.new("metric" => "test2", "value" => "2.5")
-  e3 = LogStash::Event.new("metric" => "test1", "value" => "3.5")
-  e4 = LogStash::Event.new("metric" => "test1", "value" => "4.5")
-  e5 = LogStash::Event.new("metric" => "test2", "value" => "5.5")
-  e6 = LogStash::Event.new("metric" => "test3", "value" => "6.5")
-  e7 = LogStash::Event.new("metric" => "test1", "value" => "7.5")
-  e8 = LogStash::Event.new("metric" => "test2", "value" => "8.5")
+  e21 = LogStash::Event.new("metric" => "test1", "value" => "1.5")
+  e22 = LogStash::Event.new("metric" => "test2", "value" => "2.5")
+  e23 = LogStash::Event.new("metric" => "test1", "value" => "3.5")
+  e24 = LogStash::Event.new("metric" => "test1", "value" => "4.5")
+  e25 = LogStash::Event.new("metric" => "test2", "value" => "5.5")
+  e26 = LogStash::Event.new("metric" => "test3", "value" => "6.5")
+  e27 = LogStash::Event.new("metric" => "test1", "value" => "7.5")
+  e28 = LogStash::Event.new("metric" => "test2", "value" => "8.5")
 
   context "adding and removing tests with different metrics" do
     subject { LogStash::Outputs::Chronix.new( "flush_size" => 1, "idle_flush_time" => 10 ) }
 
-    let(:events) { [e1, e2, e3, e4, e5, e6, e7, e8] }
+    let(:events) { [e21, e22, e23, e24, e25, e26, e27, e28] }
 
     it "should have 3 different metrics" do
       expect(solr.size).to eq(3)
@@ -116,9 +197,10 @@ describe LogStash::Outputs::Chronix do
   # test2[0]: 1 elem, test2[1]: 2 elem
   # test3[0]: 1 elem
   context "adding and removing tests with different metrics and buffer-settings" do
+
     subject { LogStash::Outputs::Chronix.new( "flush_size" => 4, "idle_flush_time" => 10 ) }
 
-    let(:events) { [e1, e2, e3, e4, e5, e6, e7, e8] }
+    let(:events) { [e21, e22, e23, e24, e25, e26, e27, e28] }
 
     it "should have 3 different metrics" do
       expect(solr.size).to eq(3)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-chronix
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Max Jalowski
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-26 00:00:00.000000000 Z
+date: 2016-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -92,14 +92,20 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.19
+    - - <
+      - !ruby/object:Gem::Version
+        version: 0.0.20
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.19
+    - - <
+      - !ruby/object:Gem::Version
+        version: 0.0.20
 description: This gem is a logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
   a stand-alone program
@@ -111,6 +117,8 @@ files:
 - lib/logstash/outputs/chronix.rb
 - lib/logstash/outputs/proto/Point.proto
 - lib/logstash/outputs/proto/Point.rb
+- lib/logstash/outputs/proto/StracePoint.proto
+- lib/logstash/outputs/proto/StracePoint.rb
 - spec/chronix_helper.rb
 - spec/outputs/chronix_spec.rb
 - spec/spec_helper.rb