fluent-plugin-eventlastvalue 0.0.2

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,52 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-eventlastvalue (0.0.1)
+      fluentd
+      redis
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    cool.io (1.3.0)
+    diff-lcs (1.2.4)
+    fluentd (0.12.7)
+      cool.io (>= 1.2.2, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      json (>= 1.4.3)
+      msgpack (>= 0.5.11, < 0.6.0)
+      sigdump (~> 0.2.2)
+      string-scrub (>= 0.0.3)
+      tzinfo (>= 1.0.0)
+      tzinfo-data (>= 1.0.0)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.6.0)
+    json (1.8.1)
+    msgpack (0.5.11)
+    rake (10.3.2)
+    redis (3.0.7)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.4)
+    rspec-expectations (2.14.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.1)
+    sigdump (0.2.2)
+    string-scrub (0.0.5)
+    thread_safe (0.3.4)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2015.2)
+      tzinfo (>= 1.0.0)
+    yajl-ruby (1.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  fluent-plugin-eventlastvalue!
+  rake
+  rspec

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Michael Arick and Change.org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,75 @@
+fluent-plugin-eventlastvalue
+==========================
+
+This plugin is designed to find the last value in a specified key and pass it through as lastvalue as a re-emit. As it's a buffered plugin it will write or re-emit at a (tunable) sane pace.
+
+##Example
+
+Given a set of input like
+
+```
+count_stream { 'time': 1413544890, 'count': 28, 'id': 1337 }
+count_stream { 'time': 1413544830, 'count': 24, 'id': 33864 }
+count_stream { 'time': 1413544860, 'count': 25, 'id': 12345, ... }
+count_stream { 'time': 1413544890, 'count': 18, 'id': 40555 }
+count_stream { 'time': 1413544830, 'count': 5, 'id': 12345 }
+count_stream { 'time': 1413544860, 'count': 6, 'id': 12345 }
+```
+
+With a conf like
+
+```
+<match count_stream>
+    type eventlastvalue
+    emit_to output.lastvalue
+    id_key id
+    comparator_key timestamp
+    last_value_key count
+    flush_interval 1m
+</match>
+```
+
+You would get
+
+```
+output.lastvalue { 'id': 12345, 'count': 6 }
+output.lastvalue { 'id': 1337, 'count': 28 }
+output.lastvalue { 'id': 33864, 'count': 24 }
+output.lastvalue { 'id': 40555, 'count': 18 }
+```
+
+##Installation
+
+OSX
+
+    /opt/td-agent/embedded/bin/gem install fluent-plugin-eventlastvalue
+
+or
+
+    fluent-gem install fluent-plugin-eventlastvalue
+
+
+##Configuration
+
+###Parameters
+
+#### Basic
+
+- **id_key** (**required**)
+    - The key within the record that identifies a group of events to select from.
+
+- **last_value_key** (**required**)
+    - the key from whose values we want to record the last
+
+- **emit_to** (optional) - *string*
+    - Tag to re-emit with
+        - *default: debug.events*
+- **comparator_key** (optional)
+    - Key to use for numeric comparison, used to ensure that the last received message is the one that is counted. If none is set it will record the last received event.
+        - *default: nil
+
+#### Other
+
+- **flush_interval** (optional)
+    - Provided from **Fluent::BufferedOutput** time in seconds between flushes
+        - *default: 60*

data/fluent-plugin-eventlastvalue.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-eventlastvalue"
+  gem.version     = "0.0.2"
+  gem.authors     = ["Michael Arick", "Change.org"]
+  gem.email       = ["marick@change.org"]
+  gem.homepage    = "https://github.com/change/fluent-plugin-eventlastvalue"
+  gem.summary     = %q{Fluentd plugin to find the last value in a time-period }
+  gem.description = %q{Fluentd plugin to find the last value in a time-period of a field and emit it or write it to redis}
+  gem.license     = "MIT"
+
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency "redis"
+  gem.add_runtime_dependency "bundler"
+
+  gem.add_development_dependency "bundler"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rspec"
+end

data/lib/fluent/plugin/out_eventlastvalue.rb

@@ -0,0 +1,45 @@
+class Fluent::EventLastValueOutput < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('eventlastvalue', self)
+  def initialize
+    super
+  end
+
+  config_param :emit_to, :string, :default => 'debug.events'
+  config_param :id_key, :string, :default => 'id'
+  config_param :last_value_key, :string # REQUIRED
+  config_param :comparator_key, :string, :default => nil
+
+  attr_accessor :last_values
+
+  def configure(conf)
+    super
+  end
+
+  def start
+    super
+  end
+
+  def format(tag, time, record)
+    return '' unless record[@last_value_key]
+    [record[@id_key], record[@last_value_key], (record[@comparator_key] || 0).to_f].to_json + "\n"
+  end
+
+  def write(chunk)
+    last_values = Hash.new {|hash, key| hash[key] = Hash.new {|h,k| h[k] = nil } }
+    last_times = Hash.new {|hash, key| hash[key] = 0}
+    chunk.open do |io|
+      items = io.read.split("\n")
+      items.each do |item|
+        key, event, t = JSON.parse(item)
+        if @comparator_key.nil? || last_times[key] < t
+          last_times[key] = t
+          last_values[key] = event
+        end
+      end
+    end
+
+    last_values.each do |key, value|
+      Fluent::Engine.emit(@emit_to, Time.now.to_i, @id_key => key, @last_value_key => value, 'ts' => Time.now.to_s)
+    end
+  end
+end

data/spec/out_eventlastvalue_spec.rb

@@ -0,0 +1,96 @@
+require 'spec_helper'
+
+describe Fluent::EventLastValueOutput do
+  before { Fluent::Test.setup }
+
+
+  describe '#format' do
+    let (:conf) {
+      %[
+        emit_to output.lastvalue
+        id_key id
+        comparator_key timestamp
+        last_value_key count
+      ]
+    }
+    let (:eventlastvalue) { Fluent::Test::BufferedOutputTestDriver.new(Fluent::EventLastValueOutput.new).configure(conf) }
+    context 'the input contains the last_value key' do
+      it 'produces the expected output' do
+        eventlastvalue.tag = 'something'
+        eventlastvalue.emit( { 'id' => '4444', 'timestamp' => 123456789, 'count' => 12345 }, Time.now )
+        eventlastvalue.expect_format ["4444", 12345, 123456789].to_json + "\n"
+        eventlastvalue.run
+      end
+    end
+
+    context 'the input does not contain the last_value_key' do
+      it 'does not add it to the buffer' do
+        eventlastvalue.tag = 'something'
+        eventlastvalue.emit( { 'not_last_value_key' => 12345}, Time.now )
+        eventlastvalue.expect_format ''
+        eventlastvalue.run
+      end
+    end
+  end
+
+  describe 'output' do
+    let (:conf) {
+      %[
+        emit_to output.lastvalue
+        id_key input_id
+        last_value_key count
+        comparator_key timestamp
+      ]
+    }
+    let (:input) {
+      %[{"email": "john.doe@example.com", "count": 12345, "timestamp": "1", "input_id": "1234"}
+        {"email": "john.doe@example.com", "count": 12340, "timestamp": "6", "input_id": "1234"}
+        {"email": "john.doe@example.com", "count": 12346, "timestamp": "2", "input_id": "1234"}
+        {"email": "john.doe@example.com", "count": 12344, "timestamp": "3", "input_id": "1234"}
+        {"email": "john.doe@example.com", "count": 12342, "timestamp": "4", "input_id": "1234"}
+        {"email": "john.doe@example.com", "count": 12349, "timestamp": "5", "input_id": "1234"}]
+    }
+    let (:eventlastvalue) { Fluent::Test::BufferedOutputTestDriver.new(Fluent::EventLastValueOutput.new).configure(conf) }
+
+    context "a comparator key is specified" do
+      it "formats the lastvalue against the provided tag" do
+        line = input.split("\n").first
+        data = JSON.parse line
+        eventlastvalue.emit data, Time.now
+        output = eventlastvalue.run
+        expect(output['1234']).to eq 12345
+      end
+
+      it "returns the latest count given the comparator key" do
+        input.split("\n").each do |line|
+          data = JSON.parse line
+          eventlastvalue.emit data, Time.now
+        end
+
+        output = eventlastvalue.run
+        expect(output['1234']).to eq 12340
+      end
+    end
+
+    context "a comparator key is NOT specified" do
+      let (:conf) {
+        %[
+          emit_to output.lastvalue
+          id_key input_id
+          last_value_key count
+        ]
+      }
+
+      let (:eventlastvalue) { Fluent::Test::BufferedOutputTestDriver.new(Fluent::EventLastValueOutput.new).configure(conf) }
+      it "returns the last received count" do
+        input.split("\n").each do |line|
+          data = JSON.parse line
+          eventlastvalue.emit data, Time.now
+        end
+
+        output = eventlastvalue.run
+        expect(output['1234']).to eq 12349
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,11 @@
+# encoding: UTF-8
+require 'rubygems'
+require 'bundler'
+Bundler.setup(:default, :test)
+Bundler.require(:default, :test)
+
+require 'fluent/test'
+require 'rspec'
+
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+require 'fluent/plugin/out_eventlastvalue'

metadata

@@ -0,0 +1,154 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-eventlastvalue
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- Michael Arick
+- Change.org
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin to find the last value in a time-period of a field and
+  emit it or write it to redis
+email:
+- marick@change.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- fluent-plugin-eventlastvalue.gemspec
+- lib/fluent/plugin/out_eventlastvalue.rb
+- spec/out_eventlastvalue_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/change/fluent-plugin-eventlastvalue
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Fluentd plugin to find the last value in a time-period
+test_files:
+- spec/out_eventlastvalue_spec.rb
+- spec/spec_helper.rb