fluent-plugin-simplearithmetic 0.0.1.pre → 0.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f7a35e0538f55c58d65296c4c8338ecf00a9667b
-  data.tar.gz: dbcec7416b39c15cc4a0f17bb6a8b923dae573fd
+  metadata.gz: 9896bc5f77125536a6e6efec0c519947c8b8c487
+  data.tar.gz: 0416741cba7d514ed3bdaf97f063ad03a3d87f33
 SHA512:
-  metadata.gz: 285494a14b1989efb7521c7aecc4e307fb1d0c75eeb88f3325bcd70ea6d62ad2bc4a88e8ff736fd946f78c5a7b5f4db22e654d44348a9838868d084e0d32eadb
-  data.tar.gz: 93a694065302c235b457bc01ccf629145b4ffec7a414a0c890279690667d606d78ff2090fc84c856e28630a755097048afc160016d38343b82ccdae102badbbc
+  metadata.gz: 6f18b7a4f4f43996d715a41231946bf83eecf67a2af69d096e82a5f72d60a7e0946c62a9a3c88482fc635b3dbd57d0905f95ba3718fdf6d35c89a2d7e4089ed7
+  data.tar.gz: 2574ec4b858814077aa20b9ceb44a4d8f95588fc233ee661229495035394aec602132399e88e2cc35446071eec68ead80be5151540a34d2beb1070a044646f0f

data/README.md

@@ -24,7 +24,7 @@ Suppose you have a message like:
 }
 ```
 
-Now you can calculate with this `td-agent.conf`:
+Now you can calculate with this configuration:
 
 ```
 <match arithmetic.test>
@@ -32,7 +32,7 @@ Now you can calculate with this `td-agent.conf`:
   tag calculated.test
 
   <formulas>
-    total_price   apple * 200 - orange * 100
+    total_price   apple * 200 + orange * 100
 
     # Calculation order is from up to down.
     budget   2000 - total_price
@@ -55,31 +55,89 @@ Calculated results will be:
 	"orange": 3,
 	"time_start": "2001-02-03T04:05:06Z",
 	"time_finish": "2001-02-03T04:06:12Z",
-	"total_price": 1100,
-	"budget": 900,
+	"total_price": 1700,
+	"budget": 300,
 	"time_elapsed": 66.0
 }
 ```
 
+If some fields are already defined before calculation, these fields will be overwritten.
+
 
 ## Configuration
 
+### tag
+The tag prefix for emitted event messages. Default is `simple_arithmetic`.
 
 ### undefined_variables
-1. `nil`
 
-2. `undefined` (default)
+A message such like `{'a':  20, 'x':  50}` with the formulas:
+
+```
+<formulas>
+  c   a + b
+</formulas>
+```
+
+When undefined_variables is `undefined` (default) you get `{'a':  20, 'x':  50}`. Field `'c'` will not be defined.
+
+When `nil` you get `{'a':  20, 'c': nil, 'x':  50}`.
+
 
 ### how_to_process_error
-1. `nil`
 
-2. `undefined`
+A message such like `{'a':  20, 'b':  "string"}` with the formulas:
 
-3. `error_string` (default)
+```
+<formulas>
+  c   a + b
+</formulas>
+```
 
+In ruby, this calculation will be raise an error:
 
-### tag
-The tag prefix for emitted event messages. Default is `simple_arithmetic`.
+```
+irb(main):052:0* a = 20
+=> 20
+irb(main):053:0> b = "String"
+=> "String"
+irb(main):054:0> c = a + b
+TypeError: String can't be coerced into Fixnum
+	from (irb):54:in `+'
+	from (irb):54
+	from /opt/td-agent/embedded/bin/irb:11:in `<main>'
+```
+
+When how_to_process_error is `nil` you get `{'a':  20, 'b': "string", 'c': nil}`.
+
+When `undefined`, you get `{'a':  20, 'b': "string"}`. If an error is raised in the calculation, the field will not be defined.
+
+When `error_string` (default), you get: `{'a':  20, 'b': "String", 'c'=>"String can't be coerced into Fixnum"}`. An error message will be assigned to the field.
+
+### replace_hyphen, replace_dollar
+
+All formulas will be evaluated as ruby sentences. Some json fields will not be fitted as ruby variables. For example, 
+
+```
+<formulas>
+   var-1   a + b
+   var$2   c * d
+</formulas>
+```
+
+will raise an syntax error in the initialize process of fluentd.
+
+To get rid of this case, you can set `replace_hyphen` and `replace_dollar` in the configuration and formulas.
+
+```
+replace_hyphen   __H__
+replace_dollar   __D__
+
+<formulas>
+   var__H__1   a + b
+   var__D__2   c * d
+</formulas>
+```
 
 
 ## Copyright

data/fluent-plugin-simplearithmetic.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-simplearithmetic"
-  spec.version       = "0.0.1.pre"
+  spec.version       = "0.0.1"
   spec.authors       = ["Takahiro Kamatani"]
   spec.email         = ["buhii314@gmail.com"]
   spec.description   = %q{Fluent plugin to calculate messages.}

data/lib/fluent/plugin/out_simple_arithmetic.rb

@@ -11,6 +11,9 @@ module Fluent
     # 'nil', 'undefined', 'error_string'
     config_param :how_to_process_error, :string, :default => 'error_string'
 
+    config_param :replace_hyphen, :string, :default => '__HYPHON__'
+    config_param :replace_dollar, :string, :default => '__DOLLAR__'
+
     attr_accessor :_formulas
 
     def initialize
@@ -41,10 +44,9 @@ module Fluent
 
       # Create functions
       @_formulas = []
-
       def create_func(var, expr)
         begin
-          f_argv = expr.scan(/[a-zA-Z][\w\d\.]*/).uniq.select{|x| not x.start_with?('Time.iso8601')}
+          f_argv = expr.scan(/[a-zA-Z\_][\w\d\.\_]*/).uniq.select{|x| not x.start_with?('Time.iso8601')}
           f = eval('lambda {|' + f_argv.join(',') + '| ' + expr + '}')
           return [f, f_argv]
         rescue SyntaxError
@@ -61,6 +63,10 @@ module Fluent
           @_formulas.push [var, f_argv, formula]
         }
       }
+      if @_formulas.empty?
+        raise Fluent::ConfigError, "No formulas found"
+      end
+
     end
 
     def has_all_keys?(record, argv)
@@ -80,7 +86,30 @@ module Fluent
       return formula.call(*argv)
     end
 
+    # functions for symbols
+    def replace_symbols(record)
+      # 'var-1' -> 'var__HYPHEN__1'
+      new_record = {}
+      record.each_pair {|key, value|
+        new_key = key.gsub('-', @replace_hyphen).gsub('$', @replace_dollar)
+        new_record[new_key] = value
+      }
+      return new_record
+    end
+
+    def restore_symbols(record)
+      # 'var__HYPHEN__1' -> 'var-1'
+      new_record = {}
+      record.each_pair {|key, value|
+        new_key = key.gsub(@replace_hyphen, '-').gsub(@replace_dollar, '$')
+        new_record[new_key] = value
+      }
+      new_record
+    end
+
     def calculate(record)
+      record = replace_symbols(record)
+
       @_formulas.each {|var, f_argv, formula|
         if not has_all_keys?(record, f_argv)
           if @undefined_variables == 'nil'
@@ -100,7 +129,8 @@ module Fluent
           end
         end
       }
-      record
+
+      restore_symbols(record)
     end
 
     def emit(tag, es, chain)

data/test/helper.rb

@@ -22,7 +22,7 @@ unless ENV.has_key?('VERBOSE')
   $log = nulllogger
 end
 
-require 'fluent/plugin/out_simplearithmetic'
+require 'fluent/plugin/out_simple_arithmetic'
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_out_simple_arithmetic.rb

@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+require 'helper'
+
+class SimpleArithmeticOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    type simple_arithmetic
+    tag calculated.test
+    undefined_variables nil
+    how_to_process_error error_string
+
+    <formulas>
+      x3     x1 * 100 - x2
+      var1   Time.iso8601(t1) - Time.iso8601(t2)
+      var2   x3 - var1
+    </formulas>
+  ]
+
+  def create_driver(conf = CONFIG, tag='test.input')
+    Fluent::Test::OutputTestDriver.new(Fluent::SimpleArithmeticOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    # No formulas
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+    # No formulas
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        <formulas>
+
+        </formulas>
+      ]
+    }
+    # Syntax error
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        <formulas>
+          var1   var2 * var3 +
+        </formulas>
+      ]
+    }
+  end
+
+  def test_replace_functions
+    d1 = create_driver %[
+      replace_hyphen  __H__
+      replace_dollar  __D__
+      <formulas>
+        var__H__1  __H__2 * var__D__3
+        __D__3  __D__1 + __D__2
+      </formulas>
+    ]
+    assert_equal '__H__', d1.instance.replace_hyphen
+    assert_equal '__D__', d1.instance.replace_dollar
+    d1.run do
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+      d1.emit({'-2'=>10, 'var$3'=>20}, time)
+      d1.emit({'$1'=>10, '$2'=>20}, time)
+    end
+    assert_equal d1.emits[0][2], {"-2"=>10, "var$3"=>20, "var-1"=>200}
+    assert_equal d1.emits[1][2], {"$1"=>10, "$2"=>20, "$3"=>30}
+  end
+
+  def test_undefined_variables
+    # undefined_variables must be either `nil` or `undefined`
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        undefined_variables   non_existent_config
+        <formulas>
+          a   b + c
+        </formulas>
+      ]
+    }
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    # nil
+    d1 = create_driver %[
+      undefined_variables   nil
+      <formulas>
+        a   b + c
+      </formulas>
+    ]
+    d1.run do
+      d1.emit({'b'=>10, 'c'=>20}, time)
+      d1.emit({'b'=>10, 'non-related'=>100}, time)
+    end
+    assert_equal d1.emits[0][2], {'a'=>30, 'b'=>10, 'c'=>20}
+    assert_equal d1.emits[1][2], {'a'=>nil, 'b'=>10, 'non-related'=>100}
+
+    # undefined
+    d2 = create_driver %[
+      undefined_variables   undefined
+      <formulas>
+        a   b + c
+      </formulas>
+    ]
+    d2.run do
+      d2.emit({'b'=>10, 'c'=>20}, time)
+      d2.emit({'b'=>10, 'non-related'=>100}, time)
+    end
+    assert_equal d2.emits[0][2], {'a'=>30, 'b'=>10, 'c'=>20}
+    assert_equal d2.emits[1][2], {'b'=>10, 'non-related'=>100}
+  end
+
+  def test_how_to_process_error
+    # undefined_variables must be either `nil` or `undefined`, `error_string`
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        how_to_process_error   non_existent_config
+        <formulas>
+          a   b + c
+        </formulas>
+      ]
+    }
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    # nil
+    d1 = create_driver %[
+      how_to_process_error   nil
+      <formulas>
+        a   b + c
+      </formulas>
+    ]
+    d1.run do
+      d1.emit({'b'=>10, 'c'=>20}, time)
+      d1.emit({'b'=>10, 'c'=>'string'}, time)
+    end
+    assert_equal d1.emits[0][2], {'a'=>30, 'b'=>10, 'c'=>20}
+    assert_equal d1.emits[1][2], {'a'=>nil, 'b'=>10, 'c'=>'string'}
+
+    # undefined
+    d2 = create_driver %[
+      how_to_process_error   undefined
+      <formulas>
+        a   b + c
+      </formulas>
+    ]
+    d2.run do
+      d2.emit({'b'=>10, 'c'=>20}, time)
+      d2.emit({'b'=>10, 'c'=>'string'}, time)
+    end
+    assert_equal d2.emits[0][2], {'a'=>30, 'b'=>10, 'c'=>20}
+    assert_equal d2.emits[1][2], {'b'=>10, 'c'=>'string'}
+
+    # error_string
+    d3 = create_driver %[
+      how_to_process_error   error_string
+      <formulas>
+        a   b + c
+      </formulas>
+    ]
+    d3.run do
+      d3.emit({'b'=>10, 'c'=>20}, time)
+      d3.emit({'b'=>10, 'c'=>'string'}, time)
+    end
+    assert_equal d3.emits[0][2], {'a'=>30, 'b'=>10, 'c'=>20}
+    assert_equal d3.emits[1][2], {'a'=>"String can't be coerced into Fixnum",
+                                  'b'=>10, 'c'=>'string'}
+  end
+
+  def test_plus_num_and_string
+    def calculated(record)
+      record['a'] = record['b'] + record['c']
+      record
+    end
+    data = [
+      {'b'=>10, 'c'=>20},
+      {'b'=>'Dr. Strangelove or: ',
+       'c'=>'How I Learned to Stop Worrying and Love the Bomb'},
+    ]
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d = create_driver %[
+      <formulas>
+        a   b + c
+      </formulas>
+    ]
+    d.run do
+      data.each {|record|
+        d.emit(record, time)
+      }
+    end
+    data.each_with_index {|record, index|
+      assert_equal d.emits[index][2], calculated(data[index])
+    }
+  end
+
+  def test_iso8601
+    d = create_driver %[
+      <formulas>
+        time_diff   Time.iso8601(time_finish) - Time.iso8601(time_start)
+      </formulas>
+    ]
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.run do
+      d.emit(
+        {
+          'time_start'  => '2001-02-03T04:05:06Z',
+          'time_finish' => '2001-02-03T04:06:12Z',
+        }, time)
+    end
+    assert_equal d.emits[0][2], {
+      'time_start' =>  '2001-02-03T04:05:06Z',
+      'time_finish'=> '2001-02-03T04:06:12Z',
+      'time_diff'=> 66.0}
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-simplearithmetic
 version: !ruby/object:Gem::Version
-  version: 0.0.1.pre
+  version: 0.0.1
 platform: ruby
 authors:
 - Takahiro Kamatani
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-11 00:00:00.000000000 Z
+date: 2014-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -53,7 +53,7 @@ files:
 - fluent-plugin-simplearithmetic.gemspec
 - lib/fluent/plugin/out_simple_arithmetic.rb
 - test/helper.rb
-- test/plugin/test_out_simplearithmetic.rb
+- test/plugin/test_out_simple_arithmetic.rb
 homepage: https://github.com/buhii/fluent-plugin-simplearithmetic
 licenses:
 - Apache License, Version 2.0
@@ -69,9 +69,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.2.2
@@ -80,4 +80,4 @@ specification_version: 4
 summary: Fluent plugin to calculate messages.
 test_files:
 - test/helper.rb
-- test/plugin/test_out_simplearithmetic.rb
+- test/plugin/test_out_simple_arithmetic.rb

data/test/plugin/test_out_simplearithmetic.rb

@@ -1,60 +0,0 @@
-# -*- coding: utf-8 -*-
-require 'helper'
-
-class SimpleArithmeticOutputTest < Test::Unit::TestCase
-  def setup
-    Fluent::Test.setup
-  end
-
-  CONFIG = %[
-    type simple_arithmetic
-    tag calculated.test
-    undefined_variables nil   # nil, undefined
-    how_to_process_error error_string   # nil, undefined, error_string
-
-    <formulas>
-      x3     x1 * 100 - x2
-      var1   Time.iso8601(t1) - Time.iso8601(t2)
-      var2   x3 - var1
-    </formulas>
-  ]
-
-  def create_driver(conf = CONFIG, tag='test.input')
-    Fluent::Test::OutputTestDriver.new(Fluent::SimpleArithmeticOutput, tag).configure(conf)
-  end
-
-  def test_configure
-    assert_raise(Fluent::ConfigError) {
-      d = create_driver('')
-    }
-    # no variables for calculation
-    assert_raise(Fluent::ConfigError) {
-      d = create_driver %[
-        <formulas>
-
-        </formulas>
-      ]
-    }
-    # Syntax Error
-    assert_raise(Fluent::ConfigError) {
-      d = create_driver %[
-        <formulas>
-          var_undefined
-        </formulas>
-      ]
-    }
-    d = create_driver %[
-        <formulas>
-          var1   var2 * var3
-        </formulas>
-    ]
-  end
-
-  def test_create_formula
-    d = create_driver
-  end
-
-  def test_write
-    d = create_driver
-  end
-end