fluent-plugin-norikra 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 438de874a5cf49d1dfeb86c755b5bb7d69ab9715
-  data.tar.gz: 9aff77e6751e8e4757954ce2ae262f263abc136d
+  metadata.gz: e15eebb32dfadbfe46e9c5d52c7c3eae438cb772
+  data.tar.gz: 25e60695315a63fd5e3e774a63e03500bb4b5bef
 SHA512:
-  metadata.gz: 1b19b1cd4f3e2a1b49ff8e6723e5aea17b7e3cd64bd3e96225d0ad55c935cfd34870ff0ca6f7df646ac87fb982dccacd19beb2eea38f823da4cb1a280dfb2dd6
-  data.tar.gz: 9774924e5b9aa84e692776ec249e3cac2363a8a21b5541b1cb9cdd88d7994a0a4e9987a1d1920618f796942ac5139bb4b507286c9c75e8421afc7fbdd6d69f46
+  metadata.gz: ab124b24f446d522b688ace53ea7f1a65ffc095c82e2fef64f44cf78327214908ed9e9fc4e0deecee30aade019973f2482be83bae33eba7b356c6741bb076dff
+  data.tar.gz: f03549e79600346c01af7806274c7d004bbcd374fdfeea589b0f955f2da1efccc6e348c46b0169aa1997291708a23bc000e4c538d6c3a75c314e6d62cb93f603

data/fluent-plugin-norikra.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-norikra"
-  spec.version       = "0.0.3"
+  spec.version       = "0.0.4"
   spec.authors       = ["TAGOMORI Satoshi"]
   spec.email         = ["tagomoris@gmail.com"]
   spec.description   = %q{process events on fluentd with SQL like query, with built-in Norikra server if needed.}

data/lib/fluent/plugin/norikra_target.rb

@@ -31,17 +31,17 @@ class Fluent::NorikraOutput
                         end
     end
 
-    def generate(target)
+    def generate(name, escaped)
       Fluent::NorikraOutput::Query.new(
-        self.class.replace_target(target, @name_template),
-        self.class.replace_target(target, @expression_template),
-        self.class.replace_target(target, @tag_template),
+        self.class.replace_target(name, @name_template),
+        self.class.replace_target(escaped, @expression_template),
+        self.class.replace_target(name, @tag_template),
         @fetch_interval
       )
     end
 
-    def self.replace_target(target, str)
-      str.gsub('${target}', target)
+    def self.replace_target(t, str)
+      str.gsub('${target}', t)
     end
 
     def self.parse_time_period(string)
@@ -192,12 +192,32 @@ class Fluent::NorikraOutput
 
   class Target
     attr_accessor :name, :fields, :queries
+    attr_reader :escaped_name
+
+    def self.escape(src)
+      if src.nil? || src.empty?
+        return 'FluentdGenerated'
+      end
+
+      dst = src.gsub(/[^_a-zA-Z0-9]/, '_')
+      unless dst =~ /^[a-zA-Z]([_a-zA-Z0-9]*[a-zA-Z0-9])?$/
+        unless dst =~ /^[a-zA-Z]/
+          dst = 'Fluentd' + dst
+        end
+        unless dst =~ /[a-zA-Z0-9]$/
+          dst = dst + 'Generated'
+        end
+      end
+      dst
+    end
 
     def initialize(target, config)
       @name = target
+      @escaped_name = self.class.escape(@name)
+
       @filter = RecordFilter.new(*([:include, :include_regexp, :exclude, :exclude_regexp].map{|s| config.filter_params[s]}))
       @fields = config.field_definitions
-      @queries = config.query_generators.map{|g| g.generate(target)}
+      @queries = config.query_generators.map{|g| g.generate(@name, @escaped_name)}
     end
 
     def filter(record)

data/lib/fluent/plugin/out_norikra.rb

@@ -157,14 +157,17 @@ module Fluent
 
       @register_thread.join
       @fetch_thread.join
-      begin
-        counter = 0
-        while !Process.waitpid(@norikra_pid, Process::WNOHANG)
-          sleep 1
-          break if counter > 3
+
+      if @execute_server
+        begin
+          counter = 0
+          while !Process.waitpid(@norikra_pid, Process::WNOHANG)
+            sleep 1
+            break if counter > 3
+          end
+        rescue Errno::ECHILD
+          # norikra server process exited.
         end
-      rescue Errno::ECHILD
-        # norikra server process exited.
       end
     end
 
@@ -268,14 +271,14 @@ module Fluent
 
         event = t.filter(record)
 
-        out << [target,event].to_msgpack
+        out << [t.escaped_name,event].to_msgpack
       end
 
       out
     end
 
     def prepared?(target_names)
-      @norikra_started && target_names.reduce(true){|r,t| r && @target_map[t]}
+      @norikra_started && target_names.reduce(true){|r,t| r && @target_map.values.any?{|target| target.escaped_name == t}}
     end
 
     def write(chunk)
@@ -300,15 +303,15 @@ module Fluent
       # target open and reserve fields
       $log.debug "Going to prepare about target"
       begin
-        unless client.targets.include?(target.name)
-          $log.debug "opening target #{target.name}"
-          client.open(target.name, target.reserve_fields)
-          $log.debug "opening target #{target.name}, done."
+        unless client.targets.include?(target.escaped_name)
+          $log.debug "opening target #{target.escaped_name}"
+          client.open(target.escaped_name, target.reserve_fields)
+          $log.debug "opening target #{target.escaped_name}, done."
         end
 
         reserving = target.reserve_fields
         reserved = []
-        client.fields(target.name).each do |field|
+        client.fields(target.escaped_name).each do |field|
           if reserving[field['name']]
             reserved.push(field['name'])
             if reserving[field['name']] != field['type']
@@ -318,10 +321,10 @@ module Fluent
         end
 
         reserving.each do |fieldname,type|
-          client.reserve(target, fieldname, type) unless reserved.include?(fieldname)
+          client.reserve(target.escaped_name, fieldname, type) unless reserved.include?(fieldname)
         end
       rescue => e
-        $log.error "failed to prepare target:#{target.name}", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
+        $log.error "failed to prepare target:#{target.escaped_name}", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
         return false
       end
 

data/test/plugin/test_out_norikra.rb

@@ -0,0 +1,11 @@
+require 'helper'
+
+class NorikraOutputTest < Test::Unit::TestCase
+  CONF = %[
+    target_map_tag yes
+  ]
+
+  def create_driver(conf=CONFIG, tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::NorikraOutput, tag).configure(conf)
+  end
+end

data/test/test_query_generator.rb

@@ -36,7 +36,7 @@ class QueryGeneratorTest < Test::Unit::TestCase
 
   def test_generate
     g = @this.new('query_${target}', 'SELECT * FROM ${target}.win:time_batch( 10 min ) WHERE x=1', 'tag.${target}')
-    q = g.generate('test')
+    q = g.generate('test', 'test')
     assert_equal 'query_test', q.name
     assert_equal 'SELECT * FROM test.win:time_batch( 10 min ) WHERE x=1', q.expression
     assert_equal 'tag.test', q.tag

data/test/test_target.rb

@@ -6,42 +6,54 @@ class TargetTest < Test::Unit::TestCase
     @this = Fluent::NorikraOutput::Target
   end
 
-  def test_instanciate
-        q1 = Fluent::Config::Element.new('query', nil, {
-        'name' => 'q1_${target}',
-        'expression' => 'SELECT * FROM ${target}.win:time_batch(10 min) WHERE q1',
-        'tag' => 'q1.${target}'
-      }, [])
-    q2 = Fluent::Config::Element.new('query', nil, {
-        'name' => 'q2_${target}',
-        'expression' => 'SELECT * FROM ${target}.win:time_batch(50 min) WHERE q2.length() > 0',
-        'tag' => 'q2.${target}'
-      }, [])
-    c1 = Fluent::Config::Element.new('default', nil, {
-        'include' => '*',
-        'exclude' => 'flag',
-        'exclude_regexp' => 'f_.*',
-        'field_string' => 's1,s2,s3',
-        'field_boolean' => 'bool1,bool2',
-        'field_int' => 'i1,i2,i3,i4',
-        'field_long' => 'num1,num2',
-        'field_float' => 'f1,f2',
-        'field_double' => 'd'
-      }, [q1,q2])
-    s1 = Fluent::NorikraOutput::ConfigSection.new(c1)
+  def test_target_name_escape
+    assert_equal 'target1', @this.escape('target1')
+    assert_equal 'target1_subtarget1', @this.escape('target1.subtarget1')
+    assert_equal 'test_tag_foo', @this.escape('test.tag.foo')
+
+    assert_equal 'FluentdGenerated', @this.escape('')
+    assert_equal 'Fluentd_Generated', @this.escape(':')
+    assert_equal 'a', @this.escape('a')
+    assert_equal 'Fluentd_a', @this.escape('_a')
+    assert_equal 'a_Generated', @this.escape('a_')
+  end
 
-    q3 = Fluent::Config::Element.new('query', nil, {
-        'name' => 'q3_test',
-        'expression' => 'SELECT * FROM ${target}.win:time_batch(30 min) WHERE q3="/"',
-        'tag' => 'q3.test'
-      }, [])
-    c2 = Fluent::Config::Element.new('target', 'test', {
-        'exclude_regexp' => '(f|g)_.*',
-        'field_double' => 'd1,d2,d3,d4'
-      }, [q3])
-    s2 = Fluent::NorikraOutput::ConfigSection.new(c2)
+  Q1 = Fluent::Config::Element.new('query', nil, {
+      'name' => 'q1_${target}',
+      'expression' => 'SELECT * FROM ${target}.win:time_batch(10 min) WHERE q1',
+      'tag' => 'q1.${target}'
+    }, [])
+  Q2 = Fluent::Config::Element.new('query', nil, {
+      'name' => 'q2_${target}',
+      'expression' => 'SELECT * FROM ${target}.win:time_batch(50 min) WHERE q2.length() > 0',
+      'tag' => 'q2.${target}'
+    }, [])
+  C1 = Fluent::Config::Element.new('default', nil, {
+      'include' => '*',
+      'exclude' => 'flag',
+      'exclude_regexp' => 'f_.*',
+      'field_string' => 's1,s2,s3',
+      'field_boolean' => 'bool1,bool2',
+      'field_int' => 'i1,i2,i3,i4',
+      'field_long' => 'num1,num2',
+      'field_float' => 'f1,f2',
+      'field_double' => 'd'
+    }, [Q1,Q2])
+  S1 = Fluent::NorikraOutput::ConfigSection.new(C1)
 
-    t = @this.new('test', s1 + s2)
+  Q3 = Fluent::Config::Element.new('query', nil, {
+      'name' => 'q3_test',
+      'expression' => 'SELECT * FROM ${target}.win:time_batch(30 min) WHERE q3="/"',
+      'tag' => 'q3.test'
+    }, [])
+  C2 = Fluent::Config::Element.new('target', 'test', {
+      'exclude_regexp' => '(f|g)_.*',
+      'field_double' => 'd1,d2,d3,d4'
+    }, [Q3])
+  S2 = Fluent::NorikraOutput::ConfigSection.new(C2)
+
+  def test_instanciate
+    t = @this.new('test', S1 + S2)
 
     assert_equal 'test', t.name
     assert_equal({
@@ -63,4 +75,22 @@ class TargetTest < Test::Unit::TestCase
         'd' => 'double', 'd1' => 'double', 'd2' => 'double', 'd3' => 'double', 'd4' => 'double'
       }, t.reserve_fields)
   end
+
+  def test_queries
+    t = @this.new('test.service', S1 + S2)
+
+    assert_equal 3, t.queries.size
+
+    assert_equal 'q1_test.service', t.queries[0].name
+    assert_equal 'SELECT * FROM test_service.win:time_batch(10 min) WHERE q1', t.queries[0].expression
+    assert_equal 'q1.test.service', t.queries[0].tag
+
+    assert_equal 'q2_test.service', t.queries[1].name
+    assert_equal 'SELECT * FROM test_service.win:time_batch(50 min) WHERE q2.length() > 0', t.queries[1].expression
+    assert_equal 'q2.test.service', t.queries[1].tag
+
+    assert_equal 'q3_test', t.queries[2].name
+    assert_equal 'SELECT * FROM test_service.win:time_batch(30 min) WHERE q3="/"', t.queries[2].expression
+    assert_equal 'q3.test', t.queries[2].tag
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-norikra
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-01 00:00:00.000000000 Z
+date: 2013-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: norikra-client
@@ -86,6 +86,7 @@ files:
 - lib/fluent/plugin/norikra_target.rb
 - lib/fluent/plugin/out_norikra.rb
 - test/helper.rb
+- test/plugin/test_out_norikra.rb
 - test/test_config_section.rb
 - test/test_query.rb
 - test/test_query_generator.rb
@@ -111,14 +112,16 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.2
+rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to do CEP with norikra
 test_files:
 - test/helper.rb
+- test/plugin/test_out_norikra.rb
 - test/test_config_section.rb
 - test/test_query.rb
 - test/test_query_generator.rb
 - test/test_record_filter.rb
 - test/test_target.rb
+has_rdoc: