fluent-plugin-norikra 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 17d22de3fa94371d074467e259195bcf675198da
-  data.tar.gz: cfe8d59434ca08f513000b392e95f39af82d3b4c
+  metadata.gz: 86ac7050ae6300621a0e0a9602f073668f43b2af
+  data.tar.gz: 2ad530592d6771c3a82f29a77147cd99d11b188d
 SHA512:
-  metadata.gz: 2138b0f8de9901ec4c5d99d49a41e8e5aa3d631dada661767306b829543810d5be9f7c8517f4c55cc3c57b4cc9b344ea3fb4a9fe9b374065eef5f21c071bec6a
-  data.tar.gz: c4437d0720878f816954b31b2e1e523240ce941de4cad94fd0fdc59f869bdd7dc79011604679bfa39d799b755160eb457f28eeaae0d1d6c0a8865b216a8c36e2
+  metadata.gz: 40a6c94ac9f9b2ee05fedcc97dd3f1a879eb50c815f7a961d5515275399d5fea21bd8d6fe3899a4f7a027bc01fb8bf64a28857fa3a15411b228f2dfda5904f9c
+  data.tar.gz: 85b3f3e7f757b4e97a3bca27a62b604ec55fc5a16dd2f418a91bc47a635f026e8bbb28054f9f3ef01ad45aaf8057a12c5937fd466be8c9c8391b5150abee9187

data/README.md

@@ -57,9 +57,15 @@ NorikraOutput plugin opens Norikra's target for newly incoming tags. You can spe
   
   target_map_tag    true  # fluentd's tag -> norikra's target
   remove_tag_prefix data
+  
   # other options:
   #   target_map_key KEY_NAME  # use specified key's value as target in fluentd event
   #   target_string  STRING    # use fixed target name specified
+  #   drop_error_record true   # drop records chunk which includes records to occur ClientError on norikra server
+  #                            #   default: true
+  #                            #   (ex: specified (non-optional) fields missing or invalid value for specified type)
+  #   drop_server_error_record true # drop records chunk when any ServerError occurs
+  #                                 #   default: false (to retry)
   
   <default>
     include *     # send all fields values to norikra
@@ -179,6 +185,7 @@ Configuration example to receive tags like `event.foo` and send norikra's target
       group      count_query_group # or default when omitted
 	  expression SELECT count(*) AS cnt FROM ${target}.win:time_batch(1 minute)
 	  tag        count.min.${target}
+      fetch_interval 10s
 	</query>
     <query>
 	  name       count_hour_${target}
@@ -234,7 +241,7 @@ Norikra's target (like table name) can be generated from:
 
 # TODO
 
-* write abou these topics
+* write about these topics
   * error logs for new target, success logs of retry
 
 # Copyright

data/example/test2.conf

@@ -6,6 +6,8 @@
   type norikra_filter
   norikra localhost:26571
   target_map_tag yes
+  drop_server_error_record yes
+  drop_error_record yes
 
   <default>
     include *

data/fluent-plugin-norikra.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-norikra"
-  spec.version       = "0.1.2"
+  spec.version       = "0.2.0"
   spec.authors       = ["TAGOMORI Satoshi"]
   spec.email         = ["tagomoris@gmail.com"]
   spec.description   = %q{process events on fluentd with SQL like query, with built-in Norikra server if needed.}

data/lib/fluent/plugin/in_norikra.rb

@@ -32,6 +32,11 @@ module Fluent
     #   tag_prefix norikra.query
     # </fetch>
 
+    # Define `log` method for v0.10.42 or earlier
+    unless method_defined?(:log)
+      define_method("log") { $log }
+    end
+
     def configure(conf)
       super
 

data/lib/fluent/plugin/norikra/input.rb

@@ -72,7 +72,7 @@ module Fluent::NorikraPlugin
         @fetch_queue.sort!
       end
     rescue => e
-      $log.error "unknown log encountered", :error_class => e.class, :message => e.message
+      log.error "unknown log encountered", :error_class => e.class, :message => e.message
     end
 
     def fetch_worker
@@ -88,7 +88,7 @@ module Fluent::NorikraPlugin
           begin
             data = req.fetch(client())
           rescue => e
-            $log.error "failed to fetch", :norikra => "#{@host}:#{@port}", :method => req.method, :target => req.target, :error => e.class, :message => e.message
+            log.error "failed to fetch", :norikra => "#{@host}:#{@port}", :method => req.method, :target => req.target, :error => e.class, :message => e.message
           end
 
           data.each do |tag, event_array|
@@ -96,7 +96,7 @@ module Fluent::NorikraPlugin
               begin
                 Fluent::Engine.emit(tag, time, event)
               rescue => e
-                $log.error "failed to emit event from norikra query", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message, :tag => tag, :record => event
+                log.error "failed to emit event from norikra query", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message, :tag => tag, :record => event
               end
             end
           end

data/lib/fluent/plugin/norikra/output.rb

@@ -81,9 +81,9 @@ module Fluent::NorikraPlugin
         targets.each do |t|
           next if @target_map[t.name]
 
-          $log.debug "Preparing norikra target #{t.name} on #{@host}:#{@port}"
+          log.debug "Preparing norikra target #{t.name} on #{@host}:#{@port}"
           if prepare_target(c, t)
-            $log.debug "success to prepare target #{t.name} on #{@host}:#{@port}"
+            log.debug "success to prepare target #{t.name} on #{@host}:#{@port}"
 
             if @enable_auto_query
               raise "bug" unless self.respond_to?(:insert_fetch_queue)
@@ -96,7 +96,7 @@ module Fluent::NorikraPlugin
             @target_map[t.name] = t
             @registered_targets.delete(t.name)
           else
-            $log.error "Failed to prepare norikra data for target:#{t.name}"
+            log.error "Failed to prepare norikra data for target:#{t.name}"
             @norikra_started.push(t)
           end
         end
@@ -105,12 +105,12 @@ module Fluent::NorikraPlugin
 
     def prepare_target(client, target)
       # target open and reserve fields
-      $log.debug "Going to prepare about target"
+      log.debug "Going to prepare about target"
       begin
         unless client.targets.include?(target.escaped_name)
-          $log.debug "opening target #{target.escaped_name}"
+          log.debug "opening target #{target.escaped_name}"
           client.open(target.escaped_name, target.reserve_fields, target.auto_field)
-          $log.debug "opening target #{target.escaped_name}, done."
+          log.debug "opening target #{target.escaped_name}, done."
         end
 
         reserving = target.reserve_fields
@@ -119,7 +119,7 @@ module Fluent::NorikraPlugin
           if reserving[field['name']]
             reserved.push(field['name'])
             if reserving[field['name']] != field['type']
-              $log.warn "field type mismatch, reserving:#{reserving[field['name']]} but reserved:#{field['type']}"
+              log.warn "field type mismatch, reserving:#{reserving[field['name']]} but reserved:#{field['type']}"
             end
           end
         end
@@ -128,7 +128,7 @@ module Fluent::NorikraPlugin
           client.reserve(target.escaped_name, fieldname, type) unless reserved.include?(fieldname)
         end
       rescue => e
-        $log.error "failed to prepare target:#{target.escaped_name}", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
+        log.error "failed to prepare target:#{target.escaped_name}", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
         return false
       end
 
@@ -138,7 +138,7 @@ module Fluent::NorikraPlugin
         target.queries.each do |query|
           if registered.has_key?(query.name) # query already registered
             if registered[query.name] != query.expression
-              $log.warn "query name and expression mismatch, check norikra server status. target query name:#{query.name}"
+              log.warn "query name and expression mismatch, check norikra server status. target query name:#{query.name}"
             end
             next
           end
@@ -148,7 +148,7 @@ module Fluent::NorikraPlugin
           fetch_event_registration(query)
         end
       rescue => e
-        $log.warn "failed to register query", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
+        log.warn "failed to register query", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
       end
     end
 
@@ -205,7 +205,15 @@ module Fluent::NorikraPlugin
       c = client()
 
       events_map.each do |target, events|
-        c.send(target, events)
+        begin
+          c.send(target, events)
+        rescue Norikra::RPC::ClientError => e
+          raise unless @drop_error_record
+          log.warn "Norikra server reports ClientError, and dropped", target: target, message: e.message
+        rescue Norikra::RPC::ServerError => e
+          raise unless @drop_server_error_record
+          log.warn "Norikra server reports ServerError, and dropped", target: target, message: e.message
+        end
       end
     end
 

data/lib/fluent/plugin/out_norikra.rb

@@ -21,10 +21,17 @@ module Fluent
     config_param :target_map_tag, :bool, :default => false
     config_param :target_map_key, :string, :default => nil
     config_param :target_string, :string, :default => nil
+    config_param :drop_error_record, :default => true
+    config_param :drop_server_error_record, :default => false
 
     # <default>
     # <target TARGET>
 
+    # Define `log` method for v0.10.42 or earlier
+    unless method_defined?(:log)
+      define_method("log") { $log }
+    end
+
     def configure(conf)
       super
 

data/lib/fluent/plugin/out_norikra_filter.rb

@@ -26,6 +26,8 @@ module Fluent
     config_param :target_map_tag, :bool, :default => false
     config_param :target_map_key, :string, :default => nil
     config_param :target_string, :string, :default => nil
+    config_param :drop_error_record, :default => true
+    config_param :drop_server_error_record, :default => false
 
     # <default>
     # <target TARGET>
@@ -48,6 +50,11 @@ module Fluent
     #   tag_prefix norikra.query
     # </fetch>
 
+    # Define `log` method for v0.10.42 or earlier
+    unless method_defined?(:log)
+      define_method("log") { $log }
+    end
+
     def configure(conf)
       super
 
@@ -129,7 +136,7 @@ module Fluent
     end
 
     def server_starter
-      $log.info "starting Norikra server process #{@host}:#{@port}"
+      log.info "starting Norikra server process #{@host}:#{@port}"
       base_options = [@execute_server_path, 'start', '-H', @host, '-P', @port.to_s]
       cmd,options = if @execute_jruby_path
                       [@execute_jruby_path, [@execute_server_path, 'start', '-H', @host, '-P', @port.to_s]]
@@ -144,24 +151,24 @@ module Fluent
         exec([cmd, 'norikra(fluentd)'], *options)
       end
       connecting = true
-      $log.info "trying to confirm norikra server status..."
+      log.info "trying to confirm norikra server status..."
       while connecting
         begin
-          $log.debug "start to connect norikra server #{@host}:#{@port}"
+          log.debug "start to connect norikra server #{@host}:#{@port}"
           client(:connect_timeout => 1, :send_timeout => 1, :receive_timeout => 1).targets
           # discard result: no exceptions is success
           connecting = false
           next
         rescue HTTPClient::TimeoutError
-          $log.debug "Norikra server test connection timeout. retrying..."
+          log.debug "Norikra server test connection timeout. retrying..."
         rescue Errno::ECONNREFUSED
-          $log.debug "Norikra server test connection refused. retrying..."
+          log.debug "Norikra server test connection refused. retrying..."
         rescue => e
-          $log.error "unknown error in confirming norikra server, #{e.class}:#{e.message}"
+          log.error "unknown error in confirming norikra server, #{e.class}:#{e.message}"
         end
         sleep 3
       end
-      $log.info "confirmed that norikra server #{@host}:#{@port} started."
+      log.info "confirmed that norikra server #{@host}:#{@port} started."
       @norikra_started = true
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-norikra
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-02 00:00:00.000000000 Z
+date: 2014-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: norikra-client