fluent-plugin-norikra 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: aa64957f7aa640bcb15c8bf4ac747e01617ecac0
+  data.tar.gz: cc7f497f0fda85c0dd5b45fb70d5470d631d4317
+SHA512:
+  metadata.gz: 0e9ecc0d5c4eda53f6426e82991dcea65c9152f03b1a7846ad202428a315b1dcc91502460f4b405e6a57bd543457d177b6b24c87441846302e118f31e67fa422
+  data.tar.gz: c81c72fba462086810c65f45dd2d29fd6fbb042d78185052ab0fd92fbec2b706f0a7f217c4b6aacd7f8e85ce4b2a1be1d19353c903b9ccb2ded4417de9eff85f

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-norikra.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2013- TAGOMORI Satoshi
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,179 @@
+# fluent-plugin-norikra
+
+Fluentd output plugin to send events to norikra server, and to fetch events (and re-send on fluentd network) from norikra server.
+
+With NorikraOutput, we can:
+
+ * execute Norikra server as built-in process dynamically
+ * generate Norikra's target automatically with Fluentd's tags
+ * register queries automatically with Fluentd's tags and messages
+ * get all events on Norikra and emit on Fluentd network automatically
+
+# Setup
+
+At first, install JRuby and Norikra on your host if you are not using stand-alone Norikra servers.
+
+1. install latest jruby
+  * (rbenv) `rbenv install jruby-1.7.4`
+  * (rvm) `rvm install jruby-1.7.4`
+  * or other tools you want.
+2. swith to jruby, and install Norikra
+  * `gem install norikra`
+3. check norikra path
+  * `which norikra`
+4. switch CRuby (with Fluentd), and install this plugin
+  * `gem install fluent-plugin-norikra` (or use `fluent-gem`)
+5. configure Fluentd, and execute.
+  * and write `path` configuration of `<server>` section (if you want)
+
+# Configuration
+
+For variations, see `example` directory.
+
+## NorikraOutput
+
+With built-in Norikra server, to receive tags like `event.foo` and send norikra's target `foo`, and get count of its records per minute, and per hour.
+
+    <match event.*>
+      type norikra
+      norikra localhost:26571 # this is default
+      <server>
+        execute yes
+        path    /home/user/.rbenv/versions/jruby-1.7.4/bin/norikra
+      </server>
+      
+      remove_tag_prefix event
+      target_map_tag    yes
+      
+      <default>
+	    <query>
+		  name       count_min_${target}
+		  expression SELECT count(*) AS cnt FROM ${target}.win:time_batch(1 minute)
+		  tag        count.min.${target}
+		</query>
+	    <query>
+		  name       count_hour_${target}
+		  expression SELECT count(*) AS cnt FROM ${target}.win:time_batch(1 hour)
+		  tag        count.hour.${target}
+		</query>
+      </default>
+    </match>
+
+With default setting, all fields are defined as 'string', so you must use `cast` for numerical processing in query (For more details, see Norikra and Esper's documents).
+
+If you know some field's types of records, you can define types of these fields. This plugin will define field types before it send records into Norikra server.
+
+    <match event.*>
+      type norikra
+      norikra localhost:26571 # this is default
+      <server>
+        execute yes
+        path    /home/user/.rbenv/versions/jruby-1.7.4/bin/norikra
+      </server>
+      
+      remove_tag_prefix event
+      target_map_tag    yes
+      
+      <default>
+        field_int    amount
+        field_long   size
+        field_double price
+        
+	    <query>
+		  name       sales_${target}
+		  expression SELECT price * amount AS  FROM ${target}.win:time_batch(1 minute) WHERE size > 0
+		  tag        sales.min.${target}
+		</query>
+      </default>
+    </match>
+
+Additional field definitions and query registrations should be written in `<target TARGET_NAME>` sections.
+
+    <default>
+      ... # for all of access logs
+    </default>
+    <target login>
+      field_string protocol # like 'oauth', 'openid', ...
+	  field_int    proto_num # integer means internal id of protocols
+	  <query>
+	    name       protocol
+		expression SELECT protocol, count(*) AS cnt FROM ${target}.win:time_batch(1 hour) WHERE proto_num != 0 GROUP BY protocol
+		tag        login.counts
+	  </query>
+    </target>
+    <target other_action>
+	  ...
+    </target>
+	# ...
+
+### Input event data filtering
+
+If you want send known fields only, specify `exclude *` and `include` or `include_regexp` like this:
+
+    <default>
+      exclude *
+      include         path,status,method,bytes,rhost,referer,agent,duration
+      include_pattern ^(query_|header_).*
+      
+      # ...
+    </default>
+
+Or you can specify to include as default, and exclude known some fields:
+
+    <default>
+      include *
+      exclude         user_secret
+      include_pattern ^(header_).*
+      
+      # ...
+    </default>
+
+NOTE: These configurations of `<target>` section overwrites of configurations in `<default>` section.
+
+### Target mapping
+
+Norikra's target (like table name) can be generated from:
+
+ * tag
+   * one target per one tag
+   * `target_map_tag yes`
+ * value of specified field
+   * targets from values in specified field of record, dynamically
+   * `target_map_key foo`
+ * fixed string (in configuration file)
+   * all records are sent in single target
+   * `target_string from_fluentd`
+
+### Event sweeping
+
+Norikra server accepts queries and events from everywhere other than Fluentd. This plugin can get events from these queries/events.
+
+To gather all events of Norikra server, including queries from outside of Fluentd configurations, write `<event>` section.
+
+    <events>
+      method sweep
+      tag    query_name
+      # tag    field FIELDNAME
+      # tag    string FIXED_STRING
+      tag_prefix norikra.event     # actual tag: norikra.event.QUERYNAME
+      sweep_interval 5s
+    </events>
+
+NOTE: 'sweep' get all events from Norikra, and other clients cannot get these events. Take care for other clients.
+
+# FAQ
+
+* TODO: write this section
+  * `fetch_interval`
+  * error logs for new target, success logs of retry
+
+# TODO
+
+* TODO: write this section
+
+# Copyright
+
+* Copyright (c) 2013- TAGOMORI Satoshi (tagomoris)
+* License
+  * Apache License, version 2.0
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/example/blank.conf

@@ -0,0 +1,15 @@
+<source>
+  type forward
+</source>
+
+<match event.*>
+  type norikra
+  norikra localhost:26571 # this is default
+  <server>
+    execute yes
+    path    /Users/tagomoris/.rbenv/versions/jruby-1.7.3/bin/norikra
+  </server>
+  
+  remove_tag_prefix event
+  target_map_tag    yes
+</match>

data/example/example1.conf

@@ -0,0 +1,68 @@
+<match event.*>
+  type norikra
+  norikra localhost:26571
+
+  <server>
+    execute yes # (default)no
+    path    /home/user/.rbenv/versions/jruby-1.7.4/bin/norikra
+  </server>
+
+  remove_tag_prefix event
+
+  target_map_tag yes
+  # or
+  # target_map_key KEYNAME
+  # or
+  # target_string TARGET_STRING
+
+  <default>
+    include *
+    exclude yyyymmdd,hhmmss
+    exclude_regexp f_.*
+    # OR
+    # exclude *
+    # include foo,bar,baz
+    # include_regexp status.*
+    field_boolean flag
+    field_int status
+    field_long    duration,bytes
+
+    <query>
+      name pv_${target}
+      expression SELECT count(*) AS cnt FROM ${target}.win:time_batch(1 minutes) WHERE not flag
+      tag pv.${target}
+      fetch_interval 15s # default -> time_batch / 4 ? -> (none) -> 60s
+      # fetch_interval is ignored when <events> section specified
+    </query>
+    <query>
+      name errors_${target}
+      expression SELECT count(*) AS cnt FROM ${target}.win:time_batch(1 minutes) WHERE status >= 500
+      tag errors.${target}
+      fetch_interval 15s
+    </query>
+  </default>
+
+  <target search>
+    field_int display
+
+    <query>
+      name search_words
+      expression SELECT count(distinct query_search) AS cnt FROM ${target}.win:time_batch(1 minutes) WHERE query_search.length() > 0
+      tag search.words
+    </query>
+    <query>
+      name search_rate
+      expression SELECT count(*) AS cnt FROM ${target}.win:time_batch(1 minutes) WHERE query_search.length() > 0
+      tag search.rate
+    </query>
+  </target>
+
+  <events>
+    method sweep # listen(not implemented)
+    tag query_name
+    # tag field FIELDNAME
+    # tag string TAG_STRING
+    tag_prefix cep
+    sweep_interval 5s
+  </events>
+</match>

data/example/test1.conf

@@ -0,0 +1,36 @@
+<source>
+  type forward
+</source>
+
+<match test.*>
+  type norikra
+  norikra localhost:26571
+  <server>
+    execute yes
+    path /Users/tagomoris/.rbenv/versions/jruby-1.7.3/bin/norikra
+  </server>
+
+  remove_tag_prefix test
+  target_map_tag yes
+
+  <default>
+    <query>
+      name count_${target}
+      expression SELECT '${target}' as target,count(*) AS cnt FROM ${target}.win:time_batch(30 sec)
+    </query>
+  </default>
+  <event>
+    method sweep
+    tag field target
+    tag_prefix count
+    sweep_interval 5s
+  </event>
+</match>
+
+<match fluent.*>
+  type null
+</match>
+
+<match **>
+  type stdout
+</match>

data/fluent-plugin-norikra.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-norikra"
+  spec.version       = "0.0.1"
+  spec.authors       = ["TAGOMORI Satoshi"]
+  spec.email         = ["tagomoris@gmail.com"]
+  spec.description   = %q{process events on fluentd with SQL like query, with built-in Norikra server if needed.}
+  spec.summary       = %q{Fluentd plugin to do CEP with norikra}
+  spec.homepage      = "https://github.com/tagomoris/fluent-plugin-norikra"
+  spec.license       = "APLv2"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "norikra-client", ">= 0.0.2"
+  spec.add_runtime_dependency "fluentd"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/lib/fluent/plugin/norikra_target.rb

@@ -0,0 +1,217 @@
+class Fluent::NorikraOutput
+  class Query
+    attr_accessor :name, :expression, :tag, :interval
+
+    def initialize(name, expression, tag, interval)
+      @name = name
+      @expression = expression
+      @tag = tag
+      @interval = interval
+    end
+  end
+
+  class QueryGenerator
+    attr_reader :fetch_interval
+
+    def initialize(name_template, expression_template, tag_template, opts={})
+      @name_template = name_template || ''
+      @expression_template = expression_template || ''
+      @tag_template = tag_template || ''
+      if @name_template.empty? || @expression_template.empty?
+        raise Fluent::ConfigError, "query's name/expression must be specified"
+      end
+      @fetch_interval = case
+                        when opts['fetch_interval']
+                          Fluent::Config.time_value(opts['fetch_interval'])
+                        when @expression_template =~ /\.win:time_batch\(([^\)]+)\)/
+                          y,mon,w,d,h,m,s,msec = self.class.parse_time_period($1)
+                          (h * 3600 + m * 60 + s) / 5
+                        else
+                          60
+                        end
+    end
+
+    def generate(target)
+      Fluent::NorikraOutput::Query.new(
+        self.class.replace_target(target, @name_template),
+        self.class.replace_target(target, @expression_template),
+        self.class.replace_target(target, @tag_template),
+        @fetch_interval
+      )
+    end
+
+    def self.replace_target(target, str)
+      str.gsub('${target}', target)
+    end
+
+    def self.parse_time_period(string)
+      #### http://esper.codehaus.org/esper-4.9.0/doc/reference/en-US/html/epl_clauses.html#epl-syntax-time-periods
+      # time-period : [year-part] [month-part] [week-part] [day-part] [hour-part] [minute-part] [seconds-part] [milliseconds-part]
+      # year-part : (number|variable_name) ("years" | "year")
+      # month-part : (number|variable_name) ("months" | "month")
+      # week-part : (number|variable_name) ("weeks" | "week")
+      # day-part : (number|variable_name) ("days" | "day")
+      # hour-part : (number|variable_name) ("hours" | "hour")
+      # minute-part : (number|variable_name) ("minutes" | "minute" | "min")
+      # seconds-part : (number|variable_name) ("seconds" | "second" | "sec")
+      # milliseconds-part : (number|variable_name) ("milliseconds" | "millisecond" | "msec")
+      m = /^\s*(\d+ years?)? ?(\d+ months?)? ?(\d+ weeks?)? ?(\d+ days?)? ?(\d+ hours?)? ?(\d+ (?:min|minute|minutes))? ?(\d+ (?:sec|second|seconds))? ?(\d+ (?:msec|millisecond|milliseconds))?/.match(string)
+      years = (m[1] || '').split(' ',2).first.to_i
+      months = (m[2] || '').split(' ',2).first.to_i
+      weeks = (m[3] || '').split(' ',2).first.to_i
+      days = (m[4] || '').split(' ',2).first.to_i
+      hours = (m[5] || '').split(' ',2).first.to_i
+      minutes = (m[6] || '').split(' ',2).first.to_i
+      seconds = (m[7] || '').split(' ',2).first.to_i
+      msecs = (m[8] || '').split(' ',2).first.to_i
+      return [years, months, weeks, days, hours, minutes, seconds, msecs]
+    end
+  end
+
+  class RecordFilter
+    attr_reader :default_policy, :include_fields, :include_regexp, :exclude_fields, :exclude_regexp
+
+    def initialize(include='', include_regexp='', exclude='', exclude_regexp='')
+      include ||= ''
+      include_regexp ||= ''
+      exclude ||= ''
+      exclude_regexp ||= ''
+
+      @default_policy = nil
+      if include == '*' && exclude == '*'
+        raise Fluent::ConfigError, "invalid configuration, both of 'include' and 'exclude' are '*'"
+      end
+      if include.empty? && include_regexp.empty? && exclude.empty? && exclude_regexp.empty? # assuming "include *"
+        @default_policy = :include
+      elsif exclude.empty? && exclude_regexp.empty? || exclude == '*' # assuming "exclude *"
+        @default_policy = :exclude
+      elsif include.empty? && include_regexp.empty? || include == '*' # assuming "include *"
+        @default_policy = :include
+      else
+        raise Fluent::ConfigError, "unknown default policy. specify 'include *' or 'exclude *'"
+      end
+
+      @include_fields = nil
+      @include_regexp = nil
+      @exclude_fields = nil
+      @exclude_regexp = nil
+
+      if @default_policy == :exclude
+        @include_fields = include.split(',')
+        @include_regexp = Regexp.new(include_regexp) unless include_regexp.empty?
+        if @include_fields.empty? && @include_regexp.nil?
+          raise Fluent::ConfigError, "no one fields specified. specify 'include' or 'include_regexp'"
+        end
+      else
+        @exclude_fields = exclude.split(',')
+        @exclude_regexp = Regexp.new(exclude_regexp) unless exclude_regexp.empty?
+      end
+    end
+
+    def filter(record)
+      if @default_policy == :include
+        if @exclude_fields.empty? && @exclude_regexp.nil?
+          record
+        else
+          record = record.dup
+          record.keys.each do |f|
+            record.delete(f) if @exclude_fields.include?(f) || @exclude_regexp &&  @exclude_regexp.match(f)
+          end
+          record
+        end
+      else # default policy exclude
+        data = {}
+        record.keys.each do |f|
+          data[f] = record[f] if @include_fields.include?(f) || @include_regexp && @include_regexp.match(f)
+        end
+        data
+      end
+    end
+  end
+
+  class ConfigSection
+    attr_accessor :target, :filter_params, :field_definitions, :query_generators
+
+    def initialize(section)
+      @target = case section.name
+                when 'default'
+                  nil
+                when 'target'
+                  section.arg
+                else
+                  raise ArgumentError, "invalid section for this class, #{section.name}: ConfigSection"
+                end
+      @filter_params = {
+        :include => section['include'],
+        :include_regexp => section['include_regexp'],
+        :exclude => section['exclude'],
+        :exclude_regexp => section['exclude_regexp']
+      }
+      @field_definitions = {
+        :string => (section['field_string'] || '').split(','),
+        :boolean => (section['field_boolean'] || '').split(','),
+        :int => (section['field_int'] || '').split(','),
+        :long => (section['field_long'] || '').split(','),
+        :float => (section['field_float'] || '').split(','),
+        :double => (section['field_double'] || '').split(',')
+      }
+      @query_generators = []
+      section.elements.each do |element|
+        if element.name == 'query'
+          opt = {}
+          if element.has_key?('fetch_interval')
+            opt['fetch_interval'] = element['fetch_interval'].to_i
+          end
+          @query_generators.push(QueryGenerator.new(element['name'], element['expression'], element['tag'], opt))
+        end
+      end
+    end
+
+    def +(other)
+      if other.nil?
+        other = self.class.new(Fluent::Config::Element.new('target', 'dummy', {}, []))
+      end
+      r = self.class.new(Fluent::Config::Element.new('target', (other.target ? other.target : self.target), {}, []))
+      others_filter = {}
+      other.filter_params.keys.each do |k|
+        others_filter[k] = other.filter_params[k] if other.filter_params[k]
+      end
+      r.filter_params = self.filter_params.merge(others_filter)
+      r.field_definitions = {
+        :string => self.field_definitions[:string] + other.field_definitions[:string],
+        :boolean => self.field_definitions[:boolean] + other.field_definitions[:boolean],
+        :int => self.field_definitions[:int] + other.field_definitions[:int],
+        :long => self.field_definitions[:long] + other.field_definitions[:long],
+        :float => self.field_definitions[:float] + other.field_definitions[:float],
+        :double => self.field_definitions[:double] + other.field_definitions[:double]
+      }
+      r.query_generators = self.query_generators + other.query_generators
+      r
+    end
+  end
+
+  class Target
+    attr_accessor :name, :fields, :queries
+
+    def initialize(target, config)
+      @name = target
+      @filter = RecordFilter.new(*([:include, :include_regexp, :exclude, :exclude_regexp].map{|s| config.filter_params[s]}))
+      @fields = config.field_definitions
+      @queries = config.query_generators.map{|g| g.generate(target)}
+    end
+
+    def filter(record)
+      @filter.filter(record)
+    end
+
+    def reserve_fields
+      f = {}
+      @fields.keys.each do |type_sym|
+        @fields[type_sym].each do |fieldname|
+          f[fieldname] = type_sym.to_s
+        end
+      end
+      f
+    end
+  end
+end