fluent-plugin-filter_where 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 14fee05a0ddb942780d1f9b4722d99549206183f
+  data.tar.gz: 040111b31bb0b6b9f56f85efc29ddbe9153b25e0
+SHA512:
+  metadata.gz: 30106ab7026afa2f66d0248309f51cae62a22d1c16fec769a2db98d46ebdb69315dd8d170a3ecaa22bf9eed3946fdd8c64baecb360b6ca7a720a553c5e026dd8
+  data.tar.gz: 911625bfdcf3351b780957639f70db66ab32ca273e5ca0cc0e014ec8d33f76a6223e0150e032cc1f2bb84b4bca3905eba3fe58f71a09f401fd2d77ca7a00ea20

data/.gitignore

@@ -0,0 +1,14 @@
+/*.gem
+~*
+#*
+*~
+.bundle
+Gemfile.lock
+.rbenv-version
+vendor
+doc/*
+tmp/*
+coverage
+.yardoc
+pkg/
+.ruby-version

data/.travis.yml

@@ -0,0 +1,9 @@
+rvm:
+  - 2.1.*
+  - 2.2.*
+  - 2.3.0
+  - 2.4.0
+gemfile:
+  - Gemfile
+  - Gemfile.fluentd.0.12
+before_install: gem update bundler

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 1.0.0
+
+First release

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/Gemfile.fluentd.0.12

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem 'fluentd', '~> 0.12.0'
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2017 Naotoshi Seo
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,141 @@
+# fluent-plugin-filter_where
+
+[![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-filter_where.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-filter_where)
+
+Fluentd plugin to filter records with SQL-like WHERE statements
+
+## Requirements
+
+See [.travis.yml](.travis.yml)
+
+`fluent-plugin-filter_where` supports both v0.14 API and v0.12 API in one gem.
+
+## Installation
+
+Use RubyGems:
+
+    gem install fluent-plugin-filter_where
+
+## Configuration
+
+- where
+
+    The SQL-like WHERE statements. See [SQL-like Syntax](#sql-like-syntax) for more details.
+
+### Example
+
+```apache
+<filter foo.**>
+  @type where
+  where string_key = 'string' OR number_key >= 0.1
+</filter>
+```
+
+## SQL-like Syntax
+
+Example:
+
+```sql
+where (string_key START_WITH 'str' AND number_key > 1.0) OR ("true_key" = true AND string_key REGEXP '^reg')
+```
+
+## Literals
+
+### Boolean Literal
+
+`true` or `TRUE` or `false` or `FALSE` are considered as a boolean literal
+
+### Number Literal
+
+Characters matching with a regular expression `-?[0-9]+(\.[0-9]+)?` is considered as a number literal
+
+### String Literal
+
+Characters surrounded by `'` such as `'foo'` is considered as a string literal
+
+### Json Literal
+
+Not supported yet
+
+### Identifier Literal
+
+Characters matching with a regular expression `[a-zA-Z_][a-zA-z0-9_]*` such as `foobar`, and characters surrounded by `"` such as `"foo-bar"`, `"foo.bar"`, and `"foo\"bar"` are considred as an identifier literal, that is, embulk's column name.
+
+## Operators
+
+### Boolean Operator
+
+* `=`
+* `!=`
+
+### Number Operator (Long and Double)
+
+* `=`
+* `!=`
+* `>`
+* `>=`
+* `<=`
+* `<`
+
+### String Operator
+
+* `=`
+* `!=`
+* `START_WITH`
+* `END_WITH`
+* `INCLUDE`
+* `REGEXP`
+
+### Json Operator
+
+Not supported yet
+
+### unary operator
+
+* "xxx IS NULL"
+* "xxx IS NOT NULL"
+* "NOT xxx"
+
+## ChangeLog
+
+See [CHANGELOG.md](CHANGELOG.md) for details.
+
+## Development
+
+Run test:
+
+```
+$ bundle exec rake test
+```
+
+Release:
+
+Modify gemspec and CHANGELOG.md, then
+
+```
+$ bundle exec rake release
+```
+
+## Development of SQL-like Syntax
+
+This plugin uses [rexical](https://github.com/tenderlove/rexical) for lexical scanner generator, and [racc](https://github.com/tenderlove/racc) for parser generator.
+
+If you modify `praser.rex` or `praser.racc`, you must compile them as:
+
+```
+$ bundle exec rake compile
+```
+
+The `test` task runs the `compile` task before running.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new [Pull Request](../../pull/new/master)
+
+## Copyright
+
+Copyright (c) 2017 - Naotoshi Seo. See [LICENSE](LICENSE) for details.

data/Rakefile

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+desc 'Run test_unit based test'
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.test_files = Dir["test/**/test_*.rb"].sort
+  t.verbose = true
+  #t.warning = true
+end
+task :default => :test
+task :test => [:compile]
+
+desc 'Open an irb session preloaded with the gem library'
+task :console do
+  sh 'irb -rubygems -I lib'
+end
+task :c => :console
+
+task :compile do
+  sh 'racc lib/fluent/plugin/filter_where/parser.racc'
+  sh 'rex lib/fluent/plugin/filter_where/parser.rex'
+end

data/example.conf

@@ -0,0 +1,14 @@
+<source>
+  type dummy
+  tag dummy
+  dummy {"message":"foo","time":1432732710,"members":["Alice"]}
+</source>
+
+<filter dummy>
+  @type where
+  where message = 'foo' AND time > 10
+</filter>
+
+<match dummy>
+  type stdout
+</match>

data/fluent-plugin-filter_where.gemspec

@@ -0,0 +1,30 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-filter_where"
+  gem.version     = "1.0.0"
+  gem.authors     = ["Naotoshi Seo"]
+  gem.email       = "sonots@gmail.com"
+  gem.homepage    = "https://github.com/sonots/fluent-plugin-filter_where"
+  gem.description = "Fluentd plugin to filter records with SQL-like WHERE statements"
+  gem.summary     = gem.description
+  gem.licenses    = ["MIT"]
+  gem.has_rdoc    = false
+
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd"
+
+  gem.add_development_dependency "racc"
+  gem.add_development_dependency "rexical"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "pry"
+  gem.add_development_dependency "pry-nav"
+  gem.add_development_dependency "test-unit"
+  gem.add_development_dependency "test-unit-rr"
+  gem.add_development_dependency "timecop"
+end

data/lib/fluent/plugin/filter_where.rb

@@ -0,0 +1,7 @@
+require 'fluent/version'
+major, minor, patch = Fluent::VERSION.split('.').map(&:to_i)
+if major > 0 || (major == 0 && minor >= 14)
+  require_relative 'filter_where/v14'
+else
+  require_relative 'filter_where/v12'
+end

data/lib/fluent/plugin/filter_where/core.rb

@@ -0,0 +1,32 @@
+require 'fluent/plugin/filter_where/parser.tab'
+
+module Fluent; module FilterWhere; end; end
+module Fluent
+  module FilterWhere::Core
+    def initialize
+      super
+    end
+    
+    def self.included(klass)
+      klass.config_param :where, :string, :desc => 'The SQL-like WHERE statement.'
+    end
+
+    def configure(conf)
+      super
+
+      parser = Fluent::FilterWhere::Parser.new
+      @scanner = parser.scan(@where)
+    end
+
+    def filter(tag, time, record)
+      if @scanner.eval(record)
+        record
+      else
+        nil # remove
+      end
+    rescue => e
+      log.warn "filter_where: #{e.class} #{e.message} #{e.backtrace.first}"
+      log.debug "filter_where:: tag:#{tag} time:#{time} record:#{record}"
+    end
+  end
+end

data/lib/fluent/plugin/filter_where/parser.racc

@@ -0,0 +1,86 @@
+class Fluent::FilterWhere::Parser
+  prechigh
+    left OR
+    left AND
+    right NOT
+  preclow
+
+  token EQ  /* = */
+  token NEQ /* <> != */
+  token GT  /* >  */ 
+  token GE  /* >= */
+  token LT  /* <  */
+  token LE  /* <= */
+  
+  token START_WITH
+  token END_WITH
+  token INCLUDE
+  token REGEXP
+  token IS
+  token NOT
+  
+  token AND
+  token OR
+  
+  token NULL
+  token BOOLEAN
+  token STRING
+  token NUMBER
+  token IDENTIFIER
+
+  options no_result_var
+rule
+
+  input: # empty string
+   | exp { val[0] }
+   ;
+  
+  exp: IDENTIFIER EQ BOOLEAN      { BooleanOpExp.new(val[0], val[2], :EQ) }
+   | IDENTIFIER NEQ BOOLEAN       { BooleanOpExp.new(val[0], val[2], :NEQ) }
+   | BOOLEAN EQ  IDENTIFIER       { BooleanOpExp.new(val[0], val[2], :EQ) }
+   | BOOLEAN NEQ IDENTIFIER       { BooleanOpExp.new(val[0], val[2], :NEQ) }
+   | IDENTIFIER EQ  NUMBER        { NumberOpExp.new(val[0], val[2], :EQ) }
+   | IDENTIFIER NEQ NUMBER        { NumberOpExp.new(val[0], val[2], :NEQ) }
+   | IDENTIFIER GT  NUMBER        { NumberOpExp.new(val[0], val[2], :GT) }
+   | IDENTIFIER GE  NUMBER        { NumberOpExp.new(val[0], val[2], :GE) }
+   | IDENTIFIER LT  NUMBER        { NumberOpExp.new(val[0], val[2], :LT) }
+   | IDENTIFIER LE  NUMBER        { NumberOpExp.new(val[0], val[2], :LE) }
+   | NUMBER EQ  IDENTIFIER        { NumberOpExp.new(val[0], val[2], :EQ) }
+   | NUMBER NEQ IDENTIFIER        { NumberOpExp.new(val[0], val[2], :NEQ) }
+   | NUMBER GT  IDENTIFIER        { NumberOpExp.new(val[0], val[2], :GT) }
+   | NUMBER GE  IDENTIFIER        { NumberOpExp.new(val[0], val[2], :GE) }
+   | NUMBER LT  IDENTIFIER        { NumberOpExp.new(val[0], val[2], :LT) }
+   | NUMBER LE  IDENTIFIER        { NumberOpExp.new(val[0], val[2], :LE) }
+   | IDENTIFIER EQ         STRING { StringOpExp.new(val[0], val[2], :EQ) }
+   | IDENTIFIER NEQ        STRING { StringOpExp.new(val[0], val[2], :NEQ) }
+   | IDENTIFIER START_WITH STRING { StringOpExp.new(val[0], val[2], :START_WITH) }
+   | IDENTIFIER END_WITH   STRING { StringOpExp.new(val[0], val[2], :END_WITH) }
+   | IDENTIFIER INCLUDE    STRING { StringOpExp.new(val[0], val[2], :INCLUDE) }
+   | STRING EQ         IDENTIFIER { StringOpExp.new(val[0], val[2], :EQ) }
+   | STRING NEQ        IDENTIFIER { StringOpExp.new(val[0], val[2], :NEQ) }
+   | STRING START_WITH IDENTIFIER { StringOpExp.new(val[0], val[2], :START_WITH) }
+   | STRING END_WITH   IDENTIFIER { StringOpExp.new(val[0], val[2], :END_WITH) }
+   | STRING INCLUDE    IDENTIFIER { StringOpExp.new(val[0], val[2], :INCLUDE) }
+   | IDENTIFIER REGEXP STRING     { RegexpOpExp.new(val[0], val[2], :REGEXP) }
+   | IDENTIFIER IS NULL           { NullOpExp.new(val[0], :EQ) }
+   | IDENTIFIER IS NOT NULL       { NullOpExp.new(val[0], :NEQ) }
+   | exp OR exp                   { LogicalOpExp.new(val[0], val[2], :OR) }
+   | exp AND exp                  { LogicalOpExp.new(val[0], val[2], :AND) }
+   | NOT exp                      { NegateOpExp.new(val[1]) }
+   | '(' exp ')'                  { val[1] }
+   ;
+end
+
+---- header ----
+#
+# generated by racc
+#
+module Fluent; module FilterWhere; end; end
+require_relative 'parser.rex'
+require_relative 'parser/literal'
+require_relative 'parser/exp'
+
+---- inner ----
+
+---- footer ----
+

data/lib/fluent/plugin/filter_where/parser.rex

@@ -0,0 +1,92 @@
+class Fluent::FilterWhere::Parser
+option
+  ignorecase
+macro
+  Number -?[0-9]+(\.[0-9]+)?
+  QuotedIdentifierChar [^\r\n\"\\]
+  NonQuotedIdentifier [a-zA-Z$][a-zA-z0-9\.\-_]*
+  StringChar [^\r\n\'\\]
+  Newline \n|\r|\r\n
+  Whitespace [\ \t]+
+rule
+  \(                    { [:"(", text] }
+  \)                    { [:")", text] }
+  \=                    { [:EQ, text] }
+  \<\>                  { [:NEQ, text] }
+  \!\=                  { [:NEQ, text] }
+  \>\=                  { [:GE, text] }
+  \>                    { [:GT, text] }
+  \<\=                  { [:LE, text] }
+  \<                    { [:LT, text] }
+
+  # number literal
+  {Number}              { [:NUMBER, NumberLiteral.new(text)] }
+
+  # identifier literal
+  {NonQuotedIdentifier} {
+                          # rexical gem does not do longest match, so following rule is wrong
+                          # rule
+                          #   NOT { [:NOT, text] }
+                          #   {NonQuotedIdentifier} { [:IDENTIFIER, text] }
+                          # because `nothing` is treated as `not` and `hing`
+                          # Because of it, I had to write everything in {NonQuotedIdentifier}
+
+                          case text.downcase
+                          when 'and'.freeze
+                            [:AND, text]
+                          when 'or'.freeze
+                            [:OR, text]
+                          when 'start_with'.freeze
+                            [:START_WITH, text]
+                          when 'end_with'.freeze
+                            [:END_WITH, text]
+                          when 'include'.freeze
+                            [:INCLUDE, text]
+                          when 'regexp'.freeze
+                            [:REGEXP, text]
+                          when 'is'.freeze
+                            [:IS, text]
+                          when 'not'.freeze
+                            [:NOT, text]
+                          when 'null'.freeze
+                            [:NULL, text]
+                          when 'true'.freeze
+                            [:BOOLEAN, BooleanLiteral.new(text)]
+                          when 'false'.freeze
+                            [:BOOLEAN, BooleanLiteral.new(text)]
+                          else
+                            [:IDENTIFIER, IdentifierLiteral.new(text)]
+                          end
+                        }
+  \"                    { @state = :IDENTIFIER; @string = ''; nil }
+
+  # string literal
+  \'                    { @state = :STRING; @string = ''; nil }
+
+  {Whitespace}          { }
+  {Newline}             { }
+
+:IDENTIFIER   \"                      { @state = nil; [:IDENTIFIER, IdentifierLiteral.new(@string)] }
+:IDENTIFIER   {QuotedIdentifierChar}+ { @string << text; nil }
+# escape sequences
+:IDENTIFIER   \\\"                    { @string << '"'; nil }
+:IDENTIFIER   \\\'                    { @string << "'"; nil }
+:IDENTIFIER   \\\\                    { @string << "\\"; nil }
+
+:STRING       \'                      { @state = nil; [:STRING, StringLiteral.new(@string)] }
+:STRING       {StringChar}+           { @string << text; nil }
+# escape sequences
+:STRING       \b                      { @string << "\b"; nil }
+:STRING       \t                      { @string << "\t"; nil }
+:STRING       \n                      { @string << "\n"; nil }
+:STRING       \f                      { @string << "\f"; nil }
+:STRING       \r                      { @string << "\r"; nil }
+:STRING       \"                      { @string << '"'; nil }
+:STRING       \'                      { @string << "'"; nil }
+:STRING       \\                      { @string << "\\"; nil }
+
+inner
+  def on_error(error_token_id, error_value, value_stack)
+    super
+  end
+end