fluent-plugin-filter_where 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/.travis.yml +9 -0
- data/CHANGELOG.md +3 -0
- data/Gemfile +3 -0
- data/Gemfile.fluentd.0.12 +4 -0
- data/LICENSE +22 -0
- data/README.md +141 -0
- data/Rakefile +24 -0
- data/example.conf +14 -0
- data/fluent-plugin-filter_where.gemspec +30 -0
- data/lib/fluent/plugin/filter_where.rb +7 -0
- data/lib/fluent/plugin/filter_where/core.rb +32 -0
- data/lib/fluent/plugin/filter_where/parser.racc +86 -0
- data/lib/fluent/plugin/filter_where/parser.rex +92 -0
- data/lib/fluent/plugin/filter_where/parser.rex.rb +206 -0
- data/lib/fluent/plugin/filter_where/parser.tab.rb +419 -0
- data/lib/fluent/plugin/filter_where/parser/exp.rb +178 -0
- data/lib/fluent/plugin/filter_where/parser/literal.rb +56 -0
- data/lib/fluent/plugin/filter_where/v12.rb +26 -0
- data/lib/fluent/plugin/filter_where/v14.rb +21 -0
- data/test/bench_filter_where.rb +23 -0
- data/test/helper.rb +98 -0
- data/test/test_filter_where.rb +37 -0
- data/test/test_where_parser.rb +109 -0
- metadata +197 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 14fee05a0ddb942780d1f9b4722d99549206183f
|
4
|
+
data.tar.gz: 040111b31bb0b6b9f56f85efc29ddbe9153b25e0
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 30106ab7026afa2f66d0248309f51cae62a22d1c16fec769a2db98d46ebdb69315dd8d170a3ecaa22bf9eed3946fdd8c64baecb360b6ca7a720a553c5e026dd8
|
7
|
+
data.tar.gz: 911625bfdcf3351b780957639f70db66ab32ca273e5ca0cc0e014ec8d33f76a6223e0150e032cc1f2bb84b4bca3905eba3fe58f71a09f401fd2d77ca7a00ea20
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2017 Naotoshi Seo
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,141 @@
|
|
1
|
+
# fluent-plugin-filter_where
|
2
|
+
|
3
|
+
[](http://travis-ci.org/sonots/fluent-plugin-filter_where)
|
4
|
+
|
5
|
+
Fluentd plugin to filter records with SQL-like WHERE statements
|
6
|
+
|
7
|
+
## Requirements
|
8
|
+
|
9
|
+
See [.travis.yml](.travis.yml)
|
10
|
+
|
11
|
+
`fluent-plugin-filter_where` supports both v0.14 API and v0.12 API in one gem.
|
12
|
+
|
13
|
+
## Installation
|
14
|
+
|
15
|
+
Use RubyGems:
|
16
|
+
|
17
|
+
gem install fluent-plugin-filter_where
|
18
|
+
|
19
|
+
## Configuration
|
20
|
+
|
21
|
+
- where
|
22
|
+
|
23
|
+
The SQL-like WHERE statements. See [SQL-like Syntax](#sql-like-syntax) for more details.
|
24
|
+
|
25
|
+
### Example
|
26
|
+
|
27
|
+
```apache
|
28
|
+
<filter foo.**>
|
29
|
+
@type where
|
30
|
+
where string_key = 'string' OR number_key >= 0.1
|
31
|
+
</filter>
|
32
|
+
```
|
33
|
+
|
34
|
+
## SQL-like Syntax
|
35
|
+
|
36
|
+
Example:
|
37
|
+
|
38
|
+
```sql
|
39
|
+
where (string_key START_WITH 'str' AND number_key > 1.0) OR ("true_key" = true AND string_key REGEXP '^reg')
|
40
|
+
```
|
41
|
+
|
42
|
+
## Literals
|
43
|
+
|
44
|
+
### Boolean Literal
|
45
|
+
|
46
|
+
`true` or `TRUE` or `false` or `FALSE` are considered as a boolean literal
|
47
|
+
|
48
|
+
### Number Literal
|
49
|
+
|
50
|
+
Characters matching with a regular expression `-?[0-9]+(\.[0-9]+)?` is considered as a number literal
|
51
|
+
|
52
|
+
### String Literal
|
53
|
+
|
54
|
+
Characters surrounded by `'` such as `'foo'` is considered as a string literal
|
55
|
+
|
56
|
+
### Json Literal
|
57
|
+
|
58
|
+
Not supported yet
|
59
|
+
|
60
|
+
### Identifier Literal
|
61
|
+
|
62
|
+
Characters matching with a regular expression `[a-zA-Z_][a-zA-z0-9_]*` such as `foobar`, and characters surrounded by `"` such as `"foo-bar"`, `"foo.bar"`, and `"foo\"bar"` are considred as an identifier literal, that is, embulk's column name.
|
63
|
+
|
64
|
+
## Operators
|
65
|
+
|
66
|
+
### Boolean Operator
|
67
|
+
|
68
|
+
* `=`
|
69
|
+
* `!=`
|
70
|
+
|
71
|
+
### Number Operator (Long and Double)
|
72
|
+
|
73
|
+
* `=`
|
74
|
+
* `!=`
|
75
|
+
* `>`
|
76
|
+
* `>=`
|
77
|
+
* `<=`
|
78
|
+
* `<`
|
79
|
+
|
80
|
+
### String Operator
|
81
|
+
|
82
|
+
* `=`
|
83
|
+
* `!=`
|
84
|
+
* `START_WITH`
|
85
|
+
* `END_WITH`
|
86
|
+
* `INCLUDE`
|
87
|
+
* `REGEXP`
|
88
|
+
|
89
|
+
### Json Operator
|
90
|
+
|
91
|
+
Not supported yet
|
92
|
+
|
93
|
+
### unary operator
|
94
|
+
|
95
|
+
* "xxx IS NULL"
|
96
|
+
* "xxx IS NOT NULL"
|
97
|
+
* "NOT xxx"
|
98
|
+
|
99
|
+
## ChangeLog
|
100
|
+
|
101
|
+
See [CHANGELOG.md](CHANGELOG.md) for details.
|
102
|
+
|
103
|
+
## Development
|
104
|
+
|
105
|
+
Run test:
|
106
|
+
|
107
|
+
```
|
108
|
+
$ bundle exec rake test
|
109
|
+
```
|
110
|
+
|
111
|
+
Release:
|
112
|
+
|
113
|
+
Modify gemspec and CHANGELOG.md, then
|
114
|
+
|
115
|
+
```
|
116
|
+
$ bundle exec rake release
|
117
|
+
```
|
118
|
+
|
119
|
+
## Development of SQL-like Syntax
|
120
|
+
|
121
|
+
This plugin uses [rexical](https://github.com/tenderlove/rexical) for lexical scanner generator, and [racc](https://github.com/tenderlove/racc) for parser generator.
|
122
|
+
|
123
|
+
If you modify `praser.rex` or `praser.racc`, you must compile them as:
|
124
|
+
|
125
|
+
```
|
126
|
+
$ bundle exec rake compile
|
127
|
+
```
|
128
|
+
|
129
|
+
The `test` task runs the `compile` task before running.
|
130
|
+
|
131
|
+
## Contributing
|
132
|
+
|
133
|
+
1. Fork it
|
134
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
135
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
136
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
137
|
+
5. Create new [Pull Request](../../pull/new/master)
|
138
|
+
|
139
|
+
## Copyright
|
140
|
+
|
141
|
+
Copyright (c) 2017 - Naotoshi Seo. See [LICENSE](LICENSE) for details.
|
data/Rakefile
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
|
4
|
+
require 'rake/testtask'
|
5
|
+
desc 'Run test_unit based test'
|
6
|
+
Rake::TestTask.new(:test) do |t|
|
7
|
+
t.libs << "test"
|
8
|
+
t.test_files = Dir["test/**/test_*.rb"].sort
|
9
|
+
t.verbose = true
|
10
|
+
#t.warning = true
|
11
|
+
end
|
12
|
+
task :default => :test
|
13
|
+
task :test => [:compile]
|
14
|
+
|
15
|
+
desc 'Open an irb session preloaded with the gem library'
|
16
|
+
task :console do
|
17
|
+
sh 'irb -rubygems -I lib'
|
18
|
+
end
|
19
|
+
task :c => :console
|
20
|
+
|
21
|
+
task :compile do
|
22
|
+
sh 'racc lib/fluent/plugin/filter_where/parser.racc'
|
23
|
+
sh 'rex lib/fluent/plugin/filter_where/parser.rex'
|
24
|
+
end
|
data/example.conf
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
$:.push File.expand_path('../lib', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.name = "fluent-plugin-filter_where"
|
6
|
+
gem.version = "1.0.0"
|
7
|
+
gem.authors = ["Naotoshi Seo"]
|
8
|
+
gem.email = "sonots@gmail.com"
|
9
|
+
gem.homepage = "https://github.com/sonots/fluent-plugin-filter_where"
|
10
|
+
gem.description = "Fluentd plugin to filter records with SQL-like WHERE statements"
|
11
|
+
gem.summary = gem.description
|
12
|
+
gem.licenses = ["MIT"]
|
13
|
+
gem.has_rdoc = false
|
14
|
+
|
15
|
+
gem.files = `git ls-files`.split("\n")
|
16
|
+
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
+
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
|
+
gem.require_paths = ['lib']
|
19
|
+
|
20
|
+
gem.add_dependency "fluentd"
|
21
|
+
|
22
|
+
gem.add_development_dependency "racc"
|
23
|
+
gem.add_development_dependency "rexical"
|
24
|
+
gem.add_development_dependency "rake"
|
25
|
+
gem.add_development_dependency "pry"
|
26
|
+
gem.add_development_dependency "pry-nav"
|
27
|
+
gem.add_development_dependency "test-unit"
|
28
|
+
gem.add_development_dependency "test-unit-rr"
|
29
|
+
gem.add_development_dependency "timecop"
|
30
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'fluent/plugin/filter_where/parser.tab'
|
2
|
+
|
3
|
+
module Fluent; module FilterWhere; end; end
|
4
|
+
module Fluent
|
5
|
+
module FilterWhere::Core
|
6
|
+
def initialize
|
7
|
+
super
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.included(klass)
|
11
|
+
klass.config_param :where, :string, :desc => 'The SQL-like WHERE statement.'
|
12
|
+
end
|
13
|
+
|
14
|
+
def configure(conf)
|
15
|
+
super
|
16
|
+
|
17
|
+
parser = Fluent::FilterWhere::Parser.new
|
18
|
+
@scanner = parser.scan(@where)
|
19
|
+
end
|
20
|
+
|
21
|
+
def filter(tag, time, record)
|
22
|
+
if @scanner.eval(record)
|
23
|
+
record
|
24
|
+
else
|
25
|
+
nil # remove
|
26
|
+
end
|
27
|
+
rescue => e
|
28
|
+
log.warn "filter_where: #{e.class} #{e.message} #{e.backtrace.first}"
|
29
|
+
log.debug "filter_where:: tag:#{tag} time:#{time} record:#{record}"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
class Fluent::FilterWhere::Parser
|
2
|
+
prechigh
|
3
|
+
left OR
|
4
|
+
left AND
|
5
|
+
right NOT
|
6
|
+
preclow
|
7
|
+
|
8
|
+
token EQ /* = */
|
9
|
+
token NEQ /* <> != */
|
10
|
+
token GT /* > */
|
11
|
+
token GE /* >= */
|
12
|
+
token LT /* < */
|
13
|
+
token LE /* <= */
|
14
|
+
|
15
|
+
token START_WITH
|
16
|
+
token END_WITH
|
17
|
+
token INCLUDE
|
18
|
+
token REGEXP
|
19
|
+
token IS
|
20
|
+
token NOT
|
21
|
+
|
22
|
+
token AND
|
23
|
+
token OR
|
24
|
+
|
25
|
+
token NULL
|
26
|
+
token BOOLEAN
|
27
|
+
token STRING
|
28
|
+
token NUMBER
|
29
|
+
token IDENTIFIER
|
30
|
+
|
31
|
+
options no_result_var
|
32
|
+
rule
|
33
|
+
|
34
|
+
input: # empty string
|
35
|
+
| exp { val[0] }
|
36
|
+
;
|
37
|
+
|
38
|
+
exp: IDENTIFIER EQ BOOLEAN { BooleanOpExp.new(val[0], val[2], :EQ) }
|
39
|
+
| IDENTIFIER NEQ BOOLEAN { BooleanOpExp.new(val[0], val[2], :NEQ) }
|
40
|
+
| BOOLEAN EQ IDENTIFIER { BooleanOpExp.new(val[0], val[2], :EQ) }
|
41
|
+
| BOOLEAN NEQ IDENTIFIER { BooleanOpExp.new(val[0], val[2], :NEQ) }
|
42
|
+
| IDENTIFIER EQ NUMBER { NumberOpExp.new(val[0], val[2], :EQ) }
|
43
|
+
| IDENTIFIER NEQ NUMBER { NumberOpExp.new(val[0], val[2], :NEQ) }
|
44
|
+
| IDENTIFIER GT NUMBER { NumberOpExp.new(val[0], val[2], :GT) }
|
45
|
+
| IDENTIFIER GE NUMBER { NumberOpExp.new(val[0], val[2], :GE) }
|
46
|
+
| IDENTIFIER LT NUMBER { NumberOpExp.new(val[0], val[2], :LT) }
|
47
|
+
| IDENTIFIER LE NUMBER { NumberOpExp.new(val[0], val[2], :LE) }
|
48
|
+
| NUMBER EQ IDENTIFIER { NumberOpExp.new(val[0], val[2], :EQ) }
|
49
|
+
| NUMBER NEQ IDENTIFIER { NumberOpExp.new(val[0], val[2], :NEQ) }
|
50
|
+
| NUMBER GT IDENTIFIER { NumberOpExp.new(val[0], val[2], :GT) }
|
51
|
+
| NUMBER GE IDENTIFIER { NumberOpExp.new(val[0], val[2], :GE) }
|
52
|
+
| NUMBER LT IDENTIFIER { NumberOpExp.new(val[0], val[2], :LT) }
|
53
|
+
| NUMBER LE IDENTIFIER { NumberOpExp.new(val[0], val[2], :LE) }
|
54
|
+
| IDENTIFIER EQ STRING { StringOpExp.new(val[0], val[2], :EQ) }
|
55
|
+
| IDENTIFIER NEQ STRING { StringOpExp.new(val[0], val[2], :NEQ) }
|
56
|
+
| IDENTIFIER START_WITH STRING { StringOpExp.new(val[0], val[2], :START_WITH) }
|
57
|
+
| IDENTIFIER END_WITH STRING { StringOpExp.new(val[0], val[2], :END_WITH) }
|
58
|
+
| IDENTIFIER INCLUDE STRING { StringOpExp.new(val[0], val[2], :INCLUDE) }
|
59
|
+
| STRING EQ IDENTIFIER { StringOpExp.new(val[0], val[2], :EQ) }
|
60
|
+
| STRING NEQ IDENTIFIER { StringOpExp.new(val[0], val[2], :NEQ) }
|
61
|
+
| STRING START_WITH IDENTIFIER { StringOpExp.new(val[0], val[2], :START_WITH) }
|
62
|
+
| STRING END_WITH IDENTIFIER { StringOpExp.new(val[0], val[2], :END_WITH) }
|
63
|
+
| STRING INCLUDE IDENTIFIER { StringOpExp.new(val[0], val[2], :INCLUDE) }
|
64
|
+
| IDENTIFIER REGEXP STRING { RegexpOpExp.new(val[0], val[2], :REGEXP) }
|
65
|
+
| IDENTIFIER IS NULL { NullOpExp.new(val[0], :EQ) }
|
66
|
+
| IDENTIFIER IS NOT NULL { NullOpExp.new(val[0], :NEQ) }
|
67
|
+
| exp OR exp { LogicalOpExp.new(val[0], val[2], :OR) }
|
68
|
+
| exp AND exp { LogicalOpExp.new(val[0], val[2], :AND) }
|
69
|
+
| NOT exp { NegateOpExp.new(val[1]) }
|
70
|
+
| '(' exp ')' { val[1] }
|
71
|
+
;
|
72
|
+
end
|
73
|
+
|
74
|
+
---- header ----
|
75
|
+
#
|
76
|
+
# generated by racc
|
77
|
+
#
|
78
|
+
module Fluent; module FilterWhere; end; end
|
79
|
+
require_relative 'parser.rex'
|
80
|
+
require_relative 'parser/literal'
|
81
|
+
require_relative 'parser/exp'
|
82
|
+
|
83
|
+
---- inner ----
|
84
|
+
|
85
|
+
---- footer ----
|
86
|
+
|
@@ -0,0 +1,92 @@
|
|
1
|
+
class Fluent::FilterWhere::Parser
|
2
|
+
option
|
3
|
+
ignorecase
|
4
|
+
macro
|
5
|
+
Number -?[0-9]+(\.[0-9]+)?
|
6
|
+
QuotedIdentifierChar [^\r\n\"\\]
|
7
|
+
NonQuotedIdentifier [a-zA-Z$][a-zA-z0-9\.\-_]*
|
8
|
+
StringChar [^\r\n\'\\]
|
9
|
+
Newline \n|\r|\r\n
|
10
|
+
Whitespace [\ \t]+
|
11
|
+
rule
|
12
|
+
\( { [:"(", text] }
|
13
|
+
\) { [:")", text] }
|
14
|
+
\= { [:EQ, text] }
|
15
|
+
\<\> { [:NEQ, text] }
|
16
|
+
\!\= { [:NEQ, text] }
|
17
|
+
\>\= { [:GE, text] }
|
18
|
+
\> { [:GT, text] }
|
19
|
+
\<\= { [:LE, text] }
|
20
|
+
\< { [:LT, text] }
|
21
|
+
|
22
|
+
# number literal
|
23
|
+
{Number} { [:NUMBER, NumberLiteral.new(text)] }
|
24
|
+
|
25
|
+
# identifier literal
|
26
|
+
{NonQuotedIdentifier} {
|
27
|
+
# rexical gem does not do longest match, so following rule is wrong
|
28
|
+
# rule
|
29
|
+
# NOT { [:NOT, text] }
|
30
|
+
# {NonQuotedIdentifier} { [:IDENTIFIER, text] }
|
31
|
+
# because `nothing` is treated as `not` and `hing`
|
32
|
+
# Because of it, I had to write everything in {NonQuotedIdentifier}
|
33
|
+
|
34
|
+
case text.downcase
|
35
|
+
when 'and'.freeze
|
36
|
+
[:AND, text]
|
37
|
+
when 'or'.freeze
|
38
|
+
[:OR, text]
|
39
|
+
when 'start_with'.freeze
|
40
|
+
[:START_WITH, text]
|
41
|
+
when 'end_with'.freeze
|
42
|
+
[:END_WITH, text]
|
43
|
+
when 'include'.freeze
|
44
|
+
[:INCLUDE, text]
|
45
|
+
when 'regexp'.freeze
|
46
|
+
[:REGEXP, text]
|
47
|
+
when 'is'.freeze
|
48
|
+
[:IS, text]
|
49
|
+
when 'not'.freeze
|
50
|
+
[:NOT, text]
|
51
|
+
when 'null'.freeze
|
52
|
+
[:NULL, text]
|
53
|
+
when 'true'.freeze
|
54
|
+
[:BOOLEAN, BooleanLiteral.new(text)]
|
55
|
+
when 'false'.freeze
|
56
|
+
[:BOOLEAN, BooleanLiteral.new(text)]
|
57
|
+
else
|
58
|
+
[:IDENTIFIER, IdentifierLiteral.new(text)]
|
59
|
+
end
|
60
|
+
}
|
61
|
+
\" { @state = :IDENTIFIER; @string = ''; nil }
|
62
|
+
|
63
|
+
# string literal
|
64
|
+
\' { @state = :STRING; @string = ''; nil }
|
65
|
+
|
66
|
+
{Whitespace} { }
|
67
|
+
{Newline} { }
|
68
|
+
|
69
|
+
:IDENTIFIER \" { @state = nil; [:IDENTIFIER, IdentifierLiteral.new(@string)] }
|
70
|
+
:IDENTIFIER {QuotedIdentifierChar}+ { @string << text; nil }
|
71
|
+
# escape sequences
|
72
|
+
:IDENTIFIER \\\" { @string << '"'; nil }
|
73
|
+
:IDENTIFIER \\\' { @string << "'"; nil }
|
74
|
+
:IDENTIFIER \\\\ { @string << "\\"; nil }
|
75
|
+
|
76
|
+
:STRING \' { @state = nil; [:STRING, StringLiteral.new(@string)] }
|
77
|
+
:STRING {StringChar}+ { @string << text; nil }
|
78
|
+
# escape sequences
|
79
|
+
:STRING \b { @string << "\b"; nil }
|
80
|
+
:STRING \t { @string << "\t"; nil }
|
81
|
+
:STRING \n { @string << "\n"; nil }
|
82
|
+
:STRING \f { @string << "\f"; nil }
|
83
|
+
:STRING \r { @string << "\r"; nil }
|
84
|
+
:STRING \" { @string << '"'; nil }
|
85
|
+
:STRING \' { @string << "'"; nil }
|
86
|
+
:STRING \\ { @string << "\\"; nil }
|
87
|
+
|
88
|
+
inner
|
89
|
+
def on_error(error_token_id, error_value, value_stack)
|
90
|
+
super
|
91
|
+
end
|
92
|
+
end
|