embulk-filter-row 0.3.3 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d1a5e3ca3f112b41b011aeedc0ac215ba78c70fd
-  data.tar.gz: 8ca385ba7785b9ed44d077dc274727e621a2c380
+  metadata.gz: 04f141506053778ed4ff092894f87c9f22acca3d
+  data.tar.gz: 8f7435bc29f4c0e97e44141771a4ce608edb429a
 SHA512:
-  metadata.gz: 7cfb6083eacafa6e1c235b49663569b1b9f23524dc455b947fb7da10d22039823d4f4a114dfc84e0e9432b76e9a9c27bb19b070e4b595e2b7a2dbe13d1e09cb8
-  data.tar.gz: b9f9da54d28074537d2bd655ba1d23acb67bb62bb88b7968ad7f2a384a648cb800f1fafb0aee920e725e7d35169ae751b89d56ca406516630cd784aa0098f358
+  metadata.gz: 5f27041d19e78633e4eedac7426d8d96e2180e047e0793f7c37b55929ba1f8da95f10966ac4643cd0dd690fe6e63ca8c82b95b0afaac64d2d96a89440f97bcf8
+  data.tar.gz: c12e1cca2eed39fa039180237808bc898e415af8a03b889c04ebe010eadcbbfd84cd82d4aaa62d18f2d17f9d71cb2382055e056c2ef30feebeceaf4ed53cc181

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 0.4.0 (2017-05-18)
+
+Enhancements
+
+* Use joni library for REGEXP operator to improve performance
+  * Note that This may introduce trival incompatibility changes
+
 # 0.3.3 (2016-08-09)
 
 Enhancements
@@ -19,8 +26,7 @@ Fxies:
 
 Changes:
 
-* Change identifier syntax from `[a-zA-Z$][a-zA-z0-9\.\-_]*` to `[a-zA-Z_][a-zA-z0-9_]*`
-  * Allow starting _. Disallow staring $. Diallow -. Disallow \.
+* Change identifier syntax from `[a-zA-Z$][a-zA-z0-9\.\-_]*` to `[a-zA-Z_][a-zA-z0-9_]*` to allow starting _, disallow staring $, disallow -, disallow . (dot).
 
 # 0.3.0 (2016-08-06)
 

data/README.md

@@ -6,93 +6,18 @@ A filter plugin for Embulk to filter out rows
 
 ## Configuration
 
-* **where**: Select only rows which matches with conditions written in SQL-like syntax. See [SQL-like Syntax](#sql-like-syntax)
-
-Following options are **deprecated**, and will be removed someday.
-
-* **condition**: AND or OR (string, default: AND).
-* **conditions**: select only rows which matches with conditions.
-  * **column**: column name (string, required)
-  * **operator** operator (string, optional, default: ==)
-    * boolean operator
-      * ==
-      * !=
-    * numeric operator (long, double, Timestamp)
-      * ==
-      * !=
-      * >
-      * >=
-      * <=
-      * <
-    * string operator
-      * ==
-      * !=
-      * start_with (or startsWith)
-      * end_with (or endsWith)
-      * include (or contains)
-    * unary operator
-      * "IS NULL"
-      * "IS NOT NULL"
-  * **argument**: argument for the operation (string, required for non-unary operators)
-  * **not**: not (boolean, optional, default: false)
-  * **format**: special option for timestamp column, specify the format of timestamp argument, parsed argument is compared with the column value as Timestamp object (string, default is `%Y-%m-%d %H:%M:%S.%N %z`)
-  * **timezone**: special option for timestamp column, specify the timezone of timestamp argument (string, default is `UTC`)
-
-NOTE: column type is automatically retrieved from input data (inputSchema)
+Versions >= 0.3.0 has `where` option to support SQL-like syntax.
 
-## Example (AND)
-
-**Deprecated**
-
-```yaml
-filters:
-  - type: row
-    condition: AND
-    conditions:
-      - {column: foo,  operator: "IS NOT NULL"}
-      - {column: id,   operator: ">=", argument: 10}
-      - {column: id,   operator: "<",  argument: 20}
-      - {column: name, opeartor: "include", argument: foo, not: true}
-      - {column: time, operator: "==", argument: "2015-07-13", format: "%Y-%m-%d"}
-```
-
-## Example (OR)
+* **where**: Select only rows which matches with conditions written in SQL-like syntax. See [SQL-like Syntax](#sql-like-syntax)
 
-**Deprecated**
+## Example
 
 ```yaml
 filters:
   - type: row
-    condition: OR
-    conditions:
-      - {column: a, operator: "IS NOT NULL"}
-      - {column: b, operator: "IS NOT NULL"}
+    where: column1 = 'str'
 ```
 
-## Example (AND of OR)
-
-**Deprecated**
-
-You can express a condition such as `(A OR B) AND (C OR D)` by combining multiple filters like
-
-```yaml
-filters:
-  - type: row
-    condition: OR
-    conditions:
-      - {column: a, operator: "IS NOT NULL"}
-      - {column: b, operator: "IS NOT NULL"}
-  - type: row
-    condition: OR
-    conditions:
-      - {column: c, operator: "IS NOT NULL"}
-      - {column: d, operator: "IS NOT NULL"}
-```
-
-## Example (WHERE)
-
-Versions >= 0.3.0 suppors SQL-like syntax like
-
 ```yaml
 filters:
   - type: row
@@ -145,6 +70,8 @@ Characters surrounded by `'` such as `'foo'` is considered as a string literal
 
 ### Timestamp Literal
 
+NOTE: It became possible to omit `TIMESTAMP` keyword on comparing with `timestamp` identifier (column) from version >= 0.3.3.
+
 `TIMESTAMP ( NumberLiteral | StringLiteral )` such as `TIMESTAMP 1470433087.747123` or `TIMESTAMP '2016-08-06 06:38:07.747123 +0900'` is considered as a timestamp literal
 
 Number is a epoch time since 1970-01-01 UTC with nano time resolution.
@@ -160,49 +87,47 @@ String is a timestamp string which matches with one of following format:
 
 The time zone for formats without `%z` is UTC, and the time resolution is micro second (caused by limitation of Embulk TimestampParser).
 
-It is possible to omit `TIMESTAMP` keyword on comparing with `timestamp` identifier from version >= 0.3.3.
-
 ### Json Literal
 
 Not supported yet
 
 ### Identifier Literal
 
-Characters matching with a regular expression `[a-zA-Z_][a-zA-z0-9_]*` such as `foobar`, and characters surrounded by `"` such as `"foo\"bar"` are considred as an identifier literal, that is, embulk's column name.
+Characters matching with a regular expression `[a-zA-Z_][a-zA-z0-9_]*` such as `foobar`, and characters surrounded by `"` such as `"foo-bar"`, `"foo.bar"`, and `"foo\"bar"` are considred as an identifier literal, that is, embulk's column name.
 
 ## Operators
 
 ### Boolean Operator
 
-* =
-* !=
+* `=`
+* `!=`
 
 ### Number Operator (Long and Double)
 
-* =
-* !=
-* >
-* >=
-* <=
-* <
+* `=`
+* `!=`
+* `>`
+* `>=`
+* `<=`
+* `<`
 
 ### String Operator
 
-* =
-* !=
-* START_WITH
-* END_WITH
-* INCLUDE
-* REGEXP
+* `=`
+* `!=`
+* `START_WITH`
+* `END_WITH`
+* `INCLUDE`
+* `REGEXP`
 
 ### Timestamp Operator
 
-* =
-* !=
-* >
-* >=
-* <=
-* <
+* `=`
+* `!=`
+* `>`
+* `>=`
+* `<=`
+* `<`
 
 ### Json Operator
 
@@ -214,9 +139,101 @@ Not supported yet
 * "xxx IS NOT NULL"
 * "NOT xxx"
 
+## Old Configuration
+
+Versions >= 0.3.0 has `where` option to supports SQL-like syntax. I recommend to use it.
+
+Following options are **deprecated**, and **will be removed someday**.
+
+* **condition**: AND or OR (string, default: AND).
+* **conditions**: select only rows which matches with conditions.
+  * **column**: column name (string, required)
+  * **operator** operator (string, optional, default: ==)
+    * boolean operator
+      * `==`
+      * `!=`
+    * numeric operator (long, double, Timestamp)
+      * `==`
+      * `!=`
+      * `>`
+      * `>=`
+      * `<=`
+      * `<`
+    * string operator
+      * `==`
+      * `!=`
+      * `start_with` (or `startsWith`)
+      * `end_with` (or `endsWith`)
+      * `include` (or `contains`)
+    * unary operator
+      * `IS NULL`
+      * `IS NOT NULL`
+  * **argument**: argument for the operation (string, required for non-unary operators)
+  * **not**: not (boolean, optional, default: false)
+  * **format**: special option for timestamp column, specify the format of timestamp argument, parsed argument is compared with the column value as Timestamp object (string, default is `%Y-%m-%d %H:%M:%S.%N %z`)
+  * **timezone**: special option for timestamp column, specify the timezone of timestamp argument (string, default is `UTC`)
+
+NOTE: column type is automatically retrieved from input data (inputSchema)
+
+## Example (AND)
+
+**Deprecated**
+
+```yaml
+filters:
+  - type: row
+    condition: AND
+    conditions:
+      - {column: foo,  operator: "IS NOT NULL"}
+      - {column: id,   operator: ">=", argument: 10}
+      - {column: id,   operator: "<",  argument: 20}
+      - {column: name, opeartor: "include", argument: foo, not: true}
+      - {column: time, operator: "==", argument: "2015-07-13", format: "%Y-%m-%d"}
+```
+
+## Example (OR)
+
+**Deprecated**
+
+```yaml
+filters:
+  - type: row
+    condition: OR
+    conditions:
+      - {column: a, operator: "IS NOT NULL"}
+      - {column: b, operator: "IS NOT NULL"}
+```
+
+## Example (AND of OR)
+
+**Deprecated**
+
+You can express a condition such as `(A OR B) AND (C OR D)` by combining multiple filters like
+
+```yaml
+filters:
+  - type: row
+    condition: OR
+    conditions:
+      - {column: a, operator: "IS NOT NULL"}
+      - {column: b, operator: "IS NOT NULL"}
+  - type: row
+    condition: OR
+    conditions:
+      - {column: c, operator: "IS NOT NULL"}
+      - {column: d, operator: "IS NOT NULL"}
+```
+
+## Comparisions
+
+* [embulk-filter-calcite](https://github.com/muga/embulk-filter-calcite)
+  * embulk-filter-calcite is a pretty nice plugin which enables us to write SQL query to filter embulk records.
+  * However, based on [my benchmark (Japanese)](http://qiita.com/sonots/items/a70482d29862de87624d), embulk-filter-calcite was 1471 times slower than embulk-filter-row in iterms of string `=` operator. Use `embulk-filter-row` (this plugin) when you need performance.
+
 ## ToDo
 
 * Support filtering by values of `type: json` with JSONPath
+* Support IN operator
 
 ## ChangeLog
 

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.3.3"
+version = "0.4.0"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
@@ -23,6 +23,9 @@ dependencies {
     compile  "org.embulk:embulk-core:0.8.+"
     provided "org.embulk:embulk-core:0.8.+"
 
+    compile  "org.jruby.joni:joni:2.1.11"
+    compile  "org.jruby.jcodings:jcodings:1.0.18"
+
     testCompile "junit:junit:4.+"
     testCompile "org.embulk:embulk-core:0.8.+:tests"
     testCompile "org.embulk:embulk-standards:0.8.+"

data/example/regexp_multibyte.yml

@@ -0,0 +1,24 @@
+in:
+  type: file
+  path_prefix: example/regexp_multibyte.csv
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: "NULL"
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: time,  type: timestamp, format: "%Y-%m-%d"}
+      - {name: foo,   type: string}
+      - {name: bar,   type: string}
+      - {name: flag,  type: boolean}
+      - {name: id,    type: long}
+      - {name: name,  type: string}
+      - {name: json,  type: json}
+      - {name: score, type: double}
+filters:
+  - type: row
+    where: name REGEXP '.*チ'
+out:
+  type: stdout

data/src/main/java/org/embulk/filter/row/where/ParserExp.java

@@ -4,8 +4,12 @@ import org.embulk.config.ConfigException;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.time.Timestamp;
 
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
+import org.jcodings.specific.UTF8Encoding;
+import org.joni.Matcher;
+import org.joni.Option;
+import org.joni.Regex;
+
+import java.nio.charset.StandardCharsets;
 
 // Operation Node of AST (Abstract Syntax Tree)
 public abstract class ParserExp extends ParserNode
@@ -282,12 +286,15 @@ class StringOpExp extends BinaryOpExp
 
 class RegexpOpExp extends BinaryOpExp
 {
-    Pattern pattern;
+    Regex regex;
 
     public RegexpOpExp(ParserLiteral left, ParserLiteral right, int operator)
     {
         super(left, right, operator);
-        this.pattern = Pattern.compile(((StringLiteral)right).val);
+
+        byte[] pattern = (((StringLiteral)right).val).getBytes(StandardCharsets.UTF_8);
+        this.regex = new Regex(pattern, 0, pattern.length, Option.NONE, UTF8Encoding.INSTANCE);
+
         if (! left.isString()) {
             throw new ConfigException(String.format("\"%s\" is not a String column", ((IdentifierLiteral)left).name));
         }
@@ -300,9 +307,10 @@ class RegexpOpExp extends BinaryOpExp
 
     public boolean eval(PageReader pageReader)
     {
-        String l = left.getString(pageReader);
-        Matcher m = pattern.matcher(l);
-        return m.find();
+        byte[] l = left.getString(pageReader).getBytes(StandardCharsets.UTF_8);
+        Matcher matcher = regex.matcher(l);
+        int result = matcher.search(0, l.length, Option.DEFAULT);
+        return result != -1;
     }
 }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-row
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-09 00:00:00.000000000 Z
+date: 2017-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -56,6 +56,8 @@ files:
 - example/example.csv
 - example/example.yml
 - example/or.yml
+- example/regexp_multibyte.csv
+- example/regexp_multibyte.yml
 - example/where.yml
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
@@ -94,7 +96,9 @@ files:
 - src/test/java/org/embulk/filter/row/condition/TestTimestampCondition.java
 - src/test/java/org/embulk/filter/row/where/TestParser.java
 - src/test/java/org/embulk/filter/row/where/TestYylex.java
-- classpath/embulk-filter-row-0.3.3.jar
+- classpath/embulk-filter-row-0.4.0.jar
+- classpath/jcodings-1.0.18.jar
+- classpath/joni-2.1.11.jar
 homepage: https://github.com/sonots/embulk-filter-row
 licenses:
 - MIT