embulk-filter-row 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e658f035d50a57d97df3a874a87754e0eafc3114
-  data.tar.gz: 505d52511a7527b4bab0585a075548af27ed0c7e
+  metadata.gz: 50dcbbafd3bb2429121a86d74143612f79ca4570
+  data.tar.gz: ee2a1c37ef931525a43764976d258004b1d24f8e
 SHA512:
-  metadata.gz: c0943ff3946f42199d898326419677f74d2b162a7d515b3ce6f6847b162b08e4469af946e42ddc61c50a4c0b3c6ca2b35530bc880a5531d60e3102c873c066a6
-  data.tar.gz: e21c28850e48e2d9d816ca8922e2ae7019cb2828976b99807f9f063190f492f89397e32b6648b2aeab4d0e20e08e3b66b987f66d606c121691c06c1be766a341
+  metadata.gz: 016cc549f49bdc513c9a92e7550779484c1d0a3a8d465f00de6be3db9ee94b39ed280dd9423ae592562f0366b671aeb54eacd612a8a8f3a7fe4494d99ee6959a
+  data.tar.gz: cc67e9ae035b6ccb4d9c5892b9f638ed51910061c38061040a3c526f0f316c1ae5f61d57aea79aa4ba94ef588382d1b1011efbbb36a0eef779cd6606779f8107

data/.gitignore

@@ -10,3 +10,6 @@ build/
 .tags
 .ruby-version
 *.iml
+*.class
+script/yacc
+script/jflex*

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.3.0 (2016-08-06)
+
+Enhancements:
+
+* Support SQL-like Syntax
+
 # 0.2.2 (2016-08-05)
 
 Enhancements:

data/README.md

@@ -33,6 +33,7 @@ A filter plugin for Embulk to filter out rows
   * **not**: not (boolean, optional, default: false)
   * **format**: special option for timestamp column, specify the format of timestamp argument, parsed argument is compared with the column value as Timestamp object (string, default is `%Y-%m-%d %H:%M:%S.%N %z`)
   * **timezone**: special option for timestamp column, specify the timezone of timestamp argument (string, default is `UTC`)
+* **where** (experimental): Write conditions with SQL-like syntax. See [SQL-like Syntax](#sql-like-syntax)
 
 NOTE: column type is automatically retrieved from input data (inputSchema)
 
@@ -63,7 +64,7 @@ filters:
 
 ## Example (AND of OR)
 
-embulk-filter-row does not directly supports complex conditions such as `((A OR B) AND (C OR D))`, but you should be able to express most of complex conditions by combining multiple filters like
+You can express a condition such as `(A OR B) AND (C OR D)` by combining multiple filters like
 
 ```yaml
 filters:
@@ -81,10 +82,129 @@ filters:
 
 This is equivalent with `((A OR B) AND (C OR D))`.
 
-## Not Supported:  More Complex Conditions
+## Example (WHERE) (Experimental)
 
-* It should be better to think using Query engine like [Apache Drill](https://drill.apache.org/) or [Presto](https://prestodb.io/)
-* With them, it is possible to send a query to local files, even to S3 files.
+Versions >= 0.3.0 suppors SQL-like syntax like
+
+```yaml
+filters:
+  - type: row
+    where: |-
+      (
+        string START_WITH 'str' AND
+        number > 1.0
+      )
+      OR
+      (
+        time = TIMESTAMP '2016-01-01 +0900' AND
+        "true" = true
+      )
+```
+
+See [SQL-like Syntax](#sql-like-syntax) for more details
+
+# SQL-like Syntax
+
+Versions >= 0.3.0 suppors SQL-like syntax as:
+
+```
+    where: |-
+      (
+        string START_WITH 'str' AND
+        number > 1.0
+      )
+      OR
+      (
+        time = TIMESTAMP '2016-01-01 +0900' AND
+        "true" = true
+      )
+```
+
+## Literals
+
+### Boolean Literal
+
+`true` or `TRUE` or `false` or `FALSE` are considered as a boolean literal
+
+### Number Literal
+
+Characters matching with a regular expression `-?[0-9]+(\.[0-9]+)?` is considered as a number literal
+
+### String Literal
+
+Characters surrounded by `'` such as `'foo'` is considered as a string literal
+
+### Timestamp Literal
+
+`TIMESTAMP ( NumberLiteral | StringLiteral )` such as `TIMESTAMP 1470433087.747123` or `TIMESTAMP '2016-08-06 06:38:07.747123 +0900'` is considered as a timestamp literal
+
+Number is a epoch time since 1970-01-01 UTC with nano time resolution.
+
+String is a timestamp string which matches with one of following format:
+
+* `%Y-%m-%d %H:%M:%S.%N %z`
+* `%Y-%m-%d %H:%M:%S.%N`
+* `%Y-%m-%d %H:%M:%S %z`
+* `%Y-%m-%d %H:%M:%S`
+* `%Y-%m-%d %z`
+* `%Y-%m-%d`
+
+The default time zone is UTC, and the time resolution is micro second (caused by limitation of Embulk TimestampParser).
+
+### Json Literal
+
+Not supported yet
+
+### Identifier Literal
+
+Characters matching with a regular expression `[a-zA-Z$][a-zA-z0-9\.\-_]*` such as `foobar`, and characters surrounded by `"` such as `"foo\"bar"` are considred as an identifier literal, that is, embulk's column name.
+
+## Operators
+
+### Boolean Operator
+
+* ==
+* !=
+
+### Number Operator (Long and Double)
+
+* ==
+* !=
+* >
+* >=
+* <=
+* <
+
+### String Operator
+
+* ==
+* !=
+* START_WITH
+* END_WITH
+* INCLUDE
+
+### Timestamp Operator
+
+* ==
+* !=
+* >
+* >=
+* <=
+* <
+
+### Json Operator
+
+Not supported yet
+
+### unary operator
+
+* "xxx IS NULL"
+* "xxx IS NOT NULL"
+* "NOT xxx"
+
+## ToDo
+
+* Support filtering by values of `type: json` with JSONPath
 
 ## ChangeLog
 

data/build.gradle

@@ -3,6 +3,8 @@ plugins {
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
     id "checkstyle"
+    id "jacoco"
+    id "findbugs"
 }
 import com.github.jrubygradle.JRubyExec
 repositories {
@@ -13,21 +15,30 @@ configurations {
     provided
 }
 
-version = "0.2.2"
+version = "0.3.0"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.11"
-    provided "org.embulk:embulk-core:0.8.11"
-    // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
+    compile  "org.embulk:embulk-core:0.8.+"
+    provided "org.embulk:embulk-core:0.8.+"
+
     testCompile "junit:junit:4.+"
+    testCompile "org.embulk:embulk-core:0.8.+:tests"
+    testCompile "org.embulk:embulk-standards:0.8.+"
 }
 
 checkstyle {
     toolVersion = '6.7'
 }
 
+task byaccj() {
+    [file("script/byaccj.sh")].execute()
+}
+
+// append dependsOn to compileJava task
+compileJava.dependsOn(byaccj)
+
 task classpath(type: Copy, dependsOn: ["jar"]) {
     doFirst { file("classpath").deleteDir() }
     from (configurations.runtime - configurations.provided + files(jar.archivePath))

data/example/and.yml

@@ -1,10 +1,3 @@
-# in:
-#   type: random
-#   rows: 100
-#   schema:
-#     id: primary_key
-#     name: string
-#     score: integer
 in:
   type: file
   path_prefix: example/example.csv

data/example/example.yml

@@ -1,10 +1,3 @@
-# in:
-#   type: random
-#   rows: 100
-#   schema:
-#     id: primary_key
-#     name: string
-#     score: integer
 in:
   type: file
   path_prefix: example/example.csv

data/example/where.yml

@@ -0,0 +1,28 @@
+in:
+  type: file
+  path_prefix: example/example.csv
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: "NULL"
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: time,  type: timestamp, format: "%Y-%m-%d"}
+      - {name: foo,   type: string}
+      - {name: bar,   type: string}
+      - {name: flag,  type: boolean}
+      - {name: id,    type: long}
+      - {name: name,  type: string}
+      - {name: json,  type: json}
+      - {name: score, type: double}
+filters:
+  - type: row
+    where: |-
+      (time = TIMESTAMP '2015-07-13' AND
+      foo IS NULL AND
+      bar = 'bar' AND score>2000 AND
+      flag=false) OR (id=9)
+out:
+  type: stdout

data/script/byaccj.sh

@@ -0,0 +1,29 @@
+#!/bin/bash -e
+BYACCJ_VERSION="1.15"
+JFLEX_VERSION="1.6.1"
+
+ROOT_PATH=$(cd $(dirname $0); cd ..; pwd)
+SRC_DIR="src/main/java/org/embulk/filter/row/where"
+
+if [ $(uname) = "Darwin" ]; then
+  platform="macosx"
+else
+  platform="linux"
+fi
+
+if [ ! -f "$ROOT_PATH/script/yacc" ]; then
+  curl -s -L "http://downloads.sourceforge.net/project/byaccj/byaccj/${BYACCJ_VERSION}/byaccj${BYACCJ_VERSION}_${platform}.tar.gz" | tar xz -C $ROOT_PATH/script/
+  mv $ROOT_PATH/script/yacc.${platform} $ROOT_PATH/script/yacc
+  chmod a+x $ROOT_PATH/script/yacc
+fi
+
+if [ ! -L "$ROOT_PATH/script/jflex" ]; then
+  curl -s -L "http://jflex.de/release/jflex-${JFLEX_VERSION}.tar.gz" | tar xz -C $ROOT_PATH/script/
+fi
+
+# yacc
+$ROOT_PATH/script/yacc -J -Jclass=Parser -Jpackage=org.embulk.filter.row.where -Jnoconstruct "$SRC_DIR/_parser.y"
+mv Parser.java ParserVal.java $SRC_DIR/
+
+# flex
+$ROOT_PATH/script/jflex/bin/jflex -d $SRC_DIR "$SRC_DIR/_lexer.l"

data/src/main/java/org/embulk/filter/row/{AbstractColumnVisitor.java → AbstractGuardColumnVisitor.java}

@@ -6,8 +6,6 @@ import org.embulk.filter.row.condition.Condition;
 import org.embulk.filter.row.condition.ConditionConfig;
 import org.embulk.filter.row.condition.ConditionFactory;
 import org.embulk.spi.Column;
-import org.embulk.spi.ColumnVisitor;
-import org.embulk.spi.PageBuilder;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 
@@ -15,25 +13,19 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 
-abstract class AbstractColumnVisitor
-        implements ColumnVisitor
+abstract class AbstractGuardColumnVisitor
 {
-    final PluginTask task;
-    final Schema inputSchema;
-    final Schema outputSchema;
-    final PageReader pageReader;
-    final PageBuilder pageBuilder;
-    HashMap<String, List<Condition>> conditionMap;
-
-    AbstractColumnVisitor(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader, PageBuilder pageBuilder)
+    PluginTask task;
+    Schema inputSchema;
+    Schema outputSchema;
+    PageReader pageReader;
+
+    AbstractGuardColumnVisitor(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader)
     {
         this.task = task;
         this.inputSchema = inputSchema;
         this.outputSchema = outputSchema;
         this.pageReader = pageReader;
-        this.pageBuilder = pageBuilder;
-
-        this.conditionMap = buildConditionMap(task, outputSchema);
     }
 
     static HashMap<String, List<Condition>> buildConditionMap(PluginTask task, Schema outputSchema)
@@ -44,7 +36,7 @@ abstract class AbstractColumnVisitor
             conditionMap.put(columnName, new ArrayList<Condition>());
         }
 
-        for (ConditionConfig conditionConfig : task.getConditions()) {
+        for (ConditionConfig conditionConfig : task.getConditions().get()) {
             String columnName = conditionConfig.getColumn();
             for (Column column : outputSchema.getColumns()) {
                 if (columnName.equals(column.getName())) {
@@ -58,5 +50,5 @@ abstract class AbstractColumnVisitor
         return conditionMap;
     }
 
-    abstract public boolean visitColumns(Schema schema);
+    abstract public boolean visitColumns(Schema inputSchema);
 }

data/src/main/java/org/embulk/filter/row/BuildColumnVisitorImpl.java

@@ -0,0 +1,98 @@
+package org.embulk.filter.row;
+
+import org.embulk.filter.row.RowFilterPlugin.PluginTask;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.Exec;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+
+import org.slf4j.Logger;
+
+public class BuildColumnVisitorImpl
+        implements ColumnVisitor
+{
+    private static final Logger logger = Exec.getLogger(RowFilterPlugin.class);
+    private final PluginTask task;
+    private final Schema inputSchema;
+    private final Schema outputSchema;
+    private final PageReader pageReader;
+    private final PageBuilder pageBuilder;
+
+    BuildColumnVisitorImpl(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader, PageBuilder pageBuilder)
+    {
+        this.task = task;
+        this.inputSchema = inputSchema;
+        this.outputSchema = outputSchema;
+        this.pageReader = pageReader;
+        this.pageBuilder = pageBuilder;
+    }
+
+    @Override
+    public void booleanColumn(Column column)
+    {
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setBoolean(column, pageReader.getBoolean(column));
+        }
+    }
+
+    @Override
+    public void longColumn(Column column)
+    {
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setLong(column, pageReader.getLong(column));
+        }
+    }
+
+    @Override
+    public void doubleColumn(Column column)
+    {
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setDouble(column, pageReader.getDouble(column));
+        }
+    }
+
+    @Override
+    public void stringColumn(Column column)
+    {
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setString(column, pageReader.getString(column));
+        }
+    }
+
+    @Override
+    public void timestampColumn(Column column)
+    {
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setTimestamp(column, pageReader.getTimestamp(column));
+        }
+    }
+
+    @Override
+    public void jsonColumn(Column column)
+    {
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setJson(column, pageReader.getJson(column));
+        }
+    }
+}

data/src/main/java/org/embulk/filter/row/{ColumnVisitorAndImpl.java → GuardColumnVisitorAndImpl.java}

@@ -9,32 +9,37 @@ import org.embulk.filter.row.condition.StringCondition;
 import org.embulk.filter.row.condition.TimestampCondition;
 
 import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.Exec;
-import org.embulk.spi.PageBuilder;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.time.Timestamp;
 
 import org.slf4j.Logger;
 
+import java.util.HashMap;
 import java.util.List;
 
-class ColumnVisitorAndImpl extends AbstractColumnVisitor
+class GuardColumnVisitorAndImpl
+        extends AbstractGuardColumnVisitor
+        implements ColumnVisitor
 {
     private static final Logger logger = Exec.getLogger(RowFilterPlugin.class);
     private boolean shouldAddRecord;
+    private HashMap<String, List<Condition>> conditionMap;
 
-    ColumnVisitorAndImpl(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader, PageBuilder pageBuilder)
+    GuardColumnVisitorAndImpl(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader)
     {
-        super(task, inputSchema, outputSchema, pageReader, pageBuilder);
+        super(task, inputSchema, outputSchema, pageReader);
+        this.conditionMap = buildConditionMap(task, outputSchema);
     }
 
-    public boolean visitColumns(Schema schema)
+    public boolean visitColumns(Schema inputSchema)
     {
         //Visitor objects are created for each thread :)
         //System.out.println(String.format("thread_id:%d object_id:%d", Thread.currentThread().getId(), this.hashCode()));
         shouldAddRecord = true;
-        for (Column column : schema.getColumns()) {
+        for (Column column : inputSchema.getColumns()) {
             column.visit(this);
         }
         return shouldAddRecord;
@@ -63,12 +68,6 @@ class ColumnVisitorAndImpl extends AbstractColumnVisitor
                 }
             }
         }
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
-        }
-        else {
-            pageBuilder.setBoolean(column, pageReader.getBoolean(column));
-        }
     }
 
     @Override
@@ -94,12 +93,6 @@ class ColumnVisitorAndImpl extends AbstractColumnVisitor
                 }
             }
         }
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
-        }
-        else {
-            pageBuilder.setLong(column, pageReader.getLong(column));
-        }
     }
 
     @Override
@@ -125,12 +118,6 @@ class ColumnVisitorAndImpl extends AbstractColumnVisitor
                 }
             }
         }
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
-        }
-        else {
-            pageBuilder.setDouble(column, pageReader.getDouble(column));
-        }
     }
 
     @Override
@@ -156,12 +143,6 @@ class ColumnVisitorAndImpl extends AbstractColumnVisitor
                 }
             }
         }
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
-        }
-        else {
-            pageBuilder.setString(column, pageReader.getString(column));
-        }
     }
 
     @Override
@@ -187,25 +168,10 @@ class ColumnVisitorAndImpl extends AbstractColumnVisitor
                 }
             }
         }
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
-        }
-        else {
-            pageBuilder.setTimestamp(column, pageReader.getTimestamp(column));
-        }
     }
 
     @Override
     public void jsonColumn(Column column)
     {
-        if (!shouldAddRecord) {
-            return;
-        }
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
-        }
-        else {
-            pageBuilder.setJson(column, pageReader.getJson(column));
-        }
     }
 }