embulk-filter-row 0.1.4 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 66c2553beb93fa0ec8460ac8645e62a9e54da8c9
-  data.tar.gz: 59bd429b89b936989ff80488b4c9d9caabfb3cf7
+  metadata.gz: 87ce06dbda2e17a52f4825208c79414b8938fb58
+  data.tar.gz: eb2bccca46662779c8e43041520b2a6fd2f4075e
 SHA512:
-  metadata.gz: cba7c85deadaefe03697785643f9612cb9e67de63895b773a05cff5e059c002731ef0abf096d1d71fbdc3d7bea5372cc56c71db7d572486c302f6e941a02bff1
-  data.tar.gz: 6a8d533156ea62bb4273ce9260cfcea67d16f210780b82c53248a82d982b02fbd0b2fca4d562a56d322b43bab2f4f9d196ff8e6a7ce894947eba24cf11c0182d
+  metadata.gz: 70708739971679cbed722a954e37e47390d9640dfdda35f291ce18ce44cd755fb125adc7a1b04b578ba0eb8a32be4fc41b701364fe3f421c47d9a17c69ea5b60
+  data.tar.gz: 25265ac26fab5c139930be8e7976a984466ee9e0bc8cae3343b1bbff94bac63d0a9fab74683132e8d6d8a68a2fc93831e9abbfae7cbcf5f207202b7ffefed979

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.2.0 (2015-12-05)
+
+Enhancements:
+
+* Support OR condition
+
 # 0.1.4
 
 Fixes:

data/README.md

@@ -6,7 +6,8 @@ A filter plugin for Embulk to filter out rows
 
 ## Configuration
 
-* **conditions**: select only rows which matches with conditions. (support only **AND** conditions)
+* **condition**: AND or OR (string, default: AND).
+* **conditions**: select only rows which matches with conditions.
   * **column**: column name (string, required)
   * **operator** operator (string, optional, default: ==)
     * boolean operator
@@ -35,11 +36,12 @@ A filter plugin for Embulk to filter out rows
 
 NOTE: column type is automatically retrieved from input data (inputSchema)
 
-## Example
+## Example (AND)
 
 ```yaml
 filters:
   - type: row
+    condition: AND
     conditions:
       - {column: foo,  operator: "IS NOT NULL"}
       - {column: id,   operator: ">=", argument: 10}
@@ -48,13 +50,41 @@ filters:
       - {column: time, operator: "==", argument: "2015-07-13", format: "%Y-%m-%d"}
 ```
 
-NOTE: column type is automatically retrieved from input data (inputSchema)
+## Example (OR)
+
+```yaml
+filters:
+  - type: row
+    condition: OR
+    conditions:
+      - {column: a, operator: "IS NOT NULL"}
+      - {column: b, operator: "IS NOT NULL"}
+```
+
+## Example (AND of OR)
+
+embulk-output-row does not directly supports complex conditions such as `((A OR B) AND (C OR D))`, but you should be able to express most of complex conditions by combining multiple filters like
+
+```yaml
+filters:
+  - type: row
+    condition: OR
+    conditions:
+      - {column: a, operator: "IS NOT NULL"}
+      - {column: b, operator: "IS NOT NULL"}
+  - type: row
+    condition: OR
+    conditions:
+      - {column: c, operator: "IS NOT NULL"}
+      - {column: d, operator: "IS NOT NULL"}
+```
+
+This is equivalent with `((A OR B) AND (C OR D))`.
 
-## ToDo
+## Not Supported:  More Complex Conditions
 
-* Support OR condition
-  * It should be better to think using Query engine like [Apache Drill](https://drill.apache.org/) or [Presto](https://prestodb.io/)
-  * With them, it is possible to send a query to local files, even to S3 files.
+* It should be better to think using Query engine like [Apache Drill](https://drill.apache.org/) or [Presto](https://prestodb.io/)
+* With them, it is possible to send a query to local files, even to S3 files.
 
 ## ChangeLog
 
@@ -66,7 +96,7 @@ Run example:
 
 ```
 $ ./gradlew classpath
-$ embulk run -I lib example.yml
+$ embulk run -I lib example/and.yml
 ```
 
 Run test:

data/build.gradle

@@ -12,7 +12,7 @@ configurations {
     provided
 }
 
-version = "0.1.4"
+version = "0.2.0"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/{example.yml → example/and.yml}

@@ -7,7 +7,7 @@
 #     score: integer
 in:
   type: file
-  path_prefix: example.csv
+  path_prefix: example/example.csv
   parser:
     type: csv
     charset: UTF-8

data/example/or.yml

@@ -0,0 +1,28 @@
+in:
+  type: file
+  path_prefix: example/example.csv
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: 'NULL'
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: time,  type: timestamp, format: "%Y-%m-%d"}
+      - {name: foo,   type: string}
+      - {name: bar,   type: string}
+      - {name: flag,  type: boolean}
+      - {name: id,    type: long}
+      - {name: name,  type: string}
+      - {name: score, type: double}
+filters:
+  - type: row
+    condition: OR
+    conditions:
+      - {column: name,  operator: ==, argument: "Vqjht6YEUBsMPXmoW1iOGFROZF27pBzz0TUkOKeDXEY"}
+      - {column: score, operator: ==, argument: 43}
+      - {column: id,    operator: ==, argument: 97}
+      - {column: flag,  operator: ==, argument: false}
+out:
+  type: stdout

data/src/main/java/org/embulk/filter/RowFilterPlugin.java

@@ -4,6 +4,7 @@ import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigException;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 
@@ -51,6 +52,10 @@ public class RowFilterPlugin implements FilterPlugin
 
     public interface PluginTask extends Task, TimestampParser.Task
     {
+        @Config("condition")
+        @ConfigDefault("\"AND\"")
+        public String getCondition();
+
         @Config("conditions")
         public List<ConditionConfig> getConditions();
     }
@@ -66,6 +71,11 @@ public class RowFilterPlugin implements FilterPlugin
             inputSchema.lookupColumn(columnName); // throw SchemaConfigException if not found
         }
 
+        String condition = task.getCondition().toLowerCase();
+        if (!condition.equals("or") && !condition.equals("and")) {
+            throw new ConfigException("condition must be either of \"or\" or \"and\".");
+        }
+
         Schema outputSchema = inputSchema;
         control.run(task.dump(), outputSchema);
     }
@@ -76,6 +86,8 @@ public class RowFilterPlugin implements FilterPlugin
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
 
+        final boolean orCondition = task.getCondition().toLowerCase().equals("or");
+
         final HashMap<String, List<Condition>> conditionMap = new HashMap<String, List<Condition>>();
         for (Column column : outputSchema.getColumns()) {
             String columnName = column.getName();
@@ -97,7 +109,8 @@ public class RowFilterPlugin implements FilterPlugin
         return new PageOutput() {
             private PageReader pageReader = new PageReader(inputSchema);
             private PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
-            private boolean shouldAddRecord = true;
+            private boolean shouldAddRecord;
+            private ColumnVisitor visitor = orCondition ? new ColumnVisitorOrImpl(pageBuilder) : new ColumnVisitorAndImpl(pageBuilder);
 
             @Override
             public void finish() {
@@ -113,18 +126,125 @@ public class RowFilterPlugin implements FilterPlugin
             public void add(Page page) {
                 pageReader.setPage(page);
 
-                ColumnVisitorImpl visitor = new ColumnVisitorImpl(pageBuilder);
                 while (pageReader.nextRecord()) {
-                    shouldAddRecord = true;
+                    shouldAddRecord = orCondition ? false : true;
                     inputSchema.visitColumns(visitor);
                     if (shouldAddRecord) pageBuilder.addRecord();
                 }
             }
 
-            class ColumnVisitorImpl implements ColumnVisitor {
+            class ColumnVisitorOrImpl implements ColumnVisitor {
+                private final PageBuilder pageBuilder;
+
+                ColumnVisitorOrImpl(PageBuilder pageBuilder) {
+                    this.pageBuilder = pageBuilder;
+                }
+
+                @Override
+                public void booleanColumn(Column column) {
+                    if (pageReader.isNull(column)) {
+                        pageBuilder.setNull(column);
+                    } else {
+                        pageBuilder.setBoolean(column, pageReader.getBoolean(column));
+                    }
+                    if (shouldAddRecord) return;
+                    List<Condition> conditionList = conditionMap.get(column.getName());
+                    for (Condition _condition : conditionList) {
+                        BooleanCondition condition = (BooleanCondition)_condition;
+                        if (pageReader.isNull(column)) {
+                            if (condition.compare(null)) { shouldAddRecord = true; break; }
+                        } else {
+                            boolean subject = pageReader.getBoolean(column);
+                            if (condition.compare(subject)) { shouldAddRecord = true; break; }
+                        }
+                    }
+                }
+
+                @Override
+                public void longColumn(Column column) {
+                    if (pageReader.isNull(column)) {
+                        pageBuilder.setNull(column);
+                    } else {
+                        pageBuilder.setLong(column, pageReader.getLong(column));
+                    }
+                    if (shouldAddRecord) return;
+                    List<Condition> conditionList = conditionMap.get(column.getName());
+                    for (Condition _condition : conditionList) {
+                        LongCondition condition = (LongCondition)_condition;
+                        if (pageReader.isNull(column)) {
+                            if (condition.compare(null)) { shouldAddRecord = true; break; }
+                        } else {
+                            long subject = pageReader.getLong(column);
+                            if (condition.compare(subject)) { shouldAddRecord = true; break; }
+                        }
+                    }
+                }
+
+                @Override
+                public void doubleColumn(Column column) {
+                    if (pageReader.isNull(column)) {
+                        pageBuilder.setNull(column);
+                    } else {
+                        pageBuilder.setDouble(column, pageReader.getDouble(column));
+                    }
+                    if (shouldAddRecord) return;
+                    List<Condition> conditionList = conditionMap.get(column.getName());
+                    for (Condition _condition : conditionList) {
+                        DoubleCondition condition = (DoubleCondition)_condition;
+                        if (pageReader.isNull(column)) {
+                            if (condition.compare(null)) { shouldAddRecord = true; break; }
+                        } else {
+                            double subject = pageReader.getDouble(column);
+                            if (condition.compare(subject)) { shouldAddRecord = true; break; }
+                        }
+                    }
+                }
+
+                @Override
+                public void stringColumn(Column column) {
+                    if (pageReader.isNull(column)) {
+                        pageBuilder.setNull(column);
+                    } else {
+                        pageBuilder.setString(column, pageReader.getString(column));
+                    }
+                    if (shouldAddRecord) return;
+                    List<Condition> conditionList = conditionMap.get(column.getName());
+                    for (Condition _condition : conditionList) {
+                        StringCondition condition = (StringCondition)_condition;
+                        if (pageReader.isNull(column)) {
+                            if (condition.compare(null)) { shouldAddRecord = true; break; }
+                        } else {
+                            String subject = pageReader.getString(column);
+                            if (condition.compare(subject)) { shouldAddRecord = true; break; }
+                        }
+                    }
+                }
+
+                @Override
+                public void timestampColumn(Column column) {
+                    if (pageReader.isNull(column)) {
+                        pageBuilder.setNull(column);
+                    } else {
+                        pageBuilder.setTimestamp(column, pageReader.getTimestamp(column));
+                    }
+                    if (shouldAddRecord) return;
+                    List<Condition> conditionList = conditionMap.get(column.getName());
+                    for (Condition _condition : conditionList) {
+                        TimestampCondition condition = (TimestampCondition)_condition;
+                        if (pageReader.isNull(column)) {
+                            if (condition.compare(null)) { shouldAddRecord = true; break; }
+                        } else {
+                            Timestamp subject = pageReader.getTimestamp(column);
+                            if (condition.compare(subject)) { shouldAddRecord = true; break; }
+                        }
+                    }
+                }
+            }
+
+            class ColumnVisitorAndImpl implements ColumnVisitor {
                 private final PageBuilder pageBuilder;
 
-                ColumnVisitorImpl(PageBuilder pageBuilder) {
+                ColumnVisitorAndImpl(PageBuilder pageBuilder) {
                     this.pageBuilder = pageBuilder;
                 }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-row
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-20 00:00:00.000000000 Z
+date: 2015-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -51,7 +51,8 @@ files:
 - LICENSE.txt
 - README.md
 - build.gradle
-- example.yml
+- example/and.yml
+- example/or.yml
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
@@ -73,7 +74,7 @@ files:
 - src/test/java/org/embulk/filter/row/TestLongCondition.java
 - src/test/java/org/embulk/filter/row/TestStringCondition.java
 - src/test/java/org/embulk/filter/row/TestTimestampCondition.java
-- classpath/embulk-filter-row-0.1.4.jar
+- classpath/embulk-filter-row-0.2.0.jar
 homepage: https://github.com/sonots/embulk-filter-row
 licenses:
 - MIT