embulk-filter-row 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dda5fc00e1df4e66e7c07fb98ffe1ed396c61544
-  data.tar.gz: 515413bd3a2c4dbe123a4a00bbd079fceadf9eda
+  metadata.gz: e658f035d50a57d97df3a874a87754e0eafc3114
+  data.tar.gz: 505d52511a7527b4bab0585a075548af27ed0c7e
 SHA512:
-  metadata.gz: aae0ff1567c1010542d9bcdc8c625d5e7679449254600552534dd3f3348cb719430a2ba0069587a3ff2b0d7c007ee3568f78fc5d72562eb06ff5ea3d0290ff45
-  data.tar.gz: 200a1d796398224163afe1417b96820e1e21c563a4029c1d0a36981ed88467b9509abd64429e8d144a2e6598fced1269204966989cc0284739352f8e6f884343
+  metadata.gz: c0943ff3946f42199d898326419677f74d2b162a7d515b3ce6f6847b162b08e4469af946e42ddc61c50a4c0b3c6ca2b35530bc880a5531d60e3102c873c066a6
+  data.tar.gz: e21c28850e48e2d9d816ca8922e2ae7019cb2828976b99807f9f063190f492f89397e32b6648b2aeab4d0e20e08e3b66b987f66d606c121691c06c1be766a341

data/.gitignore

@@ -9,3 +9,4 @@ build/
 *.csv
 .tags
 .ruby-version
+*.iml

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.2.2 (2016-08-05)
+
+Enhancements:
+
+* Support `type: json` (not supporting filtering with json yet, though)
+
 # 0.2.1 (2016-06-27)
 
 Enhancements:

data/README.md

@@ -96,7 +96,7 @@ Run example:
 
 ```
 $ ./gradlew classpath
-$ embulk run -I lib example/and.yml
+$ embulk preview -I lib example/example.yml
 ```
 
 Run test:

data/build.gradle

@@ -13,13 +13,13 @@ configurations {
     provided
 }
 
-version = "0.2.1"
+version = "0.2.2"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.7.0"
-    provided "org.embulk:embulk-core:0.7.0"
+    compile  "org.embulk:embulk-core:0.8.11"
+    provided "org.embulk:embulk-core:0.8.11"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
     testCompile "junit:junit:4.+"
 }

data/example/and.yml

@@ -12,7 +12,7 @@ in:
     type: csv
     charset: UTF-8
     newline: CRLF
-    null_string: 'NULL'
+    null_string: "NULL"
     skip_header_lines: 1
     comment_line_marker: '#'
     columns:
@@ -22,17 +22,16 @@ in:
       - {name: flag,  type: boolean}
       - {name: id,    type: long}
       - {name: name,  type: string}
+      - {name: json,  type: json}
       - {name: score, type: double}
 filters:
   - type: row
     conditions:
-      - {column: flag,  operator: ==,  argument: true}
-      - {column: foo,   operator: "IS NULL"}
-      - {column: id,    operator: ==,  argument: 97}
-      - {column: name,  operator: ==,  argument: "xxxx", not: true}
+      - {column: time,  operator: ==,  argument: "2015-07-13", format: "%Y-%m-%d"}
+      - {column: foo,   operator: "IS NULL" }
+      - {column: bar,   operator: ==,  argument: "bar"}
       - {column: score, operator: ">", argument: 2000}
       - {column: score, operator: "<", argument: 6000}
-      - {column: time,  operator: ==,  argument: "2015-07-13", format: "%Y-%m-%d"}
       # - {column: missing, operator: "==", argument: 6000}
 out:
   type: stdout

data/example/example.yml

@@ -0,0 +1,37 @@
+# in:
+#   type: random
+#   rows: 100
+#   schema:
+#     id: primary_key
+#     name: string
+#     score: integer
+in:
+  type: file
+  path_prefix: example/example.csv
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: "NULL"
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: time,  type: timestamp, format: "%Y-%m-%d"}
+      - {name: foo,   type: string}
+      - {name: bar,   type: string}
+      - {name: flag,  type: boolean}
+      - {name: id,    type: long}
+      - {name: name,  type: string}
+      - {name: json,  type: json}
+      - {name: score, type: double}
+filters:
+  - type: row
+    conditions:
+      - {column: time,  operator: ==,  argument: "2015-07-13", format: "%Y-%m-%d"}
+      - {column: foo,   operator: "IS NULL" }
+      - {column: bar,   operator: ==,  argument: "bar"}
+      - {column: score, operator: ">", argument: 2000}
+      - {column: score, operator: "<", argument: 6000}
+      # - {column: missing, operator: "==", argument: 6000}
+out:
+  type: stdout

data/example/or.yml

@@ -15,6 +15,7 @@ in:
       - {name: flag,  type: boolean}
       - {name: id,    type: long}
       - {name: name,  type: string}
+      - {name: json,  type: json}
       - {name: score, type: double}
 filters:
   - type: row
@@ -22,7 +23,7 @@ filters:
     conditions:
       - {column: name,  operator: ==, argument: "Vqjht6YEUBsMPXmoW1iOGFROZF27pBzz0TUkOKeDXEY"}
       - {column: score, operator: ==, argument: 43}
-      - {column: id,    operator: ==, argument: 97}
+      - {column: id,    operator: ==, argument: 9}
       - {column: flag,  operator: ==, argument: false}
 out:
   type: stdout

data/lib/embulk/filter/row.rb

@@ -1,3 +1,3 @@
 Embulk::JavaPlugin.register_filter(
-  "row", "org.embulk.filter.RowFilterPlugin",
+  "row", "org.embulk.filter.row.RowFilterPlugin",
   File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/filter/row/AbstractColumnVisitor.java

@@ -0,0 +1,62 @@
+package org.embulk.filter.row;
+
+import org.embulk.filter.row.RowFilterPlugin.PluginTask;
+import org.embulk.filter.row.condition.Condition;
+
+import org.embulk.filter.row.condition.ConditionConfig;
+import org.embulk.filter.row.condition.ConditionFactory;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+abstract class AbstractColumnVisitor
+        implements ColumnVisitor
+{
+    final PluginTask task;
+    final Schema inputSchema;
+    final Schema outputSchema;
+    final PageReader pageReader;
+    final PageBuilder pageBuilder;
+    HashMap<String, List<Condition>> conditionMap;
+
+    AbstractColumnVisitor(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader, PageBuilder pageBuilder)
+    {
+        this.task = task;
+        this.inputSchema = inputSchema;
+        this.outputSchema = outputSchema;
+        this.pageReader = pageReader;
+        this.pageBuilder = pageBuilder;
+
+        this.conditionMap = buildConditionMap(task, outputSchema);
+    }
+
+    static HashMap<String, List<Condition>> buildConditionMap(PluginTask task, Schema outputSchema)
+    {
+        HashMap<String, List<Condition>> conditionMap = new HashMap<>();
+        for (Column column : outputSchema.getColumns()) {
+            String columnName = column.getName();
+            conditionMap.put(columnName, new ArrayList<Condition>());
+        }
+
+        for (ConditionConfig conditionConfig : task.getConditions()) {
+            String columnName = conditionConfig.getColumn();
+            for (Column column : outputSchema.getColumns()) {
+                if (columnName.equals(column.getName())) {
+                    ConditionFactory factory = new ConditionFactory(task.getJRuby(), column, conditionConfig);
+                    Condition condition = factory.createCondition();
+                    conditionMap.get(columnName).add(condition);
+                    break;
+                }
+            }
+        }
+        return conditionMap;
+    }
+
+    abstract public boolean visitColumns(Schema schema);
+}

data/src/main/java/org/embulk/filter/row/ColumnVisitorAndImpl.java

@@ -0,0 +1,211 @@
+package org.embulk.filter.row;
+
+import org.embulk.filter.row.RowFilterPlugin.PluginTask;
+import org.embulk.filter.row.condition.BooleanCondition;
+import org.embulk.filter.row.condition.Condition;
+import org.embulk.filter.row.condition.DoubleCondition;
+import org.embulk.filter.row.condition.LongCondition;
+import org.embulk.filter.row.condition.StringCondition;
+import org.embulk.filter.row.condition.TimestampCondition;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+import org.embulk.spi.time.Timestamp;
+
+import org.slf4j.Logger;
+
+import java.util.List;
+
+class ColumnVisitorAndImpl extends AbstractColumnVisitor
+{
+    private static final Logger logger = Exec.getLogger(RowFilterPlugin.class);
+    private boolean shouldAddRecord;
+
+    ColumnVisitorAndImpl(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader, PageBuilder pageBuilder)
+    {
+        super(task, inputSchema, outputSchema, pageReader, pageBuilder);
+    }
+
+    public boolean visitColumns(Schema schema)
+    {
+        //Visitor objects are created for each thread :)
+        //System.out.println(String.format("thread_id:%d object_id:%d", Thread.currentThread().getId(), this.hashCode()));
+        shouldAddRecord = true;
+        for (Column column : schema.getColumns()) {
+            column.visit(this);
+        }
+        return shouldAddRecord;
+    }
+
+    @Override
+    public void booleanColumn(Column column)
+    {
+        if (!shouldAddRecord) {
+            return;
+        }
+        List<Condition> conditionList = conditionMap.get(column.getName());
+        for (Condition tempCondition : conditionList) {
+            BooleanCondition condition = (BooleanCondition) tempCondition;
+            if (pageReader.isNull(column)) {
+                if (!condition.compare(null)) {
+                    shouldAddRecord = false;
+                    break;
+                }
+            }
+            else {
+                boolean subject = pageReader.getBoolean(column);
+                if (!condition.compare(subject)) {
+                    shouldAddRecord = false;
+                    break;
+                }
+            }
+        }
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setBoolean(column, pageReader.getBoolean(column));
+        }
+    }
+
+    @Override
+    public void longColumn(Column column)
+    {
+        if (!shouldAddRecord) {
+            return;
+        }
+        List<Condition> conditionList = conditionMap.get(column.getName());
+        for (Condition tempCondition : conditionList) {
+            LongCondition condition = (LongCondition) tempCondition;
+            if (pageReader.isNull(column)) {
+                if (!condition.compare(null)) {
+                    shouldAddRecord = false;
+                    break;
+                }
+            }
+            else {
+                long subject = pageReader.getLong(column);
+                if (!condition.compare(subject)) {
+                    shouldAddRecord = false;
+                    break;
+                }
+            }
+        }
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setLong(column, pageReader.getLong(column));
+        }
+    }
+
+    @Override
+    public void doubleColumn(Column column)
+    {
+        if (!shouldAddRecord) {
+            return;
+        }
+        List<Condition> conditionList = conditionMap.get(column.getName());
+        for (Condition tempCondition : conditionList) {
+            DoubleCondition condition = (DoubleCondition) tempCondition;
+            if (pageReader.isNull(column)) {
+                if (!condition.compare(null)) {
+                    shouldAddRecord = false;
+                    break;
+                }
+            }
+            else {
+                double subject = pageReader.getDouble(column);
+                if (!condition.compare(subject)) {
+                    shouldAddRecord = false;
+                    break;
+                }
+            }
+        }
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setDouble(column, pageReader.getDouble(column));
+        }
+    }
+
+    @Override
+    public void stringColumn(Column column)
+    {
+        if (!shouldAddRecord) {
+            return;
+        }
+        List<Condition> conditionList = conditionMap.get(column.getName());
+        for (Condition tempCondition : conditionList) {
+            StringCondition condition = (StringCondition) tempCondition;
+            if (pageReader.isNull(column)) {
+                if (!condition.compare(null)) {
+                    shouldAddRecord = false;
+                    break;
+                }
+            }
+            else {
+                String subject = pageReader.getString(column);
+                if (!condition.compare(subject)) {
+                    shouldAddRecord = false;
+                    break;
+                }
+            }
+        }
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setString(column, pageReader.getString(column));
+        }
+    }
+
+    @Override
+    public void timestampColumn(Column column)
+    {
+        if (!shouldAddRecord) {
+            return;
+        }
+        List<Condition> conditionList = conditionMap.get(column.getName());
+        for (Condition tempCondition : conditionList) {
+            TimestampCondition condition = (TimestampCondition) tempCondition;
+            if (pageReader.isNull(column)) {
+                if (!condition.compare(null)) {
+                    shouldAddRecord = false;
+                    break;
+                }
+            }
+            else {
+                Timestamp subject = pageReader.getTimestamp(column);
+                if (!condition.compare(subject)) {
+                    shouldAddRecord = false;
+                    break;
+                }
+            }
+        }
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setTimestamp(column, pageReader.getTimestamp(column));
+        }
+    }
+
+    @Override
+    public void jsonColumn(Column column)
+    {
+        if (!shouldAddRecord) {
+            return;
+        }
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setJson(column, pageReader.getJson(column));
+        }
+    }
+}

data/src/main/java/org/embulk/filter/row/ColumnVisitorOrImpl.java

@@ -0,0 +1,211 @@
+package org.embulk.filter.row;
+
+import org.embulk.filter.row.RowFilterPlugin.PluginTask;
+import org.embulk.filter.row.condition.BooleanCondition;
+import org.embulk.filter.row.condition.Condition;
+import org.embulk.filter.row.condition.DoubleCondition;
+import org.embulk.filter.row.condition.LongCondition;
+import org.embulk.filter.row.condition.StringCondition;
+import org.embulk.filter.row.condition.TimestampCondition;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+import org.embulk.spi.time.Timestamp;
+
+import org.slf4j.Logger;
+
+import java.util.List;
+
+class ColumnVisitorOrImpl extends AbstractColumnVisitor
+{
+    private static final Logger logger = Exec.getLogger(RowFilterPlugin.class);
+    private boolean shouldAddRecord;
+
+    ColumnVisitorOrImpl(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader, PageBuilder pageBuilder)
+    {
+        super(task, inputSchema, outputSchema, pageReader, pageBuilder);
+    }
+
+    public boolean visitColumns(Schema schema)
+    {
+        //Visitor objects are created for each thread :)
+        //System.out.println(String.format("thread_id:%d object_id:%d", Thread.currentThread().getId(), this.hashCode()));
+        shouldAddRecord = false;
+        for (Column column : schema.getColumns()) {
+            column.visit(this);
+        }
+        return shouldAddRecord;
+    }
+
+    @Override
+    public void booleanColumn(Column column)
+    {
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setBoolean(column, pageReader.getBoolean(column));
+        }
+        if (shouldAddRecord) {
+            return;
+        }
+        List<Condition> conditionList = conditionMap.get(column.getName());
+        for (Condition tempCondition : conditionList) {
+            BooleanCondition condition = (BooleanCondition) tempCondition;
+            if (pageReader.isNull(column)) {
+                if (condition.compare(null)) {
+                    shouldAddRecord = true;
+                    break;
+                }
+            }
+            else {
+                boolean subject = pageReader.getBoolean(column);
+                if (condition.compare(subject)) {
+                    shouldAddRecord = true;
+                    break;
+                }
+            }
+        }
+    }
+
+    @Override
+    public void longColumn(Column column)
+    {
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setLong(column, pageReader.getLong(column));
+        }
+        if (shouldAddRecord) {
+            return;
+        }
+        List<Condition> conditionList = conditionMap.get(column.getName());
+        for (Condition tempCondition : conditionList) {
+            LongCondition condition = (LongCondition) tempCondition;
+            if (pageReader.isNull(column)) {
+                if (condition.compare(null)) {
+                    shouldAddRecord = true;
+                    break;
+                }
+            }
+            else {
+                long subject = pageReader.getLong(column);
+                if (condition.compare(subject)) {
+                    shouldAddRecord = true;
+                    break;
+                }
+            }
+        }
+    }
+
+    @Override
+    public void doubleColumn(Column column)
+    {
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setDouble(column, pageReader.getDouble(column));
+        }
+        if (shouldAddRecord) {
+            return;
+        }
+        List<Condition> conditionList = conditionMap.get(column.getName());
+        for (Condition tempCondition : conditionList) {
+            DoubleCondition condition = (DoubleCondition) tempCondition;
+            if (pageReader.isNull(column)) {
+                if (condition.compare(null)) {
+                    shouldAddRecord = true;
+                    break;
+                }
+            }
+            else {
+                double subject = pageReader.getDouble(column);
+                if (condition.compare(subject)) {
+                    shouldAddRecord = true;
+                    break;
+                }
+            }
+        }
+    }
+
+    @Override
+    public void stringColumn(Column column)
+    {
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setString(column, pageReader.getString(column));
+        }
+        if (shouldAddRecord) {
+            return;
+        }
+        List<Condition> conditionList = conditionMap.get(column.getName());
+        for (Condition tempCondition : conditionList) {
+            StringCondition condition = (StringCondition) tempCondition;
+            if (pageReader.isNull(column)) {
+                if (condition.compare(null)) {
+                    shouldAddRecord = true;
+                    break;
+                }
+            }
+            else {
+                String subject = pageReader.getString(column);
+                if (condition.compare(subject)) {
+                    shouldAddRecord = true;
+                    break;
+                }
+            }
+        }
+    }
+
+    @Override
+    public void timestampColumn(Column column)
+    {
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setTimestamp(column, pageReader.getTimestamp(column));
+        }
+        if (shouldAddRecord) {
+            return;
+        }
+        List<Condition> conditionList = conditionMap.get(column.getName());
+        for (Condition tempCondition : conditionList) {
+            TimestampCondition condition = (TimestampCondition) tempCondition;
+            if (pageReader.isNull(column)) {
+                if (condition.compare(null)) {
+                    shouldAddRecord = true;
+                    break;
+                }
+            }
+            else {
+                Timestamp subject = pageReader.getTimestamp(column);
+                if (condition.compare(subject)) {
+                    shouldAddRecord = true;
+                    break;
+                }
+            }
+        }
+    }
+
+    @Override
+    public void jsonColumn(Column column)
+    {
+        if (!shouldAddRecord) {
+            return;
+        }
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setJson(column, pageReader.getJson(column));
+        }
+    }
+}