embulk-filter-row 0.2.2 → 0.3.0

data/src/main/java/org/embulk/filter/row/{ColumnVisitorOrImpl.java → GuardColumnVisitorOrImpl.java}

@@ -9,32 +9,37 @@ import org.embulk.filter.row.condition.StringCondition;
 import org.embulk.filter.row.condition.TimestampCondition;
 
 import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.Exec;
-import org.embulk.spi.PageBuilder;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.time.Timestamp;
 
 import org.slf4j.Logger;
 
+import java.util.HashMap;
 import java.util.List;
 
-class ColumnVisitorOrImpl extends AbstractColumnVisitor
+class GuardColumnVisitorOrImpl
+        extends AbstractGuardColumnVisitor
+        implements ColumnVisitor
 {
     private static final Logger logger = Exec.getLogger(RowFilterPlugin.class);
     private boolean shouldAddRecord;
+    private HashMap<String, List<Condition>> conditionMap;
 
-    ColumnVisitorOrImpl(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader, PageBuilder pageBuilder)
+    GuardColumnVisitorOrImpl(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader)
     {
-        super(task, inputSchema, outputSchema, pageReader, pageBuilder);
+        super(task, inputSchema, outputSchema, pageReader);
+        this.conditionMap = buildConditionMap(task, outputSchema);
     }
 
-    public boolean visitColumns(Schema schema)
+    public boolean visitColumns(Schema inputSchema)
     {
         //Visitor objects are created for each thread :)
         //System.out.println(String.format("thread_id:%d object_id:%d", Thread.currentThread().getId(), this.hashCode()));
         shouldAddRecord = false;
-        for (Column column : schema.getColumns()) {
+        for (Column column : inputSchema.getColumns()) {
             column.visit(this);
         }
         return shouldAddRecord;
@@ -43,12 +48,6 @@ class ColumnVisitorOrImpl extends AbstractColumnVisitor
     @Override
     public void booleanColumn(Column column)
     {
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
-        }
-        else {
-            pageBuilder.setBoolean(column, pageReader.getBoolean(column));
-        }
         if (shouldAddRecord) {
             return;
         }
@@ -74,12 +73,6 @@ class ColumnVisitorOrImpl extends AbstractColumnVisitor
     @Override
     public void longColumn(Column column)
     {
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
-        }
-        else {
-            pageBuilder.setLong(column, pageReader.getLong(column));
-        }
         if (shouldAddRecord) {
             return;
         }
@@ -105,12 +98,6 @@ class ColumnVisitorOrImpl extends AbstractColumnVisitor
     @Override
     public void doubleColumn(Column column)
     {
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
-        }
-        else {
-            pageBuilder.setDouble(column, pageReader.getDouble(column));
-        }
         if (shouldAddRecord) {
             return;
         }
@@ -136,12 +123,6 @@ class ColumnVisitorOrImpl extends AbstractColumnVisitor
     @Override
     public void stringColumn(Column column)
     {
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
-        }
-        else {
-            pageBuilder.setString(column, pageReader.getString(column));
-        }
         if (shouldAddRecord) {
             return;
         }
@@ -167,12 +148,6 @@ class ColumnVisitorOrImpl extends AbstractColumnVisitor
     @Override
     public void timestampColumn(Column column)
     {
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
-        }
-        else {
-            pageBuilder.setTimestamp(column, pageReader.getTimestamp(column));
-        }
         if (shouldAddRecord) {
             return;
         }
@@ -198,14 +173,5 @@ class ColumnVisitorOrImpl extends AbstractColumnVisitor
     @Override
     public void jsonColumn(Column column)
     {
-        if (!shouldAddRecord) {
-            return;
-        }
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
-        }
-        else {
-            pageBuilder.setJson(column, pageReader.getJson(column));
-        }
     }
 }

data/src/main/java/org/embulk/filter/row/GuardColumnVisitorWhereImpl.java

@@ -0,0 +1,28 @@
+package org.embulk.filter.row;
+
+import org.embulk.filter.row.RowFilterPlugin.PluginTask;
+
+import org.embulk.filter.row.where.ParserExp;
+import org.embulk.spi.Exec;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+
+import org.slf4j.Logger;
+
+class GuardColumnVisitorWhereImpl
+        extends AbstractGuardColumnVisitor
+{
+    private static final Logger logger = Exec.getLogger(RowFilterPlugin.class);
+    ParserExp parserExp;
+
+    GuardColumnVisitorWhereImpl(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader, ParserExp parserExp)
+    {
+        super(task, inputSchema, outputSchema, pageReader);
+        this.parserExp = parserExp;
+    }
+
+    public boolean visitColumns(Schema inputSchema)
+    {
+        return parserExp.eval(pageReader);
+    }
+}

data/src/main/java/org/embulk/filter/row/RowFilterPlugin.java

@@ -1,5 +1,7 @@
 package org.embulk.filter.row;
 
+import com.google.common.base.Optional;
+
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigException;
@@ -8,8 +10,10 @@ import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 
 import org.embulk.filter.row.condition.ConditionConfig;
+import org.embulk.filter.row.where.Parser;
+import org.embulk.filter.row.where.ParserExp;
 
-import org.embulk.spi.ColumnVisitor;
+import org.embulk.filter.row.where.ParserLiteral;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FilterPlugin;
 import org.embulk.spi.Page;
@@ -26,6 +30,7 @@ import java.util.List;
 public class RowFilterPlugin implements FilterPlugin
 {
     private static final Logger logger = Exec.getLogger(RowFilterPlugin.class);
+    private ParserExp parserExp = null;
 
     public RowFilterPlugin() {}
 
@@ -36,7 +41,12 @@ public class RowFilterPlugin implements FilterPlugin
         public String getCondition();
 
         @Config("conditions")
-        public List<ConditionConfig> getConditions();
+        @ConfigDefault("null")
+        public Optional<List<ConditionConfig>> getConditions();
+
+        @Config("where")
+        @ConfigDefault("null")
+        public Optional<String> getWhere();
     }
 
     @Override
@@ -44,6 +54,7 @@ public class RowFilterPlugin implements FilterPlugin
             FilterPlugin.Control control)
     {
         PluginTask task = config.loadConfig(PluginTask.class);
+        ParserLiteral.setJRuby(task.getJRuby());
 
         configure(task, inputSchema);
         Schema outputSchema = inputSchema;
@@ -53,14 +64,24 @@ public class RowFilterPlugin implements FilterPlugin
 
     void configure(PluginTask task, Schema inputSchema) throws ConfigException
     {
-        for (ConditionConfig conditionConfig : task.getConditions()) {
-            String columnName = conditionConfig.getColumn();
-            inputSchema.lookupColumn(columnName); // throw SchemaConfigException if not found
-        }
+        if (task.getConditions().isPresent()) {
+            for (ConditionConfig conditionConfig : task.getConditions().get()) {
+                String columnName = conditionConfig.getColumn();
+                inputSchema.lookupColumn(columnName); // throw SchemaConfigException if not found
+            }
 
-        String condition = task.getCondition().toLowerCase();
-        if (!condition.equals("or") && !condition.equals("and")) {
-            throw new ConfigException("condition must be either of \"or\" or \"and\".");
+            String condition = task.getCondition().toLowerCase();
+            if (!condition.equals("or") && !condition.equals("and")) {
+                throw new ConfigException("condition must be either of \"or\" or \"and\".");
+            }
+        }
+        else if (task.getWhere().isPresent()) {
+            String where = task.getWhere().get();
+            Parser parser = new Parser(inputSchema);
+            parserExp = parser.parse(where); // throw ConfigException if something wrong
+        }
+        else {
+            throw new ConfigException("Either of `conditions` or `where` must be set.");
         }
     }
 
@@ -70,14 +91,24 @@ public class RowFilterPlugin implements FilterPlugin
     {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
         final boolean orCondition = task.getCondition().toLowerCase().equals("or");
+        final PageReader pageReader = new PageReader(inputSchema);
+        final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
 
-        return new PageOutput() {
-            private PageReader pageReader = new PageReader(inputSchema);
-            private PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
-            private AbstractColumnVisitor visitor = orCondition ?
-                    new ColumnVisitorOrImpl(task, inputSchema, outputSchema, pageReader, pageBuilder) :
-                    new ColumnVisitorAndImpl(task, inputSchema, outputSchema, pageReader, pageBuilder);
+        final AbstractGuardColumnVisitor guradVisitor;
+        if (task.getWhere().isPresent()) {
+            guradVisitor = new GuardColumnVisitorWhereImpl(task, inputSchema, outputSchema, pageReader, parserExp);
+        }
+        else if (orCondition) {
+            guradVisitor = new GuardColumnVisitorOrImpl(task, inputSchema, outputSchema, pageReader);
+        }
+        else {
+            guradVisitor = new GuardColumnVisitorAndImpl(task, inputSchema, outputSchema, pageReader);
+        }
+
+        final BuildColumnVisitorImpl buildVisitor;
+        buildVisitor = new BuildColumnVisitorImpl(task, inputSchema, outputSchema, pageReader, pageBuilder);
 
+        return new PageOutput() {
             @Override
             public void finish()
             {
@@ -96,7 +127,9 @@ public class RowFilterPlugin implements FilterPlugin
                 pageReader.setPage(page);
 
                 while (pageReader.nextRecord()) {
-                    if (visitor.visitColumns(inputSchema)) {
+                    if (guradVisitor.visitColumns(inputSchema)) {
+                        // output.add(page); did not work, double release() error occurred. We need to copy from reader to builder...
+                        outputSchema.visitColumns(buildVisitor);
                         pageBuilder.addRecord();
                     }
                 }