embulk-filter-column 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7e3f1644fcdf2eee67ab2101e46eb5f546b9c373
-  data.tar.gz: 825008e7b6b9c4b1c3ce5da34e42fd055b627148
+  metadata.gz: d4b22fc2b07962eb1295a87ad76ecf7d373e8244
+  data.tar.gz: 4c81bf7c05f93f544c883adf165751108a62c7bd
 SHA512:
-  metadata.gz: 4823b5a3e833c0ae0ae6302b7043027ce7e9b8740910c9b1753d312540a01d22e32dfe42b7af65cdb5227173d988255336bcc73d2bc1a4239ca6044d62c164c6
-  data.tar.gz: 78fa9b4a98d493a292e65e7e3387db08d997683937dfd594a69d45aecc8304d129ded9e3f3dfa409d6591323452a1dcd00389a30f0419e7794276145bffaa9c0
+  metadata.gz: cc307a1b813dfd435d8d94c60f44886410546dd1065170218cc8c331930b940d578ed99a92f38f705566b73dc2c08878456762a3924cb8b5adf39e8144bdbd83
+  data.tar.gz: a304d053cc7cf08b562afc84dc663909114b6c73da262fb4be2b82b9c58f17975014ad0d59e3a4d383f30e6f636fae4873f96a704f65dbf962f684eb84eebb74

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.1.5
+
+Enhancements:
+
+* Support timestamp default
+
 # 0.1.4
 
 Enhancements:

data/README.md

@@ -7,6 +7,10 @@ A filter plugin for Embulk to filter out columns
 - **columns**: columns (array of hash, required)
   - **name**: name of column
   - **default**: default value used if input is null
+  - **format**: special option for timestamp column, specify the format of timestamp default (string, default is `%Y-%m-%d %H:%M:%S.%N %z`)
+  - **timezone**: special option for timestamp column, specify the timezone of timestamp default (string, default is `UTC`)
+
+NOTE: column type is automatically retrieved from input data (inputSchema)
 
 ## Example
 
@@ -14,11 +18,12 @@ A filter plugin for Embulk to filter out columns
 filters:
   - type: column
     columns:
+      - {name: time, default: "2015-07-13", format: "%Y-%m-%d"}
       - {name: id}
-      - {name: name, default: 'foo'}
+      - {name: name, default: "foo"}
 ```
 
-reduces columns to only `id` and `name` columns.
+reduces columns to only `time`, `id`, and `name` columns.
 
 ## Development
 
@@ -29,6 +34,12 @@ $ ./gradlew classpath
 $ embulk run -I lib example.yml
 ```
 
+Run test:
+
+```
+$ ./gradew test
+```
+
 Release gem:
 
 ```

data/build.gradle

@@ -12,11 +12,11 @@ configurations {
     provided
 }
 
-version = "0.1.4"
+version = "0.1.5"
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.6.12"
-    provided "org.embulk:embulk-core:0.6.12"
+    compile  "org.embulk:embulk-core:0.6.16"
+    provided "org.embulk:embulk-core:0.6.16"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
     testCompile "junit:junit:4.+"
 }

data/example.yml

@@ -16,12 +16,14 @@ in:
     skip_header_lines: 1
     comment_line_marker: '#'
     columns:
+      - {name: time,  type: timestamp, format: "%Y-%m-%d"}
       - {name: id, type: long}
       - {name: name, type: string}
       - {name: score, type: double}
 filters:
   - type: column
     columns:
+      - {name: time, default: "2015-07-13", format: "%Y-%m-%d"}
       - {name: name, default: "foo"}
       - {name: foo,  default: 1}
       - {name: id}

data/src/main/java/org/embulk/filter/ColumnFilterPlugin.java

@@ -1,5 +1,10 @@
 package org.embulk.filter;
 
+import java.util.List;
+import java.util.HashMap;
+import com.google.common.collect.ImmutableList;
+import org.slf4j.Logger;
+
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
@@ -7,11 +12,12 @@ import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 
-import java.util.List;
-import java.util.HashMap;
 import org.embulk.spi.type.Type;
+import org.embulk.spi.type.BooleanType;
+import org.embulk.spi.type.LongType;
+import org.embulk.spi.type.DoubleType;
+import org.embulk.spi.type.StringType;
 import org.embulk.spi.type.TimestampType;
-import com.google.common.collect.ImmutableList;
 
 import org.embulk.spi.FilterPlugin;
 import org.embulk.spi.Exec;
@@ -25,9 +31,16 @@ import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.filter.column.ColumnConfig; // note: different with spi.ColumnConfig
 
+import org.joda.time.DateTimeZone;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.time.TimestampParseException;
+import com.google.common.base.Throwables;
+
 public class ColumnFilterPlugin implements FilterPlugin
 {
-    public interface PluginTask extends Task
+    public interface PluginTask
+            extends Task, TimestampParser.Task
     {
         @Config("columns")
         public List<ColumnConfig> getColumns();
@@ -58,12 +71,20 @@ public class ColumnFilterPlugin implements FilterPlugin
         control.run(task.dump(), outputSchema);
     }
 
+    private final Logger log;
+
+    public ColumnFilterPlugin()
+    {
+        log = Exec.getLogger(ColumnFilterPlugin.class);
+    }
+
     @Override
     public PageOutput open(TaskSource taskSource, Schema inputSchema,
             Schema outputSchema, PageOutput output)
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
 
+        // Map outputColumn => inputColumn
         HashMap<Column, Column> outputInputColumnMap = new HashMap<Column, Column>();
         for (Column outputColumn: outputSchema.getColumns()) {
             for (Column inputColumn: inputSchema.getColumns()) {
@@ -74,12 +95,52 @@ public class ColumnFilterPlugin implements FilterPlugin
             }
         }
 
-        HashMap<Column, ColumnConfig> outputColumnConfigMap = new HashMap<Column, ColumnConfig>();
+        // Map outputColumn => default value if present
+        HashMap<Column, Object> outputDefaultMap = new HashMap<Column, Object>();
         for (Column outputColumn: outputSchema.getColumns()) {
+            Type columnType = outputColumn.getType();
+
             for (ColumnConfig columnConfig : task.getColumns()) {
                 if (columnConfig.getName().equals(outputColumn.getName())) {
-                    outputColumnConfigMap.put(outputColumn, columnConfig);
-                    break;
+
+                    if (columnType instanceof BooleanType) {
+                        if (columnConfig.getDefault().isPresent()) {
+                            Boolean default_value = (Boolean)columnConfig.getDefault().get();
+                            outputDefaultMap.put(outputColumn, default_value);
+                        }
+                    }
+                    else if (columnType instanceof LongType) {
+                        if (columnConfig.getDefault().isPresent()) {
+                            Long default_value = new Long(columnConfig.getDefault().get().toString());
+                            outputDefaultMap.put(outputColumn, default_value);
+                        }
+                    }
+                    else if (columnType instanceof DoubleType) {
+                        if (columnConfig.getDefault().isPresent()) {
+                            Double default_value = new Double(columnConfig.getDefault().get().toString());
+                            outputDefaultMap.put(outputColumn, default_value);
+                        }
+                    }
+                    else if (columnType instanceof StringType) {
+                        if (columnConfig.getDefault().isPresent()) {
+                            String default_value = (String)columnConfig.getDefault().get();
+                            outputDefaultMap.put(outputColumn, default_value);
+                        }
+                    }
+                    else if (columnType instanceof TimestampType) {
+                        if (columnConfig.getDefault().isPresent()) {
+                            String time            = (String)columnConfig.getDefault().get();
+                            String format          = (String)columnConfig.getFormat().get();
+                            DateTimeZone timezone  = DateTimeZone.forID((String)columnConfig.getTimezone().get());
+                            TimestampParser parser = new TimestampParser(task.getJRuby(), format, timezone);
+                            try {
+                                Timestamp default_value = parser.parse(time);
+                                outputDefaultMap.put(outputColumn, default_value);
+                            } catch(TimestampParseException ex) {
+                                throw Throwables.propagate(ex);
+                            }
+                        }
+                    }
                 }
             }
         }
@@ -120,11 +181,9 @@ public class ColumnFilterPlugin implements FilterPlugin
                 public void booleanColumn(Column outputColumn) {
                     Column inputColumn = outputInputColumnMap.get(outputColumn);
                     if (pageReader.isNull(inputColumn)) {
-                        ColumnConfig columnConfig = outputColumnConfigMap.get(outputColumn);
-                        // ToDo: speed up
-                        if (columnConfig.getDefault().isPresent()) {
-                            boolean default_value = ((Boolean)columnConfig.getDefault().get()).booleanValue();
-                            pageBuilder.setBoolean(outputColumn, default_value);
+                        Boolean default_value = (Boolean)outputDefaultMap.get(outputColumn);
+                        if (default_value != null) {
+                            pageBuilder.setBoolean(outputColumn, default_value.booleanValue());
                         } else {
                             pageBuilder.setNull(outputColumn);
                         }
@@ -137,11 +196,9 @@ public class ColumnFilterPlugin implements FilterPlugin
                 public void longColumn(Column outputColumn) {
                     Column inputColumn = outputInputColumnMap.get(outputColumn);
                     if (pageReader.isNull(inputColumn)) {
-                        ColumnConfig columnConfig = outputColumnConfigMap.get(outputColumn);
-                        // ToDo: speed up
-                        if (columnConfig.getDefault().isPresent()) {
-                            long default_value = ((Integer)columnConfig.getDefault().get()).longValue();
-                            pageBuilder.setLong(outputColumn, default_value);
+                        Long default_value = (Long)outputDefaultMap.get(outputColumn);
+                        if (default_value != null) {
+                            pageBuilder.setLong(outputColumn, default_value.longValue());
                         } else {
                             pageBuilder.setNull(outputColumn);
                         }
@@ -154,10 +211,9 @@ public class ColumnFilterPlugin implements FilterPlugin
                 public void doubleColumn(Column outputColumn) {
                     Column inputColumn = outputInputColumnMap.get(outputColumn);
                     if (pageReader.isNull(inputColumn)) {
-                        ColumnConfig columnConfig = outputColumnConfigMap.get(outputColumn);
-                        if (columnConfig.getDefault().isPresent()) {
-                            double default_value = ((Double)columnConfig.getDefault().get()).doubleValue();
-                            pageBuilder.setDouble(outputColumn, default_value);
+                        Double default_value = (Double)outputDefaultMap.get(outputColumn);
+                        if (default_value != null) {
+                            pageBuilder.setDouble(outputColumn, default_value.doubleValue());
                         } else {
                             pageBuilder.setNull(outputColumn);
                         }
@@ -170,9 +226,8 @@ public class ColumnFilterPlugin implements FilterPlugin
                 public void stringColumn(Column outputColumn) {
                     Column inputColumn = outputInputColumnMap.get(outputColumn);
                     if (pageReader.isNull(inputColumn)) {
-                        ColumnConfig columnConfig = outputColumnConfigMap.get(outputColumn);
-                        if (columnConfig.getDefault().isPresent()) {
-                            String default_value = (String)columnConfig.getDefault().get();
+                        String default_value = (String)outputDefaultMap.get(outputColumn);
+                        if (default_value != null) {
                             pageBuilder.setString(outputColumn, default_value);
                         } else {
                             pageBuilder.setNull(outputColumn);
@@ -186,8 +241,12 @@ public class ColumnFilterPlugin implements FilterPlugin
                 public void timestampColumn(Column outputColumn) {
                     Column inputColumn = outputInputColumnMap.get(outputColumn);
                     if (pageReader.isNull(inputColumn)) {
-                        // ToDo: default for timestamp
-                        pageBuilder.setNull(outputColumn);
+                        Timestamp default_value = (Timestamp)outputDefaultMap.get(outputColumn);
+                        if (default_value != null) {
+                            pageBuilder.setTimestamp(outputColumn, default_value);
+                        } else {
+                            pageBuilder.setNull(outputColumn);
+                        }
                     } else {
                         pageBuilder.setTimestamp(outputColumn, pageReader.getTimestamp(inputColumn));
                     }

data/src/main/java/org/embulk/filter/column/ColumnConfig.java

@@ -13,4 +13,12 @@ public interface ColumnConfig extends Task
     @Config("default")
     @ConfigDefault("null")
     public Optional<Object> getDefault();
+
+    @Config("format")
+    @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%N %z\"")
+    public Optional<String> getFormat();
+
+    @Config("timezone")
+    @ConfigDefault("\"UTC\"")
+    public Optional<String> getTimezone();
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-column
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-09 00:00:00.000000000 Z
+date: 2015-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -60,7 +60,7 @@ files:
 - src/main/java/org/embulk/filter/ColumnFilterPlugin.java
 - src/main/java/org/embulk/filter/column/ColumnConfig.java
 - src/test/java/org/embulk/filter/TestColumnFilterPlugin.java
-- classpath/embulk-filter-column-0.1.4.jar
+- classpath/embulk-filter-column-0.1.5.jar
 homepage: https://github.com/sonots/embulk-filter-column
 licenses:
 - MIT