embulk-filter-column 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ad887d441f728829ef11a8f5e6d6e76c5abb7ebe
-  data.tar.gz: 091933497cb175efdeeb0ecef88c2654dc9dd3cd
+  metadata.gz: 9337f13b7cc9bc47527a688296647de4193fd51c
+  data.tar.gz: b0a272b270b293c95c05dcd800c384aea0ad43df
 SHA512:
-  metadata.gz: 7a0ca14d8629cff148e580cf4f62f312fa22dbb0c6caf7fc35dea67cb4b8f699ac21f987dee17b0a497cc7310c0076613cbe1c46bc3dd9e5912c928db56101ca
-  data.tar.gz: 6c12d777e927becf2fd95fee004b9c021dd2b239db818e6bc539424db2386f62fd91aac4d4dc6ae8759ec6da88b96e1e95781c203ed62dbd63dacbf9a9dbad51
+  metadata.gz: e61f1a5005e3d066d21664ece6e2c1385fc8f9f955b869f26bc6ac0c0c6beab2de91238fe5eb0448b67e9c97e42bc4cc1fcf4b6a12b3af5d831082e6fc2f7930
+  data.tar.gz: a0921c3670203b94d148339e59f016dcf5caccd949307369b917743d5a99458c21bee252de16e4506cfec94e0e958e37a1d3229b2548f3b16870c3efab2fcd52

data/CHANGELOG.md

@@ -1,4 +1,11 @@
-# 0.2.0
+# 0.3.0 (2015-10-27)
+
+Enhancements:
+
+* Add `default_timestamp_format` option
+* Add `default_timezone` option
+
+# 0.2.0 (2015-10-27)
 
 Enhancements:
 

data/README.md

@@ -10,18 +10,18 @@ A filter plugin for Embulk to filter out columns
   - **name**: name of column (required)
   - **type**: type of column (required to add)
   - **default**: default value used if input is null (required to add)
-  - **format**: special option for timestamp column, specify the format of the default timestamp (string, default is `%Y-%m-%d %H:%M:%S.%N %z`, required to add)
-  - **timezone**: special option for timestamp column, specify the timezone of the default timestamp (string, default is `UTC`, required to add)
+  - **format**: special option for timestamp column, specify the format of the default timestamp (string, default is `default_timestamp_format`)
+  - **timezone**: special option for timestamp column, specify the timezone of the default timestamp (string, default is `default_timezone`)
 - **add_columns**: columns to add (array of hash)
   - **name**: name of column (required)
   - **type**: type of column (required)
   - **default**: value of column (required)
-  - **format**: special option for timestamp column, specify the format of the default timestamp (string, default is `%Y-%m-%d %H:%M:%S.%N %z`)
-  - **timezone**: special option for timestamp column, specify the timezone of the default timestamp (string, default is `UTC`)
+  - **format**: special option for timestamp column, specify the format of the default timestamp (string, default is `default_timestamp_format`)
+  - **timezone**: special option for timestamp column, specify the timezone of the default timestamp (string, default is `default_timezone`)
 - **drop_columns**: columns to drop (array of hash)
   - **name**: name of column (required)
-
-NOTE: column type is automatically retrieved from input data (inputSchema)
+- **default_timestamp_format**: default timestamp format for timestamp columns (string, default is `%Y-%m-%d %H:%M:%S.%N %z`)
+- **default_timezone**: default timezone for timestamp columns (string, default is `UTC`)
 
 ## Example (columns)
 
@@ -51,6 +51,8 @@ reduces columns to only `time`, `id`, and `key` columns as:
 2015-07-13,2,C40P5H1WcBx-aWFDJCI8th6QPEI2DOUgupt_gB8UutE
 ```
 
+Note that column types are automatically retrieved from input data (inputSchema).
+
 ## Example (add_columns)
 
 Say input.csv is as follows:
@@ -66,7 +68,7 @@ time,id,key,score
 filters:
   - type: column
     add_columns:
-      - {key: d, default: "2015-07-13", format: "%Y-%m-%d"}
+      - {key: d, type: timestamp, default: "2015-07-13", format: "%Y-%m-%d"}
 ```
 
 add `d` column as:
@@ -96,7 +98,7 @@ filters:
       - {key: id}
 ```
 
-add `time` and `id` columns as:
+drop `time` and `id` columns as:
 
 ```
 Vqjht6YEUBsMPXmoW1iOGFROZF27pBzz0TUkOKeDXEY,1370

data/build.gradle

@@ -12,7 +12,7 @@ configurations {
     provided
 }
 
-version = "0.2.0"
+version = "0.3.0"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/example/add_columns.yml

@@ -15,8 +15,11 @@ in:
       - {name: score, type: double}
 filters:
   - type: column
+    default_timezone: "Asia/Tokyo"
+    default_timestamp_format: "%Y-%m-%d"
     add_columns:
       - {name: foo, type: long, default: 1 }
-      - {name: d, type: timestamp, default: "2015-07-13", format: "%Y-%m-%d" }
+      - {name: d, type: timestamp, default: "2015-07-13" }
+      - {name: t, type: timestamp, default: "2015-07-13 00:00:00", timezone: "UTC", format: "%Y-%m-%d %H:%M:%S" }
 out:
   type: stdout

data/src/main/java/org/embulk/filter/ColumnFilterPlugin.java

@@ -40,7 +40,6 @@ import com.google.common.base.Throwables;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import com.google.common.base.Optional;
-import org.jruby.embed.ScriptingContainer;
 import org.embulk.spi.SchemaConfigException;
 
 public class ColumnFilterPlugin implements FilterPlugin
@@ -66,12 +65,12 @@ public class ColumnFilterPlugin implements FilterPlugin
         public Optional<Object> getDefault();
 
         @Config("format")
-        @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%N %z\"")
+        @ConfigDefault("null")
         public Optional<String> getFormat();
 
         @Config("timezone")
-        @ConfigDefault("\"UTC\"")
-        public Optional<String> getTimezone();
+        @ConfigDefault("null")
+        public Optional<DateTimeZone> getTimeZone();
     }
 
     public interface PluginTask extends Task, TimestampParser.Task
@@ -87,6 +86,8 @@ public class ColumnFilterPlugin implements FilterPlugin
         @Config("drop_columns")
         @ConfigDefault("[]")
         public List<ColumnConfig> getDropColumns();
+
+        // See TimestampParser for default_timestamp_format, and default_timezone
     }
 
     @Override
@@ -183,7 +184,7 @@ public class ColumnFilterPlugin implements FilterPlugin
         return null;
     }
 
-    private Object getDefault(String name, Type type, List<ColumnConfig> columnConfigs, ScriptingContainer jruby) {
+    private Object getDefault(String name, Type type, List<ColumnConfig> columnConfigs, PluginTask task) {
         for (ColumnConfig columnConfig : columnConfigs) {
             if (columnConfig.getName().equals(name)) {
                 if (type instanceof BooleanType) {
@@ -208,10 +209,20 @@ public class ColumnFilterPlugin implements FilterPlugin
                 }
                 else if (type instanceof TimestampType) {
                     if (columnConfig.getDefault().isPresent()) {
-                        String time            = (String)columnConfig.getDefault().get();
-                        String format          = (String)columnConfig.getFormat().get();
-                        DateTimeZone timezone  = DateTimeZone.forID((String)columnConfig.getTimezone().get());
-                        TimestampParser parser = new TimestampParser(jruby, format, timezone);
+                        String time   = (String)columnConfig.getDefault().get();
+                        String format = null;
+                        if (columnConfig.getFormat().isPresent()) {
+                            format = columnConfig.getFormat().get();
+                        } else {
+                            format = task.getDefaultTimestampFormat();
+                        }
+                        DateTimeZone timezone = null;
+                        if (columnConfig.getTimeZone().isPresent()) {
+                            timezone = columnConfig.getTimeZone().get();
+                        } else {
+                            timezone = task.getDefaultTimeZone();
+                        }
+                        TimestampParser parser = new TimestampParser(task.getJRuby(), format, timezone);
                         try {
                             Timestamp default_value = parser.parse(time);
                             return default_value;
@@ -245,9 +256,9 @@ public class ColumnFilterPlugin implements FilterPlugin
             String name = outputColumn.getName();
             Type   type = outputColumn.getType();
 
-            Object default_value = getDefault(name, type, task.getColumns(), task.getJRuby());
+            Object default_value = getDefault(name, type, task.getColumns(), task);
             if (default_value == null) {
-                default_value = getDefault(name, type, task.getAddColumns(), task.getJRuby());
+                default_value = getDefault(name, type, task.getAddColumns(), task);
             }
             if (default_value != null) {
                 outputDefaultMap.put(outputColumn, default_value);

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-column
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Naotoshi Seo
@@ -61,7 +61,7 @@ files:
 - lib/embulk/filter/column.rb
 - src/main/java/org/embulk/filter/ColumnFilterPlugin.java
 - src/test/java/org/embulk/filter/TestColumnFilterPlugin.java
-- classpath/embulk-filter-column-0.2.0.jar
+- classpath/embulk-filter-column-0.3.0.jar
 homepage: https://github.com/sonots/embulk-filter-column
 licenses:
 - MIT