embulk-filter-column 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9337f13b7cc9bc47527a688296647de4193fd51c
-  data.tar.gz: b0a272b270b293c95c05dcd800c384aea0ad43df
+  metadata.gz: c666e71247649e05789afed9c460ac1699a459cb
+  data.tar.gz: ab41a4b1b49b5b46f1c50630f460cdf6be4fb1cc
 SHA512:
-  metadata.gz: e61f1a5005e3d066d21664ece6e2c1385fc8f9f955b869f26bc6ac0c0c6beab2de91238fe5eb0448b67e9c97e42bc4cc1fcf4b6a12b3af5d831082e6fc2f7930
-  data.tar.gz: a0921c3670203b94d148339e59f016dcf5caccd949307369b917743d5a99458c21bee252de16e4506cfec94e0e958e37a1d3229b2548f3b16870c3efab2fcd52
+  metadata.gz: 4d56b7c8a9cb017c337b2b84649db0184629a176b259d5fec0f0db72ba7eaa5d24284a6146e30c4612a849b137e673c040b1e42769c63da4a951aed09252b749
+  data.tar.gz: 6a4e0c0db4a8462c9ff9612afa2fd51e6a2eab528b2318401b1d4ff15cc459c38bb94dbca501fbbe82002dca9feb74ea1b6bbd71259021a45fb7654f2b707aae

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.3.1 (2015-11-09)
+
+Enhancements:
+
+* Add copy column feature (thanks to hidepin)
+
 # 0.3.0 (2015-10-27)
 
 Enhancements:

data/README.md

@@ -8,14 +8,16 @@ A filter plugin for Embulk to filter out columns
 
 - **columns**: columns to retain (array of hash)
   - **name**: name of column (required)
-  - **type**: type of column (required to add)
-  - **default**: default value used if input is null (required to add)
+  - **src**: src column name to be copied (optional, default is `name`)
+  - **default**: default value used if input is null (optional)
+  - **type**: type of the default value (required for `default`)
   - **format**: special option for timestamp column, specify the format of the default timestamp (string, default is `default_timestamp_format`)
   - **timezone**: special option for timestamp column, specify the timezone of the default timestamp (string, default is `default_timezone`)
 - **add_columns**: columns to add (array of hash)
   - **name**: name of column (required)
-  - **type**: type of column (required)
-  - **default**: value of column (required)
+  - **src**: src column name to be copied (either of `src` or `default` is required)
+  - **default**: value of column (either of `src` or `default` is required)
+  - **type**: type of the default value (required for `default`)
   - **format**: special option for timestamp column, specify the format of the default timestamp (string, default is `default_timestamp_format`)
   - **timezone**: special option for timestamp column, specify the timezone of the default timestamp (string, default is `default_timezone`)
 - **drop_columns**: columns to drop (array of hash)
@@ -38,9 +40,9 @@ time,id,key,score
 filters:
   - type: column
     columns:
-      - {key: time, default: "2015-07-13", format: "%Y-%m-%d"}
-      - {key: id}
-      - {key: key, default: "foo"}
+      - {name: time, default: "2015-07-13", format: "%Y-%m-%d"}
+      - {name: id}
+      - {name: key, default: "foo"}
 ```
 
 reduces columns to only `time`, `id`, and `key` columns as:
@@ -68,15 +70,16 @@ time,id,key,score
 filters:
   - type: column
     add_columns:
-      - {key: d, type: timestamp, default: "2015-07-13", format: "%Y-%m-%d"}
+      - {name: d, type: timestamp, default: "2015-07-13", format: "%Y-%m-%d"}
+      - {name: copy_id, src: id}
 ```
 
-add `d` column as:
+add `d` column, and `copy_id` column which is a copy of `id` column as:
 
 ```
-2015-07-13,0,Vqjht6YEUBsMPXmoW1iOGFROZF27pBzz0TUkOKeDXEY,1370,2015-07-13
-2015-07-13,1,VmjbjAA0tOoSEPv_vKAGMtD_0aXZji0abGe7_VXHmUQ,3962,2015-07-13
-2015-07-13,2,C40P5H1WcBx-aWFDJCI8th6QPEI2DOUgupt_gB8UutE,7323,2015-07,13
+2015-07-13,0,Vqjht6YEUBsMPXmoW1iOGFROZF27pBzz0TUkOKeDXEY,1370,2015-07-13,0
+2015-07-13,1,VmjbjAA0tOoSEPv_vKAGMtD_0aXZji0abGe7_VXHmUQ,3962,2015-07-13,1
+2015-07-13,2,C40P5H1WcBx-aWFDJCI8th6QPEI2DOUgupt_gB8UutE,7323,2015-07,13,2
 ```
 
 ## Example (drop_columns)
@@ -94,8 +97,8 @@ time,id,key,score
 filters:
   - type: column
     drop_columns:
-      - {key: time}
-      - {key: id}
+      - {name: time}
+      - {name: id}
 ```
 
 drop `time` and `id` columns as:

data/build.gradle

@@ -12,7 +12,7 @@ configurations {
     provided
 }
 
-version = "0.3.0"
+version = "0.3.1"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/example/add_columns.yml

@@ -18,8 +18,9 @@ filters:
     default_timezone: "Asia/Tokyo"
     default_timestamp_format: "%Y-%m-%d"
     add_columns:
-      - {name: foo, type: long, default: 1 }
-      - {name: d, type: timestamp, default: "2015-07-13" }
-      - {name: t, type: timestamp, default: "2015-07-13 00:00:00", timezone: "UTC", format: "%Y-%m-%d %H:%M:%S" }
+      - {name: foo, type: long, default: 1}
+      - {name: d, type: timestamp, default: "2015-07-13"}
+      - {name: t, type: timestamp, default: "2015-07-13 00:00:00", timezone: "UTC", format: "%Y-%m-%d %H:%M:%S"}
+      - {name: copy_score, src: score}
 out:
   type: stdout

data/example/columns.yml

@@ -27,5 +27,6 @@ filters:
       - {name: name, default: "foo"}
       - {name: foo,  default: 1, type: long}
       - {name: id}
+      - {name: copy_score, src: score}
 out:
   type: stdout

data/src/main/java/org/embulk/filter/ColumnFilterPlugin.java

@@ -71,6 +71,10 @@ public class ColumnFilterPlugin implements FilterPlugin
         @Config("timezone")
         @ConfigDefault("null")
         public Optional<DateTimeZone> getTimeZone();
+
+        @Config("src")
+        @ConfigDefault("null")
+        public Optional<String> getSrc();
     }
 
     public interface PluginTask extends Task, TimestampParser.Task
@@ -131,9 +135,11 @@ public class ColumnFilterPlugin implements FilterPlugin
                 String name                   = column.getName();
                 Optional<Type>   type         = column.getType();
                 Optional<Object> defaultValue = column.getDefault();
+                Optional<String> src          = column.getSrc();
 
-                Column inputColumn = getColumn(name, inputSchema);
-                if (inputColumn != null) { // filter column
+                String srcName = src.isPresent() ? src.get() : name;
+                Column inputColumn = getColumn(srcName, inputSchema);
+                if (inputColumn != null) { // filter or copy column
                     Column outputColumn = new Column(i++, name, inputColumn.getType());
                     builder.add(outputColumn);
                 }
@@ -142,7 +148,7 @@ public class ColumnFilterPlugin implements FilterPlugin
                     builder.add(outputColumn);
                 }
                 else {
-                    throw new SchemaConfigException(String.format("columns: Column '%s' is not found in inputSchema. Column '%s' does not have \"type\" and \"default\"", name, name));
+                    throw new SchemaConfigException(String.format("columns: Column src '%s' is not found in inputSchema. Column '%s' does not have \"type\" and \"default\"", srcName, name));
                 }
             }
         } else {
@@ -158,13 +164,24 @@ public class ColumnFilterPlugin implements FilterPlugin
                 String name                   = column.getName();
                 Optional<Type> type           = column.getType();
                 Optional<Object> defaultValue = column.getDefault();
+                Optional<String> src          = column.getSrc();
 
-                if (type.isPresent() && defaultValue.isPresent()) { // add column
+                String srcName = null;
+                Column inputColumn = null;
+                if (src.isPresent()) {
+                    srcName = src.get();
+                    inputColumn = getColumn(srcName, inputSchema);
+                }
+                if (inputColumn != null) { // copy column
+                    Column outputColumn = new Column(i++, name, inputColumn.getType());
+                    builder.add(outputColumn);
+                }
+                else if (type.isPresent() && defaultValue.isPresent()) { // add column
                     Column outputColumn = new Column(i++, name, type.get());
                     builder.add(outputColumn);
                 }
                 else {
-                    throw new SchemaConfigException(String.format("add_columns: Column '%s' does not have \"type\" and \"default\"", name));
+                    throw new SchemaConfigException(String.format("add_columns: Column src '%s' is not found in inputSchema, Column '%s' does not have \"type\" and \"default\"", srcName, name));
                 }
             }
         }
@@ -184,6 +201,16 @@ public class ColumnFilterPlugin implements FilterPlugin
         return null;
     }
 
+    private String getSrc(String name, List<ColumnConfig> columnConfigs) {
+        for (ColumnConfig columnConfig : columnConfigs) {
+            if (columnConfig.getName().equals(name) &&
+                columnConfig.getSrc().isPresent()) {
+                return (String)columnConfig.getSrc().get();
+            }
+        }
+        return null;
+    }
+
     private Object getDefault(String name, Type type, List<ColumnConfig> columnConfigs, PluginTask task) {
         for (ColumnConfig columnConfig : columnConfigs) {
             if (columnConfig.getName().equals(name)) {
@@ -246,7 +273,15 @@ public class ColumnFilterPlugin implements FilterPlugin
         // Map outputColumn => inputColumn
         final HashMap<Column, Column> outputInputColumnMap = new HashMap<Column, Column>();
         for (Column outputColumn: outputSchema.getColumns()) {
-            Column inputColumn = getColumn(outputColumn.getName(), inputSchema);
+            String name    = outputColumn.getName();
+            String srcName = getSrc(name, task.getColumns());
+            if (srcName == null) {
+                srcName = getSrc(name, task.getAddColumns());
+            }
+            if (srcName == null) {
+                srcName = name;
+            }
+            Column inputColumn = getColumn(srcName, inputSchema);
             outputInputColumnMap.put(outputColumn, inputColumn); // NOTE: inputColumn would be null
         }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-column
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-27 00:00:00.000000000 Z
+date: 2015-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -61,7 +61,7 @@ files:
 - lib/embulk/filter/column.rb
 - src/main/java/org/embulk/filter/ColumnFilterPlugin.java
 - src/test/java/org/embulk/filter/TestColumnFilterPlugin.java
-- classpath/embulk-filter-column-0.3.0.jar
+- classpath/embulk-filter-column-0.3.1.jar
 homepage: https://github.com/sonots/embulk-filter-column
 licenses:
 - MIT