embulk-filter-column 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 253e3fd65c6c834be802f0b4018864e73acc4776
-  data.tar.gz: b28d9c6008091ffeb9092933f35dc8c7f95dd257
+  metadata.gz: 7e3f1644fcdf2eee67ab2101e46eb5f546b9c373
+  data.tar.gz: 825008e7b6b9c4b1c3ce5da34e42fd055b627148
 SHA512:
-  metadata.gz: 4577f23c95ae14a8a4bedc419ef71aa60fcd5a9707e4bb55faac7098a3e2491aa0a71cbb0e2be0259291c9bc53599985bda8ac020fe77eb0b30e7ca26cfdef58
-  data.tar.gz: fbd551878d849ff891ed96dd78a9a28efb58d071ab03f56d1ae7bc8d2359bd53228a7322645348d261bf66ad23ff7ff9809867251be35b1c3d142981bb3ba1eb
+  metadata.gz: 4823b5a3e833c0ae0ae6302b7043027ce7e9b8740910c9b1753d312540a01d22e32dfe42b7af65cdb5227173d988255336bcc73d2bc1a4239ca6044d62c164c6
+  data.tar.gz: 78fa9b4a98d493a292e65e7e3387db08d997683937dfd594a69d45aecc8304d129ded9e3f3dfa409d6591323452a1dcd00389a30f0419e7794276145bffaa9c0

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.1.4
+
+Enhancements:
+
+* Add default option
+
 # 0.1.3
 
 Changes:

data/README.md

@@ -4,7 +4,9 @@ A filter plugin for Embulk to filter out columns
 
 ## Configuration
 
-- **columns**: column names (array of hash, required)
+- **columns**: columns (array of hash, required)
+  - **name**: name of column
+  - **default**: default value used if input is null
 
 ## Example
 
@@ -13,7 +15,7 @@ filters:
   - type: column
     columns:
       - {name: id}
-      - {name: name}
+      - {name: name, default: 'foo'}
 ```
 
 reduces columns to only `id` and `name` columns.

data/build.gradle

@@ -12,7 +12,7 @@ configurations {
     provided
 }
 
-version = "0.1.3"
+version = "0.1.4"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.6.12"

data/example.yml

@@ -1,15 +1,29 @@
+# in:
+#   type: random
+#   rows: 100
+#   schema:
+#     id: primary_key
+#     name: string
+#     score: integer
 in:
-  type: random
-  rows: 100
-  schema:
-    id: primary_key
-    name: string
-    score: integer
+  type: file
+  path_prefix: example.csv
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: 'NULL'
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: id, type: long}
+      - {name: name, type: string}
+      - {name: score, type: double}
 filters:
   - type: column
     columns:
-      - {name: name}
-      - {name: foo}
+      - {name: name, default: "foo"}
+      - {name: foo,  default: 1}
       - {name: id}
 out:
   type: stdout

data/src/main/java/org/embulk/filter/ColumnFilterPlugin.java

@@ -21,16 +21,16 @@ import org.embulk.spi.PageOutput;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfig;
-import org.embulk.spi.ColumnConfig;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
+import org.embulk.filter.column.ColumnConfig; // note: different with spi.ColumnConfig
 
 public class ColumnFilterPlugin implements FilterPlugin
 {
     public interface PluginTask extends Task
     {
         @Config("columns")
-        public SchemaConfig getColumns();
+        public List<ColumnConfig> getColumns();
     }
 
     @Override
@@ -39,18 +39,15 @@ public class ColumnFilterPlugin implements FilterPlugin
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        //Schema outputSchema = task.getColumns().toSchema();
         // Automatically get column type from inputSchema
-        SchemaConfig schemaConfig = task.getColumns();
-        List<ColumnConfig> outputColumnConfigs = schemaConfig.getColumns();
+        List<ColumnConfig> columnConfigs = task.getColumns();
         ImmutableList.Builder<Column> builder = ImmutableList.builder();
         int i = 0;
-        for (ColumnConfig outputColumnConfig : outputColumnConfigs) {
-            String outputColumnName = outputColumnConfig.getName();
+        for (ColumnConfig columnConfig : columnConfigs) {
+            String columnName = columnConfig.getName();
             for (Column inputColumn: inputSchema.getColumns()) {
-                if (inputColumn.getName().equals(outputColumnName)) {
-                    Type outputColumnType = inputColumn.getType();
-                    Column outputColumn = new Column(i++, outputColumnName, outputColumnType);
+                if (inputColumn.getName().equals(columnName)) {
+                    Column outputColumn = new Column(i++, columnName, inputColumn.getType());
                     builder.add(outputColumn);
                     break;
                 }
@@ -67,11 +64,21 @@ public class ColumnFilterPlugin implements FilterPlugin
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
 
-        HashMap<Column, Column> columnMap = new HashMap<Column, Column>();
+        HashMap<Column, Column> outputInputColumnMap = new HashMap<Column, Column>();
         for (Column outputColumn: outputSchema.getColumns()) {
             for (Column inputColumn: inputSchema.getColumns()) {
                 if (inputColumn.getName().equals(outputColumn.getName())) {
-                    columnMap.put(outputColumn, inputColumn);
+                    outputInputColumnMap.put(outputColumn, inputColumn);
+                    break;
+                }
+            }
+        }
+
+        HashMap<Column, ColumnConfig> outputColumnConfigMap = new HashMap<Column, ColumnConfig>();
+        for (Column outputColumn: outputSchema.getColumns()) {
+            for (ColumnConfig columnConfig : task.getColumns()) {
+                if (columnConfig.getName().equals(outputColumn.getName())) {
+                    outputColumnConfigMap.put(outputColumn, columnConfig);
                     break;
                 }
             }
@@ -111,9 +118,16 @@ public class ColumnFilterPlugin implements FilterPlugin
 
                 @Override
                 public void booleanColumn(Column outputColumn) {
-                    Column inputColumn = columnMap.get(outputColumn);
+                    Column inputColumn = outputInputColumnMap.get(outputColumn);
                     if (pageReader.isNull(inputColumn)) {
-                        pageBuilder.setNull(outputColumn);
+                        ColumnConfig columnConfig = outputColumnConfigMap.get(outputColumn);
+                        // ToDo: speed up
+                        if (columnConfig.getDefault().isPresent()) {
+                            boolean default_value = ((Boolean)columnConfig.getDefault().get()).booleanValue();
+                            pageBuilder.setBoolean(outputColumn, default_value);
+                        } else {
+                            pageBuilder.setNull(outputColumn);
+                        }
                     } else {
                         pageBuilder.setBoolean(outputColumn, pageReader.getBoolean(inputColumn));
                     }
@@ -121,9 +135,16 @@ public class ColumnFilterPlugin implements FilterPlugin
 
                 @Override
                 public void longColumn(Column outputColumn) {
-                    Column inputColumn = columnMap.get(outputColumn);
+                    Column inputColumn = outputInputColumnMap.get(outputColumn);
                     if (pageReader.isNull(inputColumn)) {
-                        pageBuilder.setNull(outputColumn);
+                        ColumnConfig columnConfig = outputColumnConfigMap.get(outputColumn);
+                        // ToDo: speed up
+                        if (columnConfig.getDefault().isPresent()) {
+                            long default_value = ((Integer)columnConfig.getDefault().get()).longValue();
+                            pageBuilder.setLong(outputColumn, default_value);
+                        } else {
+                            pageBuilder.setNull(outputColumn);
+                        }
                     } else {
                         pageBuilder.setLong(outputColumn, pageReader.getLong(inputColumn));
                     }
@@ -131,9 +152,15 @@ public class ColumnFilterPlugin implements FilterPlugin
 
                 @Override
                 public void doubleColumn(Column outputColumn) {
-                    Column inputColumn = columnMap.get(outputColumn);
+                    Column inputColumn = outputInputColumnMap.get(outputColumn);
                     if (pageReader.isNull(inputColumn)) {
-                        pageBuilder.setNull(outputColumn);
+                        ColumnConfig columnConfig = outputColumnConfigMap.get(outputColumn);
+                        if (columnConfig.getDefault().isPresent()) {
+                            double default_value = ((Double)columnConfig.getDefault().get()).doubleValue();
+                            pageBuilder.setDouble(outputColumn, default_value);
+                        } else {
+                            pageBuilder.setNull(outputColumn);
+                        }
                     } else {
                         pageBuilder.setDouble(outputColumn, pageReader.getDouble(inputColumn));
                     }
@@ -141,9 +168,15 @@ public class ColumnFilterPlugin implements FilterPlugin
 
                 @Override
                 public void stringColumn(Column outputColumn) {
-                    Column inputColumn = columnMap.get(outputColumn);
+                    Column inputColumn = outputInputColumnMap.get(outputColumn);
                     if (pageReader.isNull(inputColumn)) {
-                        pageBuilder.setNull(outputColumn);
+                        ColumnConfig columnConfig = outputColumnConfigMap.get(outputColumn);
+                        if (columnConfig.getDefault().isPresent()) {
+                            String default_value = (String)columnConfig.getDefault().get();
+                            pageBuilder.setString(outputColumn, default_value);
+                        } else {
+                            pageBuilder.setNull(outputColumn);
+                        }
                     } else {
                         pageBuilder.setString(outputColumn, pageReader.getString(inputColumn));
                     }
@@ -151,8 +184,9 @@ public class ColumnFilterPlugin implements FilterPlugin
 
                 @Override
                 public void timestampColumn(Column outputColumn) {
-                    Column inputColumn = columnMap.get(outputColumn);
+                    Column inputColumn = outputInputColumnMap.get(outputColumn);
                     if (pageReader.isNull(inputColumn)) {
+                        // ToDo: default for timestamp
                         pageBuilder.setNull(outputColumn);
                     } else {
                         pageBuilder.setTimestamp(outputColumn, pageReader.getTimestamp(inputColumn));

data/src/main/java/org/embulk/filter/column/ColumnConfig.java

@@ -0,0 +1,16 @@
+package org.embulk.filter.column;
+
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.Task;
+import com.google.common.base.Optional;
+
+public interface ColumnConfig extends Task
+{
+    @Config("name")
+    public String getName();
+
+    @Config("default")
+    @ConfigDefault("null")
+    public Optional<Object> getDefault();
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-column
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Naotoshi Seo
@@ -50,6 +50,7 @@ files:
 - LICENSE.txt
 - README.md
 - build.gradle
+- example.csv
 - example.yml
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
@@ -57,8 +58,9 @@ files:
 - gradlew.bat
 - lib/embulk/filter/column.rb
 - src/main/java/org/embulk/filter/ColumnFilterPlugin.java
+- src/main/java/org/embulk/filter/column/ColumnConfig.java
 - src/test/java/org/embulk/filter/TestColumnFilterPlugin.java
-- classpath/embulk-filter-column-0.1.3.jar
+- classpath/embulk-filter-column-0.1.4.jar
 homepage: https://github.com/sonots/embulk-filter-column
 licenses:
 - MIT