embulk-filter-base58 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b6646b2971f7197bf9636fad6b586f32637f1afa
-  data.tar.gz: 79e3ed639b082594ce38d77c27a11ede5accf05b
+  metadata.gz: 8068d8efa29a9afbd6b88d304b4af1a8da0ccda9
+  data.tar.gz: 60795ea043c03d3b9f7f2e62559adc21caa86108
 SHA512:
-  metadata.gz: 3c2bffac02a0f2191df9fb6c1974e24c74bc7df002ab87ca6fa4f947b87c2abbc2db7e03a67c357999dc71a8e6fe02d5cb0b0b30392c2306f52a015c44beb339
-  data.tar.gz: 34e7d6ad1d928a30ea939695b0f73b2999d02e7ff8e43f5cea1c51e9ef4e177632bb1de49d52dd9d200a021235a22578b70bc2ef99766df4e2cff9b7fa060e0b
+  metadata.gz: 41aebd1779a57dddca726ad4b42184476428322c9a4fbbb5ab9cefe09f61ea8838937c471a5c3b41b4e0fb86e737a98a674238bf0c6e32bed82320184e3730d2
+  data.tar.gz: 421523c5edac63a00fd0c7197fa31c3dc66c0b44fab08175d851200bf15ea7b925496362251ca1f08da2426ae70ed97e8858560514c3be313b14acfaa0bff342

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.1.3"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.+"

data/src/main/java/org/embulk/filter/base58/Base58FilterPlugin.java

@@ -88,7 +88,7 @@ public class Base58FilterPlugin implements FilterPlugin {
 
         final PluginTask task = taskSource.loadTask(PluginTask.class);
         final Map<String, List<Base58Column>> base58ColumnMap = convertBase58ColumnListToMap(task.getColumns());
-        final Map<String, Column> outputColumnMap = convertColumnListToMap(outputSchema.getColumns());
+        final Map<String, Column> outputColumnMap = convertColumnListToMap(outputSchema.getColumns(), logger);
 
         return new PageOutput() {
             private PageReader reader = new PageReader(inputSchema);
@@ -117,62 +117,66 @@ public class Base58FilterPlugin implements FilterPlugin {
 
     void setValue(final Map<String, List<Base58Column>> base58ColumnMap, final Map<String, Column> outputColumnMap, final PageReader reader, final Schema outputSchema, final PageBuilder builder) {
 
-        // Do base58 conversions ahead of iterating output columns
-        final Map<String, String> base58OutputMap = new HashMap<>();
-        for (List<Base58Column> columnSet : base58ColumnMap.values()) {
-            for (Base58Column column : columnSet) {
+        // Build a map of output columns that should be mapped to encoded/decoded original columns
+        final Map<String, Base58Column> modifiedColumnMap = new HashMap<>(); // output column -> base58 column config
+        for (List<Base58Column> base58SourceColumns : base58ColumnMap.values()) {
+            for (Base58Column base58Column : base58SourceColumns) {
+                String outputColumnName = base58Column.getNewName().isPresent() ? base58Column.getNewName().get() : base58Column.getName();
+                modifiedColumnMap.put(outputColumnName, base58Column);
+            }
+        }
+
+        // Set values on all our output columns
+        List<Column> columns = outputSchema.getColumns();
+        for (Column outputColumn : columns) {
+
+            // Convert the value if configured to do so
+            if (modifiedColumnMap.containsKey(outputColumn.getName())) {
+                Base58Column base58Column = modifiedColumnMap.get(outputColumn.getName());
+                Column sourceColumn = outputColumnMap.get(base58Column.getName());
                 String inputValue, convertedValue = null;
 
-                // If the original column is not a string, then forget about it (it should be hex)
-                Column originalColumn = outputColumnMap.get(column.getName());
-                if (Types.STRING.equals(originalColumn.getType())) {
-                    inputValue = reader.getString(originalColumn);
+                // Don't bother setting it if the source is null
+                if (reader.isNull(sourceColumn)) {
+                    builder.setNull(outputColumn);
+                    continue;
+                }
+
+                // Get the source value
+                if (Types.STRING.equals(sourceColumn.getType())) {
+                    inputValue = reader.getString(sourceColumn);
                 } else {
                     logger.error("cannot convert base58 value of non-string values. name: {}, type: {}, index: {}",
-                            originalColumn.getName(),
-                            originalColumn.getType(),
-                            originalColumn.getIndex());
-                    throw new DataException("Unexpected string type in column `" + originalColumn.getName() + "`. Got: " + originalColumn.getType());
+                            sourceColumn.getName(),
+                            sourceColumn.getType(),
+                            sourceColumn.getIndex());
+                    throw new DataException("Unexpected non-string type in column `" + sourceColumn.getName() + "`. Got: " + sourceColumn.getType());
                 }
 
-                // Convert the value
+                // Convert the source value
                 try {
-                    convertedValue = convertValue(inputValue, column.getIsEncode().or(true), column.getPrefix().or(""));
+                    convertedValue = convertValue(inputValue, base58Column.getIsEncode().or(true), base58Column.getPrefix().or(""));
                 } catch (Exception e) {
                     // Failed to do the conversion. Probably misconfigured or malformed value
                     logger.error("failed to encode/decode base58 column value. name: {}, type: {}, index: {}, value: {}, method: {}, prefix: {}, target_name: {}",
-                            originalColumn.getName(),
-                            originalColumn.getType(),
-                            originalColumn.getIndex(),
+                            sourceColumn.getName(),
+                            sourceColumn.getType(),
+                            sourceColumn.getIndex(),
                             inputValue,
-                            column.getIsEncode().get() ? "encode" : "decode",
-                            column.getPrefix(),
-                            column.getNewName().or(column.getName()));
+                            base58Column.getIsEncode().get() ? "encode" : "decode",
+                            base58Column.getPrefix(),
+                            base58Column.getNewName().or(base58Column.getName()));
                     logger.error("base58 conversion exception", e);
                     // Don't crash the import if a single value is screwed up. Just log it for now
                 }
 
-                // Add it to the output mappings
-                if (column.getNewName().isPresent()) {
-                    base58OutputMap.put(column.getNewName().get(), convertedValue);
-                } else {
-                    base58OutputMap.put(column.getName(), convertedValue);
-                }
-            }
-        }
-
-        List<Column> columns = outputSchema.getColumns();
-        for (Column outputColumn : columns) {
-
-            // Did we convert the value for this column?
-            if (base58OutputMap.containsKey(outputColumn.getName())) {
-                String value = base58OutputMap.get(outputColumn.getName());
-                if (value == null) {
+                if (convertedValue == null) {
                     builder.setNull(outputColumn);
+                    continue;
                 } else {
-                    builder.setString(outputColumn, base58OutputMap.get(outputColumn.getName()));
+                    builder.setString(outputColumn, convertedValue);
+                    continue;
                 }
-                continue;
             }
 
             // No value?
@@ -199,6 +203,9 @@ public class Base58FilterPlugin implements FilterPlugin {
             }
             else if (Types.JSON.equals(outputColumn.getType())) {
                 builder.setJson(outputColumn, reader.getJson(outputColumn));
+            } else {
+                // NO VALUE? It shall be null.
+                builder.setNull(outputColumn);
             }
         }
     }
@@ -222,9 +229,12 @@ public class Base58FilterPlugin implements FilterPlugin {
         return result;
     }
 
-    static Map<String, Column> convertColumnListToMap(List<Column> columns) {
+    static Map<String, Column> convertColumnListToMap(List<Column> columns, Logger logger) {
         Map<String, Column> result = new HashMap<>();
         for (Column column : columns) {
+            if (result.containsKey(column.getName())) {
+                logger.warn("Output column ({}) already defined. Do you have duplicate column names in your source?", column.getName());
+            }
             result.put(column.getName(), column);
         }
         return result;

data/src/test/java/org/embulk/filter/base58/TestBase58FilterImpl.java

@@ -57,7 +57,7 @@ public class TestBase58FilterImpl {
         PageBuilder pageBuilder = new PageBuilder(runtime.getBufferAllocator(), outputSchema, output);
         PageReader pageReader = new PageReader(inputSchema);
         final Map<String, List<Base58FilterPlugin.Base58Column>> base58ColumnMap = Base58FilterPlugin.convertBase58ColumnListToMap(task.getColumns());
-        final Map<String, Column> outputColumnMap = Base58FilterPlugin.convertColumnListToMap(outputSchema.getColumns());
+        final Map<String, Column> outputColumnMap = Base58FilterPlugin.convertColumnListToMap(outputSchema.getColumns(), null);
 
         List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema, objects);
         for (Page page : pages) {
@@ -141,7 +141,14 @@ public class TestBase58FilterImpl {
                 .build();
 
         List<Object[]> records = filter(task, inputSchema,
-                "54f5f8b37c158c2f12ee1c64", Timestamp.ofEpochSecond(0), null, new Boolean(true), new Long(0), new Double(0.5), ValueFactory.newString("json"));
+                "54f5f8b37c158c2f12ee1c64",
+                Timestamp.ofEpochSecond(0),
+                null,
+                new Boolean(true),
+                new Long(0),
+                new Double(0.5),
+                ValueFactory.newString("json")
+        );
 
         assertEquals(1, records.size());
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-base58
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Kevin Fitzgerald
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-06-26 00:00:00.000000000 Z
+date: 2017-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -59,7 +59,7 @@ files:
 - src/main/java/org/embulk/filter/base58/Base58FilterPlugin.java
 - src/test/java/org/embulk/filter/base58/TestBase58FilterImpl.java
 - src/test/java/org/embulk/filter/base58/TestBase58FilterPlugin.java
-- classpath/embulk-filter-base58-0.1.2.jar
+- classpath/embulk-filter-base58-0.1.3.jar
 homepage: https://github.com/kfitzgerald/embulk-filter-base58
 licenses:
 - MIT