embulk-filter-base64 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d01a8a26033d897ab2bfb48614706211c882bf12
-  data.tar.gz: 86ff16dd3e6a85676af9c2a91953edfd25847b4d
+  metadata.gz: 920673effb3067f179e366a12a6b1ea760a9c199
+  data.tar.gz: a77a241890b994b2fabc95d1ae175f314f70f6ce
 SHA512:
-  metadata.gz: cf2b41351d1dfb2ba5742721f2f608e063f99a715d504177b139ae12728d8cba93f0b3abac937f6b37241b8488f2b386dbe54d49cbf452da970599d1fbdc40e5
-  data.tar.gz: 18ba893338e8c53f3dc28d4d63abf6dde24ac7befaaa3625e3ff4270b4be7fa77df194fc16a9871b808abda927ac02bab7396da667fa5ee582fb11cd1982ffcb
+  metadata.gz: 85586eecb14aeba081c8706353c393c11d7b3592b1ef3d3f3a5a90bcd8bce8268adaad14c044c35cab95a2da1c94eb6ebdadfb899ffcb2d982a7e287ef635e80
+  data.tar.gz: db625af1af9e2ba4a3b4f38de4c23d5f1f48276be36b45b89edd0408b6eb7cc69f7f88f699edc96f373baa66ec06a0bc283f23cb9fd36b53fa8832bc67e31d3c

data/README.md

@@ -17,6 +17,7 @@ An Embulk filter plugin to encode/decode string by Base64.
 
 ## Limitation
 
+* Java8 environment is needed because this plugin uses [java.util.Base64](https://docs.oracle.com/javase/8/docs/api/java/util/Base64.html) of Java8
 * Type of input value to be encoded must be string.
   - encoded value is string and is is needed to align the type of input and output value
     + e.g. 1234(string) is encoded into MTIzNA==(string)
@@ -102,7 +103,8 @@ timestamp to decode (timestamp) : 2017-01-01 00:00:00 UTC
 
 ### Todo
 
-* Write tests
+* [Support base64 in apache commons codec](https://github.com/ysk24ok/embulk-filter-base64/issues/1)
+* [Support encoder/decoder of URL and MIME](https://github.com/ysk24ok/embulk-filter-base64/issues/2)
 
 ## Build
 

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.1"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -23,6 +23,7 @@ dependencies {
     provided "org.embulk:embulk-core:0.8.15"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
     testCompile "junit:junit:4.+"
+    testCompile "org.embulk:embulk-core:0.8.15:tests"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/src/main/java/org/embulk/filter/base64/Base64FilterPlugin.java

@@ -8,7 +8,6 @@ import java.util.Base64;
 import com.google.common.base.Optional;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
@@ -41,6 +40,8 @@ public class Base64FilterPlugin
         @Config("name")
         public String getName();
 
+        // TODO: getType
+
         @Config("encode")
         @ConfigDefault("false")
         public Optional<Boolean> getDoEncode();
@@ -50,15 +51,35 @@ public class Base64FilterPlugin
         public Optional<Boolean> getDoDecode();
     }
 
+    public void validate(PluginTask pluginTask, Schema inputSchema)
+    {
+        for (Base64ColumnTask task : pluginTask.getColumns()) {
+            // throws exception if the column name does not exist
+            inputSchema.lookupColumn(task.getName());
+            boolean doEncode = task.getDoEncode().get();
+            boolean doDecode = task.getDoDecode().get();
+            boolean bothTrue = doEncode && doDecode;
+            boolean bothFalse = !doEncode && !doDecode;
+            if (bothTrue || bothFalse) {
+                String errMsg = "Specify either 'encode: true' or 'decode: true'";
+                throw new DataException(errMsg);
+            }
+        }
+        for (Column column : inputSchema.getColumns()) {
+            Type colType = column.getType();
+            if (!Types.STRING.equals(colType)) {
+                String errMsg = "Type of input columns must be string";
+                throw new DataException(errMsg);
+            }
+        }
+    }
+
     @Override
     public void transaction(ConfigSource config, Schema inputSchema,
             FilterPlugin.Control control)
     {
         PluginTask task = config.loadConfig(PluginTask.class);
-        for (Base64ColumnTask base64ColTask: task.getColumns()) {
-            // throws exception if the column name does not exist
-            inputSchema.lookupColumn(base64ColTask.getName());
-        }
+        validate(task, inputSchema);
         Schema outputSchema = inputSchema;
         control.run(task.dump(), outputSchema);
     }
@@ -67,157 +88,52 @@ public class Base64FilterPlugin
     public PageOutput open(TaskSource taskSource, final Schema inputSchema,
             final Schema outputSchema, final PageOutput output)
     {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
-        final Map<String, Base64ColumnTask> base64ColumnMap = getBase64ColumnMap(task.getColumns());
-
-        return new PageOutput() {
-            private PageReader reader = new PageReader(inputSchema);
-            private PageBuilder builder = new PageBuilder(
-                Exec.getBufferAllocator(), outputSchema, output);
-            private ColumnVisitorImpl visitor = new ColumnVisitorImpl(builder);
-
-            @Override
-            public void add(Page page)
-            {
-                reader.setPage(page);
-                while (reader.nextRecord()) {
-                    for (Column column: inputSchema.getColumns()) {
-                        String colName = column.getName();
-                        Type colType = column.getType();
-                        Base64ColumnTask colTask = base64ColumnMap.get(colName);
-                        // columns where nothing to be done
-                        if (colTask == null) {
-                            column.visit(visitor);
-                            continue;
-                        }
-                        Boolean doEncode = colTask.getDoEncode().get();
-                        Boolean doDecode = colTask.getDoDecode().get();
-                        Boolean bothTrue = doEncode && doDecode;
-                        Boolean bothFalse = !doEncode && !doDecode;
-                        if (bothTrue || bothFalse) {
-                            String errMsg = "Specify either 'encode: true' or 'decode: true'";
-                            throw new Base64ValidateException(errMsg);
-                        }
-                        if (!Types.STRING.equals(colType)) {
-                            String errMsg = "Type of input columns must be string";
-                            throw new Base64ValidateException(errMsg);
-                        }
-                        // encode
-                        if (doEncode) {
-                            String raw = reader.getString(column);
-                            String encoded = Base64.getEncoder().encodeToString(raw.getBytes());
-                            builder.setString(column, encoded);
-                        }
-                        // decode
-                        if (doDecode) {
-                            String encoded = reader.getString(column);
-                            String decoded = new String(Base64.getDecoder().decode(encoded));
-                            builder.setString(column, decoded);
-                        }
-                    }
-                    builder.addRecord();
-                }
-            }
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
+        PageBuilder pageBuilder = new PageBuilder(
+            Exec.getBufferAllocator(), outputSchema, output);
+        PageReader pageReader = new PageReader(inputSchema);
+        ColumnVisitorImpl visitor = new ColumnVisitorImpl(
+            task, pageReader, pageBuilder);
+
+        return new PageOutputImpl(
+            pageReader, pageBuilder, outputSchema, visitor);
+    }
 
-            @Override
-            public void finish()
-            {
-                builder.finish();
-            }
+    public static class PageOutputImpl implements PageOutput
+    {
+        private PageReader pageReader;
+        private PageBuilder pageBuilder;
+        private Schema outputSchema;
+        private ColumnVisitorImpl visitor;
 
-            @Override
-            public void close()
-            {
-                builder.close();
-            }
+        PageOutputImpl(PageReader pageReader, PageBuilder pageBuilder, Schema outputSchema, ColumnVisitorImpl visitor)
+        {
+            this.pageReader = pageReader;
+            this.pageBuilder = pageBuilder;
+            this.outputSchema = outputSchema;
+            this.visitor = visitor;
+        }
 
-            class ColumnVisitorImpl implements ColumnVisitor
-            {
-                private final PageBuilder builder;
-
-                ColumnVisitorImpl(PageBuilder builder) {
-                    this.builder = builder;
-                }
-
-                @Override
-                public void booleanColumn(Column outputColumn)
-                {
-                    if (reader.isNull(outputColumn)) {
-                        builder.setNull(outputColumn);
-                    } else {
-                        builder.setBoolean(outputColumn, reader.getBoolean(outputColumn));
-                    }
-                }
-
-                @Override
-                public void longColumn(Column outputColumn)
-                {
-                    if (reader.isNull(outputColumn)) {
-                        builder.setNull(outputColumn);
-                    } else {
-                        builder.setLong(outputColumn, reader.getLong(outputColumn));
-                    }
-                }
-
-                @Override
-                public void doubleColumn(Column outputColumn)
-                {
-                    if (reader.isNull(outputColumn)) {
-                        builder.setNull(outputColumn);
-                    } else {
-                        builder.setDouble(outputColumn, reader.getDouble(outputColumn));
-                    }
-                }
-
-                @Override
-                public void stringColumn(Column outputColumn)
-                {
-                    if (reader.isNull(outputColumn)) {
-                        builder.setNull(outputColumn);
-                    } else {
-                        builder.setString(outputColumn, reader.getString(outputColumn));
-                    }
-                }
-
-                @Override
-                public void timestampColumn(Column outputColumn)
-                {
-                    if (reader.isNull(outputColumn)) {
-                        builder.setNull(outputColumn);
-                    } else {
-                        builder.setTimestamp(outputColumn, reader.getTimestamp(outputColumn));
-                    }
-                }
-
-                @Override
-                public void jsonColumn(Column outputColumn)
-                {
-                    if (reader.isNull(outputColumn)) {
-                        builder.setNull(outputColumn);
-                    } else {
-                        builder.setJson(outputColumn, reader.getJson(outputColumn));
-                    }
-                }
+        @Override
+        public void add(Page page)
+        {
+            pageReader.setPage(page);
+            while (pageReader.nextRecord()) {
+                outputSchema.visitColumns(visitor);
+                pageBuilder.addRecord();
             }
-        };
-    }
+        }
 
-    static Map<String, Base64ColumnTask> getBase64ColumnMap(
-            List<Base64ColumnTask> columnTasks)
-    {
-        Map<String, Base64ColumnTask> m = new HashMap<>();
-        for (Base64ColumnTask columnTask: columnTasks) {
-            m.put(columnTask.getName(), columnTask);
+        @Override
+        public void finish()
+        {
+            pageBuilder.finish();
         }
-        return m;
-    }
 
-    static class Base64ValidateException
-            extends DataException
-    {
-        Base64ValidateException(String message)
+        @Override
+        public void close()
         {
-            super(message);
+            pageBuilder.close();
         }
-    }
+    };
 }

data/src/main/java/org/embulk/filter/base64/ColumnVisitorImpl.java

@@ -0,0 +1,148 @@
+package org.embulk.filter.base64;
+
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Base64;
+
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.Task;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.DataException;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+
+import org.embulk.filter.base64.Base64FilterPlugin.Base64ColumnTask;
+import org.embulk.filter.base64.Base64FilterPlugin.PluginTask;
+
+public class ColumnVisitorImpl
+        implements ColumnVisitor
+{
+    private final PageReader pageReader;
+    private final PageBuilder pageBuilder;
+    private final Map<String, Base64ColumnTask> base64ColumnMap;
+
+    ColumnVisitorImpl(PluginTask task, PageReader reader, PageBuilder builder)
+    {
+        this.pageReader = reader;
+        this.pageBuilder = builder;
+        this.base64ColumnMap = getBase64ColumnMap(task.getColumns());
+    }
+
+    private static Map<String, Base64ColumnTask> getBase64ColumnMap(
+            List<Base64ColumnTask> columnTasks)
+    {
+        Map<String, Base64ColumnTask> m = new HashMap<>();
+        for (Base64ColumnTask columnTask : columnTasks) {
+            m.put(columnTask.getName(), columnTask);
+        }
+        return m;
+    }
+
+    private Base64ColumnTask getTask(Column column)
+    {
+        String colName = column.getName();
+        return base64ColumnMap.get(colName);
+    }
+
+    private String executeTask(Base64ColumnTask task, Column column)
+    {
+        boolean doEncode = task.getDoEncode().get();
+        boolean doDecode = task.getDoDecode().get();
+        // encode
+        if (doEncode) {
+            String raw = pageReader.getString(column);
+            return Base64.getEncoder().encodeToString(raw.getBytes());
+        }
+        // decode
+        //else if (doDecode) {
+        else {
+            String encoded = pageReader.getString(column);
+            return new String(Base64.getDecoder().decode(encoded));
+        }
+    }
+
+    @Override
+    public void booleanColumn(Column outputColumn)
+    {
+        if (pageReader.isNull(outputColumn)) {
+            pageBuilder.setNull(outputColumn);
+        }
+        else {
+            pageBuilder.setBoolean(
+                outputColumn, pageReader.getBoolean(outputColumn));
+        }
+    }
+
+    @Override
+    public void longColumn(Column outputColumn)
+    {
+        if (pageReader.isNull(outputColumn)) {
+            pageBuilder.setNull(outputColumn);
+        }
+        else {
+            pageBuilder.setLong(
+                outputColumn, pageReader.getLong(outputColumn));
+        }
+    }
+
+    @Override
+    public void doubleColumn(Column outputColumn)
+    {
+        if (pageReader.isNull(outputColumn)) {
+            pageBuilder.setNull(outputColumn);
+        }
+        else {
+            pageBuilder.setDouble(
+                outputColumn, pageReader.getDouble(outputColumn));
+        }
+    }
+
+    @Override
+    public void stringColumn(Column outputColumn)
+    {
+        if (pageReader.isNull(outputColumn)) {
+            pageBuilder.setNull(outputColumn);
+        }
+        else {
+            Base64ColumnTask task = getTask(outputColumn);
+            // when there is no task executed on this column
+            if (task == null) {
+                pageBuilder.setString(
+                    outputColumn, pageReader.getString(outputColumn));
+            // when there is a task
+            }
+            else {
+                String str = executeTask(task, outputColumn);
+                pageBuilder.setString(outputColumn, str);
+            }
+        }
+    }
+
+    @Override
+    public void timestampColumn(Column outputColumn)
+    {
+        if (pageReader.isNull(outputColumn)) {
+            pageBuilder.setNull(outputColumn);
+        }
+        else {
+            pageBuilder.setTimestamp(
+                outputColumn, pageReader.getTimestamp(outputColumn));
+        }
+    }
+
+    @Override
+    public void jsonColumn(Column outputColumn)
+    {
+        if (pageReader.isNull(outputColumn)) {
+            pageBuilder.setNull(outputColumn);
+        }
+        else {
+            pageBuilder.setJson(
+                outputColumn, pageReader.getJson(outputColumn));
+        }
+    }
+}

data/src/test/java/org/embulk/filter/base64/TestBase64FilterPlugin.java

@@ -1,5 +1,92 @@
 package org.embulk.filter.base64;
 
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfigException;
+import org.embulk.spi.type.Types;
+import org.embulk.filter.base64.Base64FilterPlugin.PluginTask;
+
+import org.junit.Rule;
+import org.junit.Test;
+
 public class TestBase64FilterPlugin
 {
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    public Base64FilterPlugin plugin = new Base64FilterPlugin();
+
+    public static PluginTask taskFromYamlString(String... lines)
+    {
+        StringBuilder builder = new StringBuilder();
+        for (String line : lines) {
+            builder.append(line).append("\n");
+        }
+        String yamlString = builder.toString();
+
+        ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
+        ConfigSource config = loader.fromYamlString(yamlString);
+        return config.loadConfig(PluginTask.class);
+    }
+
+    @Test(expected = SchemaConfigException.class)
+    public void testValidate_noColumn()
+    {
+        PluginTask task = taskFromYamlString(
+            "type: base64",
+            "columns:",
+            "  - {name: to encode, type: string, encode: true}",
+            "  - {name: to decode, type: string, decode: true}"
+        );
+        Schema inputSchema = Schema.builder()
+            .add("to encode", Types.STRING)
+            .build();
+        plugin.validate(task, inputSchema);
+    }
+
+    @Test(expected = DataException.class)
+    public void testValidate_bothSpecified()
+    {
+        PluginTask task = taskFromYamlString(
+            "type: base64",
+            "columns:",
+            "  - {name: to encode, type: string, encode: true, decode: true}"
+        );
+        Schema inputSchema = Schema.builder()
+            .add("to encode", Types.STRING)
+            .build();
+        plugin.validate(task, inputSchema);
+    }
+
+    @Test(expected = DataException.class)
+    public void testValidate_bothNotSpecified()
+    {
+        PluginTask task = taskFromYamlString(
+            "type: base64",
+            "columns:",
+            "  - {name: to encode, type: string}"
+        );
+        Schema inputSchema = Schema.builder()
+            .add("to encode", Types.STRING)
+            .build();
+        plugin.validate(task, inputSchema);
+    }
+
+    @Test(expected = DataException.class)
+    public void testValidate_invalidInputType()
+    {
+        PluginTask task = taskFromYamlString(
+            "type: base64",
+            "columns:",
+            "  - {name: to encode, type: string, type: double}"
+        );
+        Schema inputSchema = Schema.builder()
+            .add("to encode", Types.DOUBLE)
+            .build();
+        plugin.validate(task, inputSchema);
+    }
 }

data/src/test/java/org/embulk/filter/base64/TestColumnVisitorImpl.java

@@ -0,0 +1,93 @@
+package org.embulk.filter.base64;
+
+import java.util.List;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.PageTestUtils;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.spi.type.Types;
+import org.embulk.spi.util.Pages;
+import org.embulk.filter.base64.Base64FilterPlugin.PluginTask;
+import org.embulk.filter.base64.Base64FilterPlugin.PageOutputImpl;
+import org.embulk.filter.base64.ColumnVisitorImpl;
+import static org.embulk.filter.base64.TestBase64FilterPlugin.taskFromYamlString;
+
+import org.junit.Rule;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class TestColumnVisitorImpl
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private List<Object[]> filter(PluginTask task, Schema inputSchema, Object... objects)
+    {
+        MockPageOutput output = new MockPageOutput();
+        Schema outputSchema = inputSchema;
+        PageBuilder pageBuilder = new PageBuilder(
+            runtime.getBufferAllocator(), outputSchema, output);
+        PageReader pageReader = new PageReader(inputSchema);
+        ColumnVisitorImpl visitor = new ColumnVisitorImpl(
+            task, pageReader, pageBuilder);
+
+        List<Page> pages = PageTestUtils.buildPage(
+            runtime.getBufferAllocator(), inputSchema, objects);
+        PageOutput mockPageOutput = new PageOutputImpl(
+            pageReader, pageBuilder, outputSchema, visitor);
+        for (Page page : pages) {
+            mockPageOutput.add(page);
+        }
+        mockPageOutput.finish();
+        mockPageOutput.close();
+        return Pages.toObjects(outputSchema, output.pages);
+    }
+
+    @Test
+    public void testExecuteTask_encode()
+    {
+        PluginTask task = taskFromYamlString(
+            "type: base64",
+            "columns:",
+            "  - {name: to encode, type: string, encode: true}"
+        );
+        Schema inputSchema = Schema.builder()
+            .add("to encode", Types.STRING)
+            .build();
+        List<Object[]> records = filter(task, inputSchema,
+                "John",
+                "David"
+        );
+        assertEquals(2, records.size());
+        assertEquals("Sm9obg==", records.get(0)[0]);
+        assertEquals("RGF2aWQ=", records.get(1)[0]);
+    }
+
+    @Test
+    public void testExecuteTask_decode()
+    {
+        PluginTask task = taskFromYamlString(
+            "type: base64",
+            "columns:",
+            "  - {name: to decode, type: string, decode: true}"
+        );
+        Schema inputSchema = Schema.builder()
+            .add("to decode", Types.STRING)
+            .build();
+        List<Object[]> records = filter(task, inputSchema,
+                "Sm9obg==",
+                "RGF2aWQ="
+        );
+        assertEquals(2, records.size());
+        assertEquals("John", records.get(0)[0]);
+        assertEquals("David", records.get(1)[0]);
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-base64
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Yusuke NISHIOKA
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-16 00:00:00.000000000 Z
+date: 2017-01-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -63,8 +63,10 @@ files:
 - gradlew.bat
 - lib/embulk/filter/base64.rb
 - src/main/java/org/embulk/filter/base64/Base64FilterPlugin.java
+- src/main/java/org/embulk/filter/base64/ColumnVisitorImpl.java
 - src/test/java/org/embulk/filter/base64/TestBase64FilterPlugin.java
-- classpath/embulk-filter-base64-0.1.0.jar
+- src/test/java/org/embulk/filter/base64/TestColumnVisitorImpl.java
+- classpath/embulk-filter-base64-0.1.1.jar
 homepage: https://github.com/ysk24ok/embulk-filter-base64
 licenses:
 - MIT