embulk-filter-encrypt 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d7ccdad7bf5592cae007e69d377dd6d8fe729fbe
-  data.tar.gz: f3a74ecd872841e10d0b740b117e7525c0aa156d
+  metadata.gz: 6de855d8ac0f1c315220c4907fe6eb8d7bd40043
+  data.tar.gz: b21751d55746b058abfef300e3927187fb8f6185
 SHA512:
-  metadata.gz: 637dcbc1f0d2ac4c261ae7a1fcd43e9620e12e3668939429b1a88a93b4ba6ab655ce7e2f04e5fc2ea5c75cf803ebad1f633bb2700964a790a5fe46a9405afbc8
-  data.tar.gz: 4971f179382b25ab01dbad3269d11f0b9e275d5f640262355da66157869c738025da408bd7cd395b0b95aaa5b8c36f483d85d9bdc5c572cccc55c854b07d5cb2
+  metadata.gz: 97fbc5a6e919eb2b053fd2b4862eebc4edd1e4b1a2245c8274fe0780fcab557610a2c8aa8feb3c67cd13ea04bba2ed06b69416b4d0ea3ce9e825366c2b6e2b1c
+  data.tar.gz: 80f240d5de8da536ad01c5a998ba273740cc5aa63c5b102dcb83d816666921e3c171204ebaa56ae8e102f434fe0214499872240fcc4f0ee241170855a8a8c738

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 0.2.0 - 2018-06-13
+
+* Add `output_encoding` config, supports "base64" and "hex".
+
 Release 0.1.0 - 2016-03-01
 
 * The first release.

data/README.md

@@ -23,7 +23,8 @@ You can apply encryption to password column and get following outputs:
 - **algorithm**: encryption algorithm (see below) (enum, required)
 - **column_names**: names of string columns to encrypt (array of string, required)
 - **key_hex**: encryption key (string, required)
-- **iv_hex**: encyrption initialization vector (string, required if mode of the algorithm is CBC)
+- **iv_hex**: encryption initialization vector (string, required if mode of the algorithm is CBC)
+- **output_encoding**: the encoding of encrypted value, can be either "base64" or "hex" (base16)
 
 ## Algorithms
 
@@ -117,9 +118,11 @@ You can use Hive's `aes_decrypt(input binary, key binary)` function (available s
 ```yaml
 filters:
   - type: encrypt
+    algorithm: AES-256-CBC
     column_names: [password, ip]
     key_hex: 098F6BCD4621D373CADE4E832627B4F60A9172716AE6428409885B8B829CCB05
     iv_hex: C9DD4BB33B827EB1FBA1B16A0074D460
+    output_encoding: hex
 ```
 
 ## Build

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.2.0"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -23,6 +23,7 @@ dependencies {
     provided "org.embulk:embulk-core:0.8.6"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
     testCompile "junit:junit:4.+"
+    testCompile "org.embulk:embulk-core:0.8.18:tests"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/src/main/java/org/embulk/filter/encrypt/EncryptFilterPlugin.java

@@ -1,47 +1,43 @@
 package org.embulk.filter.encrypt;
 
-import java.util.List;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import javax.crypto.Cipher;
-import javax.crypto.SecretKey;
-import javax.crypto.SecretKeyFactory;
-import javax.crypto.spec.SecretKeySpec;
-import javax.crypto.spec.IvParameterSpec;
-import javax.crypto.spec.PBEKeySpec;
-import javax.crypto.NoSuchPaddingException;
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import java.security.AlgorithmParameters;
-import java.security.InvalidKeyException;
-import java.security.NoSuchAlgorithmException;
-import java.security.InvalidAlgorithmParameterException;
-import java.security.spec.KeySpec;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;
 import com.google.common.base.Optional;
 import com.google.common.io.BaseEncoding;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigException;
 import org.embulk.spi.Column;
-import org.embulk.spi.DataException;
 import org.embulk.spi.ColumnVisitor;
-import org.embulk.spi.type.StringType;
+import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FilterPlugin;
 import org.embulk.spi.Page;
 import org.embulk.spi.PageBuilder;
-import org.embulk.spi.PageReader;
 import org.embulk.spi.PageOutput;
+import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
-import org.embulk.spi.time.Timestamp;
+import org.slf4j.Logger;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.spec.IvParameterSpec;
+import javax.crypto.spec.SecretKeySpec;
+
+import java.security.InvalidAlgorithmParameterException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.util.EnumSet;
+import java.util.List;
+
+import static java.lang.String.format;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.commons.lang3.StringUtils.join;
 
 public class EncryptFilterPlugin
         implements FilterPlugin
@@ -49,12 +45,11 @@ public class EncryptFilterPlugin
     public static enum Algorithm
     {
         AES_256_CBC("AES/CBC/PKCS5Padding", "AES", 256, true, "AES", "AES-256", "AES-256-CBC"),
-        AES_192_CBC("AES/CBC/PKCS5Padding", "AES", 192, true, "AES", "AES-192", "AES-192-CBC"),
-        AES_128_CBC("AES/CBC/PKCS5Padding", "AES", 128, true, "AES", "AES-128", "AES-128-CBC"),
-        AES_256_ECB("AES/ECB/PKCS5Padding", "AES", 256, false, "AES", "AES-256", "AES-256-ECB"),
-        AES_192_ECB("AES/ECB/PKCS5Padding", "AES", 192, false, "AES", "AES-192", "AES-192-ECB"),
-        AES_128_ECB("AES/ECB/PKCS5Padding", "AES", 128, false, "AES", "AES-128", "AES-128-ECB"),
-        ;
+        AES_192_CBC("AES/CBC/PKCS5Padding", "AES", 192, true, "AES-192", "AES-192-CBC"),
+        AES_128_CBC("AES/CBC/PKCS5Padding", "AES", 128, true, "AES-128", "AES-128-CBC"),
+        AES_256_ECB("AES/ECB/PKCS5Padding", "AES", 256, false, "AES-256-ECB"),
+        AES_192_ECB("AES/ECB/PKCS5Padding", "AES", 192, false, "AES-192-ECB"),
+        AES_128_ECB("AES/ECB/PKCS5Padding", "AES", 128, false, "AES-128-ECB");
 
         private final String javaName;
         private final String javaKeySpecName;
@@ -112,15 +107,60 @@ public class EncryptFilterPlugin
         }
     }
 
+    public enum Encoder
+    {
+        BASE64("base64", BaseEncoding.base64()),
+        HEX("hex", BaseEncoding.base16());
+
+        private final BaseEncoding encoding;
+        private final String name;
+
+        Encoder(String name, BaseEncoding encoding)
+        {
+            this.name = name;
+            this.encoding = encoding;
+        }
+
+        public String encode(byte[] bytes)
+        {
+            return encoding.encode(bytes);
+        }
+
+        @JsonCreator
+        public static Encoder fromName(String name)
+        {
+            EnumSet<Encoder> encoders = EnumSet.allOf(Encoder.class);
+            for (Encoder encoder : encoders) {
+                if (encoder.name.equals(name)) {
+                    return encoder;
+                }
+            }
+            throw new ConfigException(
+                    format("Unsupported output encoding '%s'. Supported encodings are %s.",
+                            name,
+                            join(encoders, ", ")));
+        }
+
+        @JsonValue
+        @Override
+        public String toString()
+        {
+            return name;
+        }
+    }
+
     public interface PluginTask
             extends Task
     {
         @Config("algorithm")
         public Algorithm getAlgorithm();
 
+        @Config("output_encoding")
+        @ConfigDefault("\"base64\"")
+        public Encoder getOutputEncoding();
+
         @Config("key_hex")
-        @ConfigDefault("null")
-        public Optional<String> getKeyHex();
+        public String getKeyHex();
 
         @Config("iv_hex")
         @ConfigDefault("null")
@@ -130,19 +170,19 @@ public class EncryptFilterPlugin
         public List<String> getColumnNames();
     }
 
+    private static final Logger log = Exec.getLogger(EncryptFilterPlugin.class);
+
     @Override
     public void transaction(ConfigSource config, Schema inputSchema,
             FilterPlugin.Control control)
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        if (!task.getKeyHex().isPresent()) {
-        }
-        else if (task.getAlgorithm().useIv() && !task.getIvHex().isPresent()) {
+        if (task.getAlgorithm().useIv() && !task.getIvHex().isPresent()) {
             throw new ConfigException("Algorithm '" + task.getAlgorithm() + "' requires initialization vector. Please generate one and set it to iv_hex option.");
         }
         else if (!task.getAlgorithm().useIv() && task.getIvHex().isPresent()) {
-            throw new ConfigException("Algorithm '" + task.getAlgorithm() + "' doesn't use initialization vector. Please remove iv_hex option.");
+            log.warn("Algorithm '" + task.getAlgorithm() + "' doesn't use initialization vector. Please remove iv_hex option.");
         }
 
         // validate configuration
@@ -166,7 +206,7 @@ public class EncryptFilterPlugin
     {
         Algorithm algo = task.getAlgorithm();
 
-        byte[] keyData = BaseEncoding.base16().decode(task.getKeyHex().get());
+        byte[] keyData = BaseEncoding.base16().decode(task.getKeyHex());
         SecretKeySpec key = new SecretKeySpec(keyData, algo.getJavaKeySpecName());
 
         if (algo.useIv()) {
@@ -188,7 +228,7 @@ public class EncryptFilterPlugin
     public PageOutput open(TaskSource taskSource, final Schema inputSchema,
             final Schema outputSchema, final PageOutput output)
     {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
 
         final Cipher cipher;
         try {
@@ -207,7 +247,7 @@ public class EncryptFilterPlugin
         return new PageOutput() {
             private final PageReader pageReader = new PageReader(inputSchema);
             private final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
-            private final BaseEncoding base64 = BaseEncoding.base64();
+            private final Encoder encoder = task.getOutputEncoding();
 
             @Override
             public void finish()
@@ -291,7 +331,7 @@ public class EncryptFilterPlugin
                                     // this must not happen because always doFinal is called
                                     throw new DataException(ex);
                                 }
-                                String encoded = base64.encode(encrypted);
+                                String encoded = encoder.encode(encrypted);
                                 pageBuilder.setString(column, encoded);
                             }
                             else {

data/src/test/java/org/embulk/filter/encrypt/TestEncryptFilterPlugin.java

@@ -1,5 +1,540 @@
 package org.embulk.filter.encrypt;
 
+import com.google.common.collect.ImmutableList;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
+import org.embulk.filter.encrypt.EncryptFilterPlugin.PluginTask;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.FilterPlugin;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.spi.type.Types;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.spec.IvParameterSpec;
+import javax.crypto.spec.SecretKeySpec;
+
+import java.security.GeneralSecurityException;
+import java.security.InvalidAlgorithmParameterException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.List;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.io.BaseEncoding.base16;
+import static com.google.common.io.BaseEncoding.base64;
+import static java.lang.String.format;
+import static java.util.Collections.emptyList;
+import static java.util.Objects.requireNonNull;
+import static org.embulk.filter.encrypt.EncryptFilterPlugin.Algorithm;
+import static org.embulk.filter.encrypt.EncryptFilterPlugin.Algorithm.AES_128_CBC;
+import static org.embulk.filter.encrypt.EncryptFilterPlugin.Algorithm.AES_128_ECB;
+import static org.embulk.filter.encrypt.EncryptFilterPlugin.Algorithm.AES_192_CBC;
+import static org.embulk.filter.encrypt.EncryptFilterPlugin.Algorithm.AES_192_ECB;
+import static org.embulk.filter.encrypt.EncryptFilterPlugin.Algorithm.AES_256_CBC;
+import static org.embulk.filter.encrypt.EncryptFilterPlugin.Algorithm.AES_256_ECB;
+import static org.embulk.filter.encrypt.EncryptFilterPlugin.Encoder;
+import static org.embulk.filter.encrypt.EncryptFilterPlugin.Encoder.BASE64;
+import static org.embulk.filter.encrypt.EncryptFilterPlugin.Encoder.HEX;
+import static org.embulk.spi.PageTestUtils.buildPage;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.fail;
+
 public class TestEncryptFilterPlugin
 {
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Rule
+    public ExpectedException expectedException = ExpectedException.none();
+
+    private EncryptFilterPlugin plugin;
+
+    private ConfigSource defaultConfig()
+    {
+        return runtime.getExec().newConfigSource()
+                .set("type", "encrypt")
+                .set("algorithm", "AES-256-CBC")
+                .set("key_hex", "D0867C9310D061F17ACD11EB30DE68265DCB79849BE5FB2BE157919D19BF2F42")
+                .set("iv_hex", "2A1D6BD59D2DB50A59364BAD3B9B6544");
+    }
+
+    @Before
+    public void setup()
+    {
+        plugin = new EncryptFilterPlugin();
+    }
+
+    @Test(expected = GeneralSecurityException.class)
+    public void encrypt_with_AES_256_CBC() throws Exception
+    {
+        ConfigSource config = defaultConfig()
+                .set("algorithm", "AES-256-CBC")
+                .set("column_names", ImmutableList.of("should_be_encrypted"));
+        Schema schema = Schema.builder()
+                .add("should_be_encrypted", Types.STRING)
+                .build();
+
+        List rawRecord = ImmutableList.of("My super secret");
+        List filteredRecord = applyFilter(config, schema, rawRecord);
+
+        String plaintext = (String) rawRecord.get(0);
+        String ciphertext = (String) filteredRecord.get(0);
+
+        assertNotEquals(plaintext, ciphertext);
+        assertEquals(plaintext, decrypt(ciphertext, AES_256_CBC, config));
+
+        // Apparently it should fail when decrypt with a different algorithm
+        decrypt(ciphertext, AES_128_ECB, config);
+    }
+
+    @Test
+    public void encrypt_with_AES_256_CBC__alias_should_work_too() throws Exception
+    {
+        ConfigSource config = defaultConfig()
+                .set("algorithm", "AES")
+                .set("column_names", ImmutableList.of("should_be_encrypted"));
+        Schema schema = Schema.builder()
+                .add("should_be_encrypted", Types.STRING)
+                .build();
+
+        String plaintext = "My super secret!";
+
+        assertEquals(
+                plaintext,
+                decrypt((String) applyFilter(config, schema, ImmutableList.of(plaintext)).get(0),
+                        AES_256_CBC,
+                        config));
+    }
+
+    @Test
+    public void encrypt_with_AES_192_CBC() throws Exception
+    {
+        ConfigSource config = defaultConfig()
+                .set("algorithm", "AES-192-CBC")
+                .set("column_names", ImmutableList.of("should_be_encrypted"));
+        Schema schema = Schema.builder()
+                .add("should_be_encrypted", Types.STRING)
+                .build();
+
+        String plaintext = "My super secret!";
+
+        assertEquals(
+                plaintext,
+                decrypt((String) applyFilter(config, schema, ImmutableList.of(plaintext)).get(0),
+                        AES_192_CBC,
+                        config));
+    }
+
+    @Test
+    public void encrypt_with_AES_128_CBC() throws Exception
+    {
+        ConfigSource config = defaultConfig()
+                .set("algorithm", "AES-128-CBC")
+                .set("column_names", ImmutableList.of("should_be_encrypted"));
+        Schema schema = Schema.builder()
+                .add("should_be_encrypted", Types.STRING)
+                .build();
+
+        String plaintext = "My super secret!";
+
+        assertEquals(
+                plaintext,
+                decrypt((String) applyFilter(config, schema, ImmutableList.of(plaintext)).get(0),
+                        AES_128_CBC,
+                        config));
+    }
+
+    @Test
+    public void encrypt_with_AES_256_ECB() throws Exception
+    {
+        ConfigSource config = defaultConfig()
+                .set("algorithm", "AES-256-ECB")
+                .remove("iv_hex")
+                .set("column_names", ImmutableList.of("should_be_encrypted"));
+        Schema schema = Schema.builder()
+                .add("should_be_encrypted", Types.STRING)
+                .build();
+
+        String plaintext = "My super secret!";
+
+        assertEquals(
+                plaintext,
+                decrypt((String) applyFilter(config, schema, ImmutableList.of(plaintext)).get(0),
+                        AES_256_ECB,
+                        config));
+    }
+
+    @Test
+    public void encrypt_with_AES_192_ECB() throws Exception
+    {
+        ConfigSource config = defaultConfig()
+                .set("algorithm", "AES-192-ECB")
+                .remove("iv_hex")
+                .set("column_names", ImmutableList.of("should_be_encrypted"));
+        Schema schema = Schema.builder()
+                .add("should_be_encrypted", Types.STRING)
+                .build();
+
+        String plaintext = "My super secret!";
+
+        assertEquals(
+                plaintext,
+                decrypt((String) applyFilter(config, schema, ImmutableList.of(plaintext)).get(0),
+                        AES_192_ECB,
+                        config));
+    }
+
+    @Test
+    public void encrypt_with_AES_128_ECB() throws Exception
+    {
+        ConfigSource config = defaultConfig()
+                .set("algorithm", "AES-128-ECB")
+                .remove("iv_hex")
+                .set("column_names", ImmutableList.of("should_be_encrypted"));
+        Schema schema = Schema.builder()
+                .add("should_be_encrypted", Types.STRING)
+                .build();
+
+        String plaintext = "My super secret!";
+
+        assertEquals(
+                plaintext,
+                decrypt((String) applyFilter(config, schema, ImmutableList.of(plaintext)).get(0),
+                        AES_128_ECB,
+                        config));
+    }
+
+    @Test
+    public void encrypt_selective_columns() throws Exception
+    {
+        ConfigSource config = defaultConfig()
+                .set("column_names", ImmutableList.of("should_be_encrypted"));
+        Schema schema = Schema.builder()
+                .add("should_be_encrypted", Types.STRING)
+                .add("should_be_unencrypted", Types.STRING)
+                .build();
+
+        List raw = ImmutableList.of("My super secret!", "Hey yo!");
+        List filtered = applyFilter(config, schema, raw);
+
+        // Encrypted column
+        assertNotEquals(raw.get(0), filtered.get(0));
+        assertEquals(raw.get(0), decrypt((String) filtered.get(0), config));
+
+        // Unencrypted column
+        assertEquals(raw.get(1), filtered.get(1));
+    }
+
+    @Test
+    public void nonstring_is_not_intact_whatsoever() throws Exception
+    {
+        ConfigSource config = defaultConfig()
+                .set("column_names", ImmutableList.of("attempt_to_encrypt"));
+        Schema schema = Schema.builder()
+                .add("attempt_to_encrypt", Types.LONG)
+                .build();
+
+        List raw = ImmutableList.of(1L);
+        List filtered = applyFilter(config, schema, raw);
+
+        assertEquals(raw, filtered);
+    }
+
+    @Test
+    public void base64_encoding() throws Exception
+    {
+        ConfigSource config = defaultConfig()
+                .set("output_encoding", "base64")
+                .set("column_names", ImmutableList.of("should_be_encrypted"));
+        Schema schema = Schema.builder()
+                .add("should_be_encrypted", Types.STRING)
+                .build();
+
+        String plaintext = "a_plaintext";
+
+        String ciphertext = (String) applyFilter(config, schema, ImmutableList.of(plaintext)).get(0);
+
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        assertEquals(
+                plaintext,
+                decrypt(ciphertext,
+                        task.getAlgorithm(),
+                        task.getKeyHex(),
+                        task.getIvHex().orNull(),
+                        BASE64));
+        try {
+            decrypt(ciphertext,
+                    task.getAlgorithm(),
+                    task.getKeyHex(),
+                    task.getIvHex().orNull(),
+                    HEX);
+        }
+        catch (IllegalArgumentException ex) {
+            return;
+        }
+        fail("Expected an IllegalArgumentException for mismatch encoding!");
+    }
+
+    @Test
+    public void hex_encoding() throws Exception
+    {
+        ConfigSource config = defaultConfig()
+                .set("output_encoding", "hex")
+                .set("column_names", ImmutableList.of("should_be_encrypted"));
+        Schema schema = Schema.builder()
+                .add("should_be_encrypted", Types.STRING)
+                .build();
+
+        String plaintext = "a_plaintext";
+
+        String ciphertext = (String) applyFilter(config, schema, ImmutableList.of(plaintext)).get(0);
+
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        assertEquals(
+                plaintext,
+                decrypt(ciphertext,
+                        task.getAlgorithm(),
+                        task.getKeyHex(),
+                        task.getIvHex().orNull(),
+                        HEX));
+        try {
+            decrypt(ciphertext,
+                    task.getAlgorithm(),
+                    task.getKeyHex(),
+                    task.getIvHex().orNull(),
+                    BASE64);
+        }
+        // Since hex/base16 is a totally valid subset of base64, this won't yield
+        // an IllegalArgumentException when encoding, but a decrypting exception instead.
+        catch (GeneralSecurityException ex) {
+            return;
+        }
+        fail("Expected an IllegalArgumentException for mismatch encoding!");
+    }
+
+    @Test
+    public void default_output_encoding_should_be_base64()
+    {
+        ConfigSource config = defaultConfig()
+                .remove("output_encoding")
+                .set("column_names", emptyList());
+        PluginTask task = config.loadConfig(PluginTask.class);
+        assertEquals(task.getOutputEncoding(), BASE64);
+    }
+
+    @Test
+    // Previously, missing key_hex does throw a ConfigException but doesn't pinpoint the problematic field
+    public void absence_of_encryption_key_should_yell_a_meaningful_ConfigException() throws Exception
+    {
+        ConfigSource config = defaultConfig()
+                .remove("key_hex")
+                .set("column_names", ImmutableList.of("attempt_to_encrypt"));
+        Schema schema = Schema.builder()
+                .add("attempt_to_encrypt", Types.STRING)
+                .build();
+
+        expectedException.expect(ConfigException.class);
+        expectedException.expectMessage("key_hex");
+        applyFilter(config, schema, ImmutableList.of("Try to encrypt me buddy!"));
+    }
+
+    @Test
+    public void absence_of_iv_on_a_required_iv_algorithm_should_yell_a_meaningful_ConfigException() throws Exception
+    {
+        ConfigSource config = defaultConfig()
+                .remove("iv_hex")
+                .set("algorithm", "AES-256-CBC")
+                .set("column_names", ImmutableList.of("attempt_to_encrypt"));
+        Schema schema = Schema.builder()
+                .add("attempt_to_encrypt", Types.STRING)
+                .build();
+
+        expectedException.expect(ConfigException.class);
+        expectedException.expectMessage("iv_hex");
+        applyFilter(config, schema, ImmutableList.of("Try to encrypt me buddy!"));
+    }
+
+    @Test
+    // Previously, this will throw
+    public void presence_of_iv_on_a_non_iv_algorithm_should_be_silent() throws Exception
+    {
+        ConfigSource config = defaultConfig()
+                .remove("iv_hex")
+                .set("algorithm", "AES-128-ECB")
+                .set("column_names", ImmutableList.of("attempt_to_encrypt"));
+        Schema schema = Schema.builder()
+                .add("attempt_to_encrypt", Types.STRING)
+                .build();
+
+        applyFilter(config, schema, ImmutableList.of("Try to encrypt me buddy!"));
+    }
+
+    /** Apply the filter to a single record */
+    private PageReader applyFilter(ConfigSource config, final Schema schema, final Object... rawRecord)
+    {
+        if (rawRecord.length > schema.getColumnCount()) {
+            throw new UnsupportedOperationException("applyFilter() only supports a single record, " +
+                    "number of supplied values exceed the schema column size.");
+        }
+        final PluginTask task = config.loadConfig(PluginTask.class);
+
+        final MockPageOutput filteredOutput = new MockPageOutput();
+
+        plugin.transaction(config, schema, new FilterPlugin.Control()
+        {
+            @Override
+            public void run(TaskSource taskSource, Schema outputSchema)
+            {
+                PageOutput originalOutput = plugin.open(task.dump(), schema, outputSchema, filteredOutput);
+                originalOutput.add(buildPage(runtime.getBufferAllocator(), schema, rawRecord).get(0));
+                originalOutput.finish();
+                originalOutput.close();
+            }
+        });
+        assert filteredOutput.pages.size() == 1;
+
+        PageReader reader = new PageReader(schema);
+        reader.setPage(filteredOutput.pages.get(0));
+        reader.nextRecord();
+
+        return reader;
+    }
+
+    /** Conveniently returning a List after apply a filter over the original list */
+    private List applyFilter(ConfigSource config, Schema schema, List rawRecord)
+    {
+        try (PageReader reader = applyFilter(config, schema, rawRecord.toArray())) {
+            return readToList(reader, schema);
+        }
+    }
+
+    private static List readToList(final PageReader reader, Schema schema)
+    {
+        final Object[] filtered = new Object[schema.getColumnCount()];
+        schema.visitColumns(new ColumnVisitor()
+        {
+            @Override
+            public void booleanColumn(Column column)
+            {
+                filtered[column.getIndex()] = reader.getBoolean(column);
+            }
+
+            @Override
+            public void longColumn(Column column)
+            {
+                filtered[column.getIndex()] = reader.getLong(column);
+            }
+
+            @Override
+            public void doubleColumn(Column column)
+            {
+                filtered[column.getIndex()] = reader.getDouble(column);
+            }
+
+            @Override
+            public void stringColumn(Column column)
+            {
+                filtered[column.getIndex()] = reader.getString(column);
+            }
+
+            @Override
+            public void timestampColumn(Column column)
+            {
+                filtered[column.getIndex()] = reader.getTimestamp(column);
+            }
+
+            @Override
+            public void jsonColumn(Column column)
+            {
+                filtered[column.getIndex()] = reader.getJson(column);
+            }
+        });
+        return Arrays.asList(filtered);
+    }
+
+    private static String decrypt(String ciphertext, Algorithm algo, String keyHex, String ivHex, Encoder encoder)
+            throws NoSuchPaddingException,
+            NoSuchAlgorithmException,
+            InvalidAlgorithmParameterException,
+            InvalidKeyException,
+            BadPaddingException,
+            IllegalBlockSizeException
+    {
+        Cipher cipher = Cipher.getInstance(algo.getJavaName());
+        SecretKeySpec key = new SecretKeySpec(base16().decode(keyHex), algo.getJavaKeySpecName());
+        if (algo.useIv()) {
+            requireNonNull(ivHex, format("IV is required for this algorithm (%s)", algo));
+            IvParameterSpec iv = new IvParameterSpec(base16().decode(ivHex));
+            cipher.init(Cipher.DECRYPT_MODE, key, iv);
+        }
+        else {
+            cipher.init(Cipher.DECRYPT_MODE, key);
+        }
+        return new String(cipher.doFinal(decode(ciphertext, encoder)), UTF_8);
+    }
+
+    private static String decrypt(String ciphertext, ConfigSource config)
+            throws NoSuchPaddingException,
+            InvalidKeyException,
+            NoSuchAlgorithmException,
+            IllegalBlockSizeException,
+            BadPaddingException,
+            InvalidAlgorithmParameterException
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        return decrypt(
+                ciphertext,
+                task.getAlgorithm(),
+                task.getKeyHex(),
+                task.getIvHex().orNull(),
+                task.getOutputEncoding());
+    }
+
+    /** Just to be explicit about the algorithm in used */
+    private static String decrypt(String ciphertext, Algorithm algo, ConfigSource config)
+            throws NoSuchPaddingException,
+            InvalidKeyException,
+            NoSuchAlgorithmException,
+            IllegalBlockSizeException,
+            BadPaddingException,
+            InvalidAlgorithmParameterException
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        return decrypt(
+                ciphertext,
+                algo,
+                task.getKeyHex(),
+                task.getIvHex().orNull(),
+                task.getOutputEncoding());
+    }
+
+    /** Decoding by reversing the originalEncoder */
+    private static byte[] decode(String encoded, Encoder originalEncoder)
+    {
+        switch (originalEncoder) {
+            case BASE64:
+                return base64().decode(encoded);
+            case HEX:
+                return base16().decode(encoded);
+            default:
+                throw new UnsupportedOperationException("Unrecognized encoder: " + originalEncoder);
+        }
+    }
 }

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-encrypt
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-02 00:00:00.000000000 Z
+date: 2018-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: bundler
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  name: rake
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
 description: Encrypt
 email:
 - frsyuki@gmail.com
@@ -62,7 +62,7 @@ files:
 - lib/embulk/filter/encrypt.rb
 - src/main/java/org/embulk/filter/encrypt/EncryptFilterPlugin.java
 - src/test/java/org/embulk/filter/encrypt/TestEncryptFilterPlugin.java
-- classpath/embulk-filter-encrypt-0.1.0.jar
+- classpath/embulk-filter-encrypt-0.2.0.jar
 homepage: https://github.com/embulk/embulk-filter-encrypt
 licenses:
 - Apache 2.0