embulk-filter-encrypt 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6de855d8ac0f1c315220c4907fe6eb8d7bd40043
-  data.tar.gz: b21751d55746b058abfef300e3927187fb8f6185
+  metadata.gz: d7278dc0b04a51ad156ea94582943fd934bd435d
+  data.tar.gz: 30091df627949d1342c7ac7389e26f353ebccb5f
 SHA512:
-  metadata.gz: 97fbc5a6e919eb2b053fd2b4862eebc4edd1e4b1a2245c8274fe0780fcab557610a2c8aa8feb3c67cd13ea04bba2ed06b69416b4d0ea3ce9e825366c2b6e2b1c
-  data.tar.gz: 80f240d5de8da536ad01c5a998ba273740cc5aa63c5b102dcb83d816666921e3c171204ebaa56ae8e102f434fe0214499872240fcc4f0ee241170855a8a8c738
+  metadata.gz: 3dfa97e8c4c1adb2cb28c67eaa07b405e5fce86643245b9f36a4458693235f73406b1353943b326bcce7b6b1a3db91d1d937e6fbcbfa33fe6199d5ca31c9874a
+  data.tar.gz: 0f0408f79ec21e587e2843225c40756da31322d4f90d5992fc7c204e3b63030f920229bd0d5de8ccb2ed948ff34620f6cc0e28a85f29f3e248da59682a8bf6ad

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 0.2.1 - 2018-08-15
+
+* Add `key_type` config, supports "inline" and "s3".
+
 Release 0.2.0 - 2018-06-13
 
 * Add `output_encoding` config, supports "base64" and "hex".

data/README.md

@@ -22,10 +22,17 @@ You can apply encryption to password column and get following outputs:
 
 - **algorithm**: encryption algorithm (see below) (enum, required)
 - **column_names**: names of string columns to encrypt (array of string, required)
-- **key_hex**: encryption key (string, required)
-- **iv_hex**: encryption initialization vector (string, required if mode of the algorithm is CBC)
+- **key_type**: encryption key (enum, optional, default: inline), can be either "inline" or "s3"
+- **key_hex**: encryption key (string, required if key_type is inline)
+- **iv_hex**: encryption initialization vector (string, required if mode of the algorithm is CBC and key_type is inline)
 - **output_encoding**: the encoding of encrypted value, can be either "base64" or "hex" (base16)
-
+- **aws_params**: AWS/S3 parameters (hash, required if key_type is s3)
+    - **region**: a valid AWS region
+    - **access_key**: a valid AWS access key
+    - **secret_key**: a valid AWS secret key
+    - **bucket**: a valid S3 bucket
+    - **path**: a valid S3 key (S3 file path)
+    
 ## Algorithms
 
 Available algorithms are:
@@ -115,6 +122,8 @@ You can use Hive's `aes_decrypt(input binary, key binary)` function (available s
 
 ## Example
 
+* Inline key type
+
 ```yaml
 filters:
   - type: encrypt
@@ -125,6 +134,23 @@ filters:
     output_encoding: hex
 ```
 
+* S3 key type
+
+```yaml
+filters:
+  - type: encrypt
+    algorithm: AES-256-CBC
+    column_names: [password, ip]
+    output_encoding: hex
+    key_type: s3
+    aws_params:
+      region: us-east-2
+      access_key: XXXXXXXXXXXXXXXXXXXX
+      secret_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+      bucket: com.sample.keys
+      path: key.aes
+```
+
 ## Build
 
 ```

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.2.0"
+version = "0.2.1"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -21,9 +21,11 @@ targetCompatibility = 1.7
 dependencies {
     compile  "org.embulk:embulk-core:0.8.6"
     provided "org.embulk:embulk-core:0.8.6"
-    // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
+    compile "com.amazonaws:aws-java-sdk-s3:1.11.253"
+
     testCompile "junit:junit:4.+"
     testCompile "org.embulk:embulk-core:0.8.18:tests"
+    testCompile "org.mockito:mockito-core:2.+"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/src/main/java/org/embulk/filter/encrypt/EncryptFilterPlugin.java

@@ -1,7 +1,17 @@
 package org.embulk.filter.encrypt;
 
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.SdkClientException;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.S3Object;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Optional;
 import com.google.common.io.BaseEncoding;
 import org.embulk.config.Config;
@@ -10,6 +20,7 @@ import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
+import org.embulk.config.YamlTagResolver;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.DataException;
@@ -21,6 +32,10 @@ import org.embulk.spi.PageOutput;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.slf4j.Logger;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.constructor.SafeConstructor;
+import org.yaml.snakeyaml.representer.Representer;
 
 import javax.crypto.BadPaddingException;
 import javax.crypto.Cipher;
@@ -29,12 +44,15 @@ import javax.crypto.NoSuchPaddingException;
 import javax.crypto.spec.IvParameterSpec;
 import javax.crypto.spec.SecretKeySpec;
 
+import java.io.IOException;
 import java.security.InvalidAlgorithmParameterException;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 import java.util.EnumSet;
 import java.util.List;
+import java.util.Map;
 
+import static com.google.common.base.Strings.isNullOrEmpty;
 import static java.lang.String.format;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.apache.commons.lang3.StringUtils.join;
@@ -42,7 +60,7 @@ import static org.apache.commons.lang3.StringUtils.join;
 public class EncryptFilterPlugin
         implements FilterPlugin
 {
-    public static enum Algorithm
+    public enum Algorithm
     {
         AES_256_CBC("AES/CBC/PKCS5Padding", "AES", 256, true, "AES", "AES-256", "AES-256-CBC"),
         AES_192_CBC("AES/CBC/PKCS5Padding", "AES", 192, true, "AES-192", "AES-192-CBC"),
@@ -57,7 +75,7 @@ public class EncryptFilterPlugin
         private final boolean useIv;
         private String[] displayNames;
 
-        private Algorithm(String javaName, String javaKeySpecName, int keyLength, boolean useIv, String... displayNames)
+        Algorithm(String javaName, String javaKeySpecName, int keyLength, boolean useIv, String... displayNames)
         {
             this.javaName = javaName;
             this.javaKeySpecName = javaKeySpecName;
@@ -149,6 +167,25 @@ public class EncryptFilterPlugin
         }
     }
 
+    public enum KeyType
+    {
+        INLINE,
+        S3;
+
+        @JsonCreator
+        public static KeyType of(String value)
+        {
+            return KeyType.valueOf(value.toUpperCase());
+        }
+
+        @Override
+        @JsonValue
+        public String toString()
+        {
+            return super.toString().toLowerCase();
+        }
+    }
+
     public interface PluginTask
             extends Task
     {
@@ -159,17 +196,49 @@ public class EncryptFilterPlugin
         @ConfigDefault("\"base64\"")
         public Encoder getOutputEncoding();
 
+        @Config("key_type")
+        @ConfigDefault("\"inline\"")
+        KeyType getKeyType();
+
         @Config("key_hex")
-        public String getKeyHex();
+        @ConfigDefault("null")
+        public Optional<String> getKeyHex();
+
+        public void setKeyHex(Optional<String> key);
 
         @Config("iv_hex")
         @ConfigDefault("null")
         public Optional<String> getIvHex();
 
+        public void setIvHex(Optional<String> iv);
+
+        @Config("aws_params")
+        @ConfigDefault("null")
+        public Optional<AWSParams> getAWSParams();
+
         @Config("column_names")
         public List<String> getColumnNames();
     }
 
+    public interface AWSParams extends Task
+    {
+        @Config("region")
+        public String getRegion();
+
+        @Config("access_key")
+        public String getAccessKey();
+
+        @Config("secret_key")
+        public String getSecretKey();
+
+        @Config("bucket")
+        public String getBucket();
+
+        @Config("path")
+        public String getPath();
+    }
+
+    private static final Yaml yaml = new Yaml(new SafeConstructor(), new Representer(), new DumperOptions(), new YamlTagResolver());
     private static final Logger log = Exec.getLogger(EncryptFilterPlugin.class);
 
     @Override
@@ -178,27 +247,134 @@ public class EncryptFilterPlugin
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        if (task.getAlgorithm().useIv() && !task.getIvHex().isPresent()) {
-            throw new ConfigException("Algorithm '" + task.getAlgorithm() + "' requires initialization vector. Please generate one and set it to iv_hex option.");
+        validateAndResolveKey(task, inputSchema);
+
+        control.run(task.dump(), inputSchema);
+    }
+
+    @VisibleForTesting
+    public Map<String, String> retrieveKey(final String bucket, final String path, final AmazonS3 client)
+    {
+        S3Object fullObject = null;
+
+        try {
+            fullObject = client.getObject(new GetObjectRequest(bucket, path));
+            if (fullObject == null) {
+                throw new ConfigException("S3 key file is not enabled to be retrieved");
+            }
+            return (Map<String, String>) yaml.load(fullObject.getObjectContent());
+        }
+        catch (AmazonServiceException e) {
+            // The call was transmitted successfully, but Amazon S3 couldn't process
+            // it, so it returned an error response.
+            if (e.getErrorType().equals(AmazonServiceException.ErrorType.Client)) {
+                // HTTP 40x errors. auth error, bucket doesn't exist, etc. See AWS document for the full list:
+                // http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+                if (e.getStatusCode() != 400
+                        || "ExpiredToken".equalsIgnoreCase(e.getErrorCode())) {
+                    throw new ConfigException(e);
+                }
+            }
+            throw e;
+        }
+        catch (ClassCastException e) {
+            throw new ConfigException("S3 key file content is unexpected format");
+        }
+        finally {
+            // To ensure that the network connection doesn't remain open, close any open input streams.
+            if (fullObject != null) {
+                try {
+                    fullObject.close();
+                }
+                catch (IOException e) {
+                    log.warn("Failure to close S3 Object input stream", e);
+                }
+            }
+        }
+    }
+
+    @VisibleForTesting
+    public AmazonS3 newS3Client(final AWSParams awsParams)
+    {
+        AWSCredentialsProvider awsCredentialsProvider = new AWSCredentialsProvider()
+        {
+            @Override
+            public AWSCredentials getCredentials()
+            {
+                return new BasicAWSCredentials(awsParams.getAccessKey(), awsParams.getSecretKey());
+            }
+
+            @Override
+            public void refresh()
+            {
+            }
+        };
+
+        try {
+            return AmazonS3ClientBuilder.standard()
+                    .withRegion(awsParams.getRegion())
+                    .withCredentials(awsCredentialsProvider)
+                    .build();
+        }
+        catch (SdkClientException e) {
+            throw new ConfigException(e);
         }
-        else if (!task.getAlgorithm().useIv() && task.getIvHex().isPresent()) {
-            log.warn("Algorithm '" + task.getAlgorithm() + "' doesn't use initialization vector. Please remove iv_hex option.");
+    }
+
+    private void validateAndResolveKey(PluginTask task, Schema schema) throws ConfigException
+    {
+        switch (task.getKeyType()) {
+            case INLINE:
+                if (!task.getKeyHex().isPresent()) {
+                    throw new ConfigException("Field 'key_hex' is required but not set");
+                }
+                if (task.getAlgorithm().useIv() && !task.getIvHex().isPresent()) {
+                    throw new ConfigException("Algorithm '" + task.getAlgorithm() + "' requires initialization vector. Please generate one and set it to iv_hex option.");
+                }
+                else if (!task.getAlgorithm().useIv() && task.getIvHex().isPresent()) {
+                    log.warn("Algorithm '" + task.getAlgorithm() + "' doesn't use initialization vector. iv_hex is ignored");
+                }
+                break;
+            case S3:
+                if (!task.getAWSParams().isPresent()) {
+                    throw new ConfigException("AWS Params are required for S3 Key type");
+                }
+                AWSParams params = task.getAWSParams().get();
+                AmazonS3 s3Client = newS3Client(params);
+                Map<String, String> keys = retrieveKey(params.getBucket(), params.getPath(), s3Client);
+
+                String key = keys.get("key_hex");
+                if (isNullOrEmpty(key)) {
+                    throw new ConfigException("Field 'key_hex' is required but not set");
+                }
+                String iv = keys.get("iv_hex");
+                if (task.getAlgorithm().useIv() && isNullOrEmpty(iv)) {
+                    throw new ConfigException("Algorithm '" + task.getAlgorithm() + "' requires initialization vector. Please generate one and set it to iv_hex option.");
+                }
+                else if (!task.getAlgorithm().useIv() && !isNullOrEmpty(iv)) {
+                    log.warn("Algorithm '" + task.getAlgorithm() + "' doesn't use initialization vector. iv_hex is ignored");
+                }
+                task.setKeyHex(Optional.of(key));
+                if (!isNullOrEmpty(iv)) {
+                    task.setIvHex(Optional.of(iv));
+                }
+                break;
+            default:
+                throw new ConfigException(String.format("Key type [%s] is not supported", task.getKeyType().toString()));
         }
 
-        // validate configuration
+        // Validate Cipher
         try {
             getCipher(Cipher.ENCRYPT_MODE, task);
         }
-        catch (Exception ex) {
-            throw new ConfigException(ex);
+        catch (Exception e) {
+            throw new ConfigException(e);
         }
 
         // validate column_names
         for (String name : task.getColumnNames()) {
-            inputSchema.lookupColumn(name);
+            schema.lookupColumn(name);
         }
-
-        control.run(task.dump(), inputSchema);
     }
 
     private Cipher getCipher(int mode, PluginTask task)
@@ -206,7 +382,7 @@ public class EncryptFilterPlugin
     {
         Algorithm algo = task.getAlgorithm();
 
-        byte[] keyData = BaseEncoding.base16().decode(task.getKeyHex());
+        byte[] keyData = BaseEncoding.base16().decode(task.getKeyHex().get());
         SecretKeySpec key = new SecretKeySpec(keyData, algo.getJavaKeySpecName());
 
         if (algo.useIv()) {

data/src/test/java/org/embulk/filter/encrypt/TestEncryptFilterPlugin.java

@@ -1,5 +1,7 @@
 package org.embulk.filter.encrypt;
 
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.Region;
 import com.google.common.collect.ImmutableList;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigException;
@@ -31,7 +33,9 @@ import java.security.InvalidAlgorithmParameterException;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import static com.google.common.base.Charsets.UTF_8;
 import static com.google.common.io.BaseEncoding.base16;
@@ -53,6 +57,9 @@ import static org.embulk.spi.PageTestUtils.buildPage;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
 
 public class TestEncryptFilterPlugin
 {
@@ -73,6 +80,22 @@ public class TestEncryptFilterPlugin
                 .set("iv_hex", "2A1D6BD59D2DB50A59364BAD3B9B6544");
     }
 
+    private ConfigSource s3Config()
+    {
+        ConfigSource awsParams = runtime.getExec().newConfigSource()
+                .set("region", "us-east-2")
+                .set("access_key", "a_access_key")
+                .set("secret_key", "a_secret_key")
+                .set("bucket", "a_bucket")
+                .set("path", "a_path");
+
+        return runtime.getExec().newConfigSource()
+                .set("type", "encrypt")
+                .set("algorithm", "AES-256-CBC")
+                .set("key_type", "s3")
+                .setNested("aws_params", awsParams);
+    }
+
     @Before
     public void setup()
     {
@@ -275,13 +298,13 @@ public class TestEncryptFilterPlugin
                 plaintext,
                 decrypt(ciphertext,
                         task.getAlgorithm(),
-                        task.getKeyHex(),
+                        task.getKeyHex().get(),
                         task.getIvHex().orNull(),
                         BASE64));
         try {
             decrypt(ciphertext,
                     task.getAlgorithm(),
-                    task.getKeyHex(),
+                    task.getKeyHex().get(),
                     task.getIvHex().orNull(),
                     HEX);
         }
@@ -311,13 +334,13 @@ public class TestEncryptFilterPlugin
                 plaintext,
                 decrypt(ciphertext,
                         task.getAlgorithm(),
-                        task.getKeyHex(),
+                        task.getKeyHex().get(),
                         task.getIvHex().orNull(),
                         HEX));
         try {
             decrypt(ciphertext,
                     task.getAlgorithm(),
-                    task.getKeyHex(),
+                    task.getKeyHex().get(),
                     task.getIvHex().orNull(),
                     BASE64);
         }
@@ -386,6 +409,260 @@ public class TestEncryptFilterPlugin
         applyFilter(config, schema, ImmutableList.of("Try to encrypt me buddy!"));
     }
 
+    @Test()
+    public void encrypt_with_required_IV_algorithm_for_s3() throws Exception
+    {
+        ConfigSource config = s3Config()
+                .set("column_names", ImmutableList.of("should_be_encrypted"));
+        Schema schema = Schema.builder()
+                .add("should_be_encrypted", Types.STRING)
+                .build();
+
+        plugin = spy(plugin);
+        Map<String, String> keys = new HashMap<>();
+        keys.put("key_hex", "D0867C9310D061F17ACD11EB30DE68265DCB79849BE5FB2BE157919D19BF2F42");
+        keys.put("iv_hex", "2A1D6BD59D2DB50A59364BAD3B9B6544");
+        doReturn(keys).when(plugin).retrieveKey(any(String.class), any(String.class), any(AmazonS3.class));
+
+        List rawRecord = ImmutableList.of("My super secret");
+        List filteredRecord = applyFilter(config, schema, rawRecord);
+
+        String plaintext = (String) rawRecord.get(0);
+        String ciphertext = (String) filteredRecord.get(0);
+
+        assertNotEquals(plaintext, ciphertext);
+        config.set("key_hex", keys.get("key_hex"));
+        config.set("iv_hex", keys.get("iv_hex"));
+        assertEquals(plaintext, decrypt(ciphertext, AES_256_CBC, config));
+    }
+
+    @Test()
+    public void encrypt_with_not_required_IV_algorithm_for_s3() throws Exception
+    {
+        ConfigSource config = s3Config()
+                .set("column_names", ImmutableList.of("should_be_encrypted"))
+                .set("algorithm", "AES-256-ECB");
+        Schema schema = Schema.builder()
+                .add("should_be_encrypted", Types.STRING)
+                .build();
+
+        plugin = spy(plugin);
+        Map<String, String> keys = new HashMap<>();
+        keys.put("key_hex", "D0867C9310D061F17ACD11EB30DE68265DCB79849BE5FB2BE157919D19BF2F42");
+        keys.put("iv_hex", "2A1D6BD59D2DB50A59364BAD3B9B6544");
+        doReturn(keys).when(plugin).retrieveKey(any(String.class), any(String.class), any(AmazonS3.class));
+
+        List rawRecord = ImmutableList.of("My super secret");
+        List filteredRecord = applyFilter(config, schema, rawRecord);
+
+        String plaintext = (String) rawRecord.get(0);
+        String ciphertext = (String) filteredRecord.get(0);
+
+        assertNotEquals(plaintext, ciphertext);
+        config.set("key_hex", keys.get("key_hex"));
+        assertEquals(plaintext, decrypt(ciphertext, AES_256_ECB, config));
+    }
+
+    @Test()
+    public void encrypt_with_not_required_IV_algorithm_for_s3_should_ignore_IV() throws Exception
+    {
+        ConfigSource config = s3Config()
+                .set("column_names", ImmutableList.of("should_be_encrypted"))
+                .set("algorithm", "AES-256-ECB");
+        Schema schema = Schema.builder()
+                .add("should_be_encrypted", Types.STRING)
+                .build();
+
+        plugin = spy(plugin);
+        Map<String, String> keys = new HashMap<>();
+        keys.put("key_hex", "D0867C9310D061F17ACD11EB30DE68265DCB79849BE5FB2BE157919D19BF2F42");
+        doReturn(keys).when(plugin).retrieveKey(any(String.class), any(String.class), any(AmazonS3.class));
+
+        List rawRecord = ImmutableList.of("My super secret");
+        List filteredRecord = applyFilter(config, schema, rawRecord);
+
+        String plaintext = (String) rawRecord.get(0);
+        String ciphertext = (String) filteredRecord.get(0);
+
+        assertNotEquals(plaintext, ciphertext);
+        config.set("key_hex", keys.get("key_hex"));
+        assertEquals(plaintext, decrypt(ciphertext, AES_256_ECB, config));
+    }
+
+    @Test
+    public void absence_of_aws_params_for_s3_should_yell_a_meaningful_ConfigException()
+    {
+        ConfigSource config = s3Config()
+                .set("column_names", ImmutableList.of("attempt_to_encrypt"))
+                .remove("aws_params");
+        Schema schema = Schema.builder()
+                .add("attempt_to_encrypt", Types.STRING)
+                .build();
+        expectedException.expect(ConfigException.class);
+        expectedException.expectMessage("AWS Params");
+        applyFilter(config, schema, ImmutableList.of("Try to encrypt me buddy!"));
+    }
+
+    @Test
+    public void absence_of_aws_region_param_for_s3_should_yell_a_meaningful_ConfigException()
+    {
+        ConfigSource config = s3Config()
+                .set("column_names", ImmutableList.of("attempt_to_encrypt"));
+
+        config.getNested("aws_params")
+            .remove("region");
+        Schema schema = Schema.builder()
+                .add("attempt_to_encrypt", Types.STRING)
+                .build();
+        expectedException.expect(ConfigException.class);
+        expectedException.expectMessage("region");
+        applyFilter(config, schema, ImmutableList.of("Try to encrypt me buddy!"));
+    }
+
+    @Test
+    public void absence_of_aws_path_param_for_s3_should_yell_a_meaningful_ConfigException()
+    {
+        ConfigSource config = s3Config()
+                .set("column_names", ImmutableList.of("attempt_to_encrypt"));
+
+        config.getNested("aws_params")
+                .remove("path");
+        Schema schema = Schema.builder()
+                .add("attempt_to_encrypt", Types.STRING)
+                .build();
+        expectedException.expect(ConfigException.class);
+        expectedException.expectMessage("path");
+        applyFilter(config, schema, ImmutableList.of("Try to encrypt me buddy!"));
+    }
+
+    @Test
+    public void absence_of_aws_bucket_param_for_s3_should_yell_a_meaningful_ConfigException()
+    {
+        ConfigSource config = s3Config()
+                .set("column_names", ImmutableList.of("attempt_to_encrypt"));
+
+        config.getNested("aws_params")
+                .remove("bucket");
+        Schema schema = Schema.builder()
+                .add("attempt_to_encrypt", Types.STRING)
+                .build();
+        expectedException.expect(ConfigException.class);
+        expectedException.expectMessage("bucket");
+        applyFilter(config, schema, ImmutableList.of("Try to encrypt me buddy!"));
+    }
+
+    @Test
+    public void absence_of_aws_access_key_param_for_s3_should_yell_a_meaningful_ConfigException()
+    {
+        ConfigSource config = s3Config()
+                .set("column_names", ImmutableList.of("attempt_to_encrypt"));
+
+        config.getNested("aws_params")
+                .remove("access_key");
+        Schema schema = Schema.builder()
+                .add("attempt_to_encrypt", Types.STRING)
+                .build();
+        expectedException.expect(ConfigException.class);
+        expectedException.expectMessage("access_key");
+        applyFilter(config, schema, ImmutableList.of("Try to encrypt me buddy!"));
+    }
+
+    @Test
+    public void absence_of_aws_secret_key_param_for_s3_should_yell_a_meaningful_ConfigException()
+    {
+        ConfigSource config = s3Config()
+                .set("column_names", ImmutableList.of("attempt_to_encrypt"));
+
+        config.getNested("aws_params")
+                .remove("secret_key");
+        Schema schema = Schema.builder()
+                .add("attempt_to_encrypt", Types.STRING)
+                .build();
+        expectedException.expect(ConfigException.class);
+        expectedException.expectMessage("secret_key");
+        applyFilter(config, schema, ImmutableList.of("Try to encrypt me buddy!"));
+    }
+
+    @Test
+    public void absence_of_encryption_key_for_s3_should_yell_a_meaningful_ConfigException()
+    {
+        ConfigSource config = s3Config()
+                .set("column_names", ImmutableList.of("attempt_to_encrypt"));
+        Schema schema = Schema.builder()
+                .add("attempt_to_encrypt", Types.STRING)
+                .build();
+
+        plugin = spy(plugin);
+        Map<String, String> keys = new HashMap<>();
+        doReturn(keys).when(plugin).retrieveKey(any(String.class), any(String.class), any(AmazonS3.class));
+
+        expectedException.expect(ConfigException.class);
+        expectedException.expectMessage("key_hex");
+        applyFilter(config, schema, ImmutableList.of("Try to encrypt me buddy!"));
+    }
+
+    @Test
+    public void absence_of_iv_on_a_required_iv_algorithm_for_s3_should_yell_a_meaningful_ConfigException()
+    {
+        ConfigSource config = s3Config()
+                .set("column_names", ImmutableList.of("attempt_to_encrypt"));
+        Schema schema = Schema.builder()
+                .add("attempt_to_encrypt", Types.STRING)
+                .build();
+
+        plugin = spy(plugin);
+        Map<String, String> keys = new HashMap<>();
+        keys.put("key_hex", "D0867C9310D061F17ACD11EB30DE68265DCB79849BE5FB2BE157919D19BF2F42");
+        doReturn(keys).when(plugin).retrieveKey(any(String.class), any(String.class), any(AmazonS3.class));
+
+        expectedException.expect(ConfigException.class);
+        expectedException.expectMessage("iv_hex");
+        applyFilter(config, schema, ImmutableList.of("Try to encrypt me buddy!"));
+    }
+
+    @Test
+    // Previously, this will throw
+    public void presence_of_iv_on_a_non_iv_algorithm_for_s3_should_be_silent()
+    {
+        ConfigSource config = s3Config()
+                .set("algorithm", "AES-128-ECB")
+                .set("column_names", ImmutableList.of("attempt_to_encrypt"));
+        Schema schema = Schema.builder()
+                .add("attempt_to_encrypt", Types.STRING)
+                .build();
+
+        plugin = spy(plugin);
+        Map<String, String> keys = new HashMap<>();
+        keys.put("key_hex", "D0867C9310D061F17ACD11EB30DE68265DCB79849BE5FB2BE157919D19BF2F42");
+        doReturn(keys).when(plugin).retrieveKey(any(String.class), any(String.class), any(AmazonS3.class));
+
+        applyFilter(config, schema, ImmutableList.of("Try to encrypt me buddy!"));
+    }
+
+    @Test
+    public void s3_client_should_reflect_region_config()
+    {
+        ConfigSource configSource = s3Config()
+                .set("column_names", ImmutableList.of("attempt_to_encrypt"));
+
+        AmazonS3 s3Client = plugin.newS3Client(configSource.loadConfig(EncryptFilterPlugin.PluginTask.class).getAWSParams().get());
+
+        assertEquals(s3Client.getRegion(), Region.US_East_2);
+    }
+
+    @Test
+    public void s3_client_invalid_region_should_yell_a_meaningful_ConfigException()
+    {
+        ConfigSource configSource = s3Config()
+                .set("column_names", ImmutableList.of("attempt_to_encrypt"));
+        configSource.getNested("aws_params")
+                .set("region", "invalid_region");
+
+        expectedException.expect(ConfigException.class);
+        expectedException.expectMessage("Unable to find a region via the region provider chain");
+        plugin.newS3Client(configSource.loadConfig(EncryptFilterPlugin.PluginTask.class).getAWSParams().get());
+    }
+
     /** Apply the filter to a single record */
     private PageReader applyFilter(ConfigSource config, final Schema schema, final Object... rawRecord)
     {
@@ -393,7 +670,6 @@ public class TestEncryptFilterPlugin
             throw new UnsupportedOperationException("applyFilter() only supports a single record, " +
                     "number of supplied values exceed the schema column size.");
         }
-        final PluginTask task = config.loadConfig(PluginTask.class);
 
         final MockPageOutput filteredOutput = new MockPageOutput();
 
@@ -402,7 +678,7 @@ public class TestEncryptFilterPlugin
             @Override
             public void run(TaskSource taskSource, Schema outputSchema)
             {
-                PageOutput originalOutput = plugin.open(task.dump(), schema, outputSchema, filteredOutput);
+                PageOutput originalOutput = plugin.open(taskSource, schema, outputSchema, filteredOutput);
                 originalOutput.add(buildPage(runtime.getBufferAllocator(), schema, rawRecord).get(0));
                 originalOutput.finish();
                 originalOutput.close();
@@ -502,7 +778,7 @@ public class TestEncryptFilterPlugin
         return decrypt(
                 ciphertext,
                 task.getAlgorithm(),
-                task.getKeyHex(),
+                task.getKeyHex().get(),
                 task.getIvHex().orNull(),
                 task.getOutputEncoding());
     }
@@ -520,7 +796,7 @@ public class TestEncryptFilterPlugin
         return decrypt(
                 ciphertext,
                 algo,
-                task.getKeyHex(),
+                task.getKeyHex().get(),
                 task.getIvHex().orNull(),
                 task.getOutputEncoding());
     }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-encrypt
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-06-14 00:00:00.000000000 Z
+date: 2018-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -62,7 +62,20 @@ files:
 - lib/embulk/filter/encrypt.rb
 - src/main/java/org/embulk/filter/encrypt/EncryptFilterPlugin.java
 - src/test/java/org/embulk/filter/encrypt/TestEncryptFilterPlugin.java
-- classpath/embulk-filter-encrypt-0.2.0.jar
+- classpath/aws-java-sdk-core-1.11.253.jar
+- classpath/aws-java-sdk-kms-1.11.253.jar
+- classpath/aws-java-sdk-s3-1.11.253.jar
+- classpath/commons-codec-1.9.jar
+- classpath/commons-logging-1.2.jar
+- classpath/embulk-filter-encrypt-0.2.1.jar
+- classpath/httpclient-4.5.2.jar
+- classpath/httpcore-4.4.4.jar
+- classpath/ion-java-1.0.2.jar
+- classpath/jackson-annotations-2.6.0.jar
+- classpath/jackson-core-2.6.7.jar
+- classpath/jackson-databind-2.6.7.1.jar
+- classpath/jackson-dataformat-cbor-2.6.7.jar
+- classpath/jmespath-java-1.11.253.jar
 homepage: https://github.com/embulk/embulk-filter-encrypt
 licenses:
 - Apache 2.0