embulk-input-dynamodb 0.2.0 → 0.3.0

data/src/test/scala/org/embulk/input/dynamodb/AwsCredentialsTest.scala

@@ -1,135 +1,188 @@
 package org.embulk.input.dynamodb
 
-import java.io.File
-import java.nio.charset.Charset
-import java.nio.file.{FileSystems, Files}
-
-import com.google.inject.{Binder, Module}
-import org.embulk.config.ConfigSource
-import org.embulk.exec.PartialExecutionException
-import org.embulk.plugin.InjectedPluginSource
-import org.embulk.spi.InputPlugin
-import org.embulk.{EmbulkEmbed, EmbulkTestRuntime}
-import org.junit.Assert.assertEquals
-import org.junit.{Before, Rule, Test}
-
-class AwsCredentialsTest {
-  private var EMBULK_DYNAMODB_TEST_REGION: String = null
-  private var EMBULK_DYNAMODB_TEST_TABLE: String = null
-  private var EMBULK_DYNAMODB_TEST_ACCESS_KEY: String = null
-  private var EMBULK_DYNAMODB_TEST_SECRET_KEY: String = null
-  private var EMBULK_DYNAMODB_TEST_PROFILE_NAME: String = null
-
-  private var embulk: EmbulkEmbed = null
-
-  @Rule
-  def runtime: EmbulkTestRuntime = new EmbulkTestRuntime
-
-  @Before
-  def createResources() {
-    // Get Environments
-    EMBULK_DYNAMODB_TEST_REGION = System.getenv("EMBULK_DYNAMODB_TEST_REGION")
-    EMBULK_DYNAMODB_TEST_TABLE = System.getenv("EMBULK_DYNAMODB_TEST_TABLE")
-    EMBULK_DYNAMODB_TEST_ACCESS_KEY = System.getenv("EMBULK_DYNAMODB_TEST_ACCESS_KEY")
-    EMBULK_DYNAMODB_TEST_SECRET_KEY = System.getenv("EMBULK_DYNAMODB_TEST_SECRET_KEY")
-    EMBULK_DYNAMODB_TEST_PROFILE_NAME = System.getenv("EMBULK_DYNAMODB_TEST_PROFILE_NAME")
-
-    val bootstrap = new EmbulkEmbed.Bootstrap()
-    bootstrap.addModules(new Module {
-      def configure(binder: Binder): Unit = {
-        InjectedPluginSource.registerPluginTo(binder,
-          classOf[InputPlugin],
-          "dynamodb",
-          classOf[DynamodbInputPlugin])
-      }
-    })
-
-    embulk = bootstrap.initializeCloseable()
+import com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException
+import org.embulk.config.{ConfigException, ConfigSource}
+import org.embulk.input.dynamodb.aws.AwsCredentials
+import org.embulk.input.dynamodb.testutil.EmbulkTestBase
+import org.hamcrest.CoreMatchers._
+import org.hamcrest.MatcherAssert.assertThat
+import org.junit.{Assume, Test}
+
+class AwsCredentialsTest extends EmbulkTestBase {
+
+  private val runAwsCredentialsTest: Boolean = Option(
+    System.getenv("RUN_AWS_CREDENTIALS_TEST")
+  ) match {
+    case Some(x) =>
+      if (x == "false") false
+      else true
+    case None => true
   }
 
-  def doTest(config: ConfigSource) {
-    embulk.run(config)
+  private lazy val EMBULK_DYNAMODB_TEST_ACCESS_KEY =
+    getEnvironmentVariableOrShowErrorMessage("EMBULK_DYNAMODB_TEST_ACCESS_KEY")
 
-    val fs = FileSystems.getDefault
-    val lines = Files.readAllLines(fs.getPath("result000.00.tsv"), Charset.forName("UTF-8"))
-    assertEquals("KEY-1\t1\tHogeHoge", lines.get(0))
+  private lazy val EMBULK_DYNAMODB_TEST_SECRET_KEY =
+    getEnvironmentVariableOrShowErrorMessage("EMBULK_DYNAMODB_TEST_SECRET_KEY")
+
+  private lazy val EMBULK_DYNAMODB_TEST_PROFILE_NAME =
+    getEnvironmentVariableOrShowErrorMessage(
+      "EMBULK_DYNAMODB_TEST_PROFILE_NAME"
+    )
+
+  private lazy val EMBULK_DYNAMODB_TEST_ASSUME_ROLE_ROLE_ARN =
+    getEnvironmentVariableOrShowErrorMessage(
+      "EMBULK_DYNAMODB_TEST_ASSUME_ROLE_ROLE_ARN"
+    )
+
+  def doTest(inConfig: ConfigSource): Unit = {
+    val task: PluginTask = inConfig.loadConfig(classOf[PluginTask])
+    val provider = AwsCredentials(task).createAwsCredentialsProvider
+    val cred = provider.getCredentials
+    assertThat(cred.getAWSAccessKeyId, notNullValue())
+    assertThat(cred.getAWSSecretKey, notNullValue())
   }
 
-  @Test
-  def notSetAuthMethod_SetCredentials() {
-    val config = embulk.newConfigLoader().fromYamlFile(
-      new File("src/test/resources/yaml/notSetAuthMethod.yml"))
+  def defaultInConfig: ConfigSource = {
+    embulk.configLoader().fromYamlString(s"""
+                                            |type: dynamodb
+                                            |region: us-east-1
+                                            |table: hoge
+                                            |operation: scan
+                                            |columns:
+                                            |  - {name: key1,   type: string}
+                                            |  - {name: key2,   type: long}
+                                            |  - {name: value1, type: string}
+                                            |""".stripMargin)
+  }
 
-    config.getNested("in")
-      .set("region", EMBULK_DYNAMODB_TEST_REGION)
-      .set("table", EMBULK_DYNAMODB_TEST_TABLE)
+  @deprecated(since = "0.3.0")
+  @Test
+  def notSetAuthMethod_SetCredentials_deprecated(): Unit = {
+    Assume.assumeTrue(runAwsCredentialsTest)
+    val inConfig: ConfigSource = defaultInConfig
       .set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
       .set("secret_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
 
-    doTest(config)
+    doTest(inConfig)
   }
 
   @Test
-  def setAuthMethod_Basic() {
-    val config = embulk.newConfigLoader().fromYamlFile(
-      new File("src/test/resources/yaml/authMethodBasic.yml"))
+  def notSetAuthMethod_SetCredentials(): Unit = {
+    Assume.assumeTrue(runAwsCredentialsTest)
+    val inConfig: ConfigSource = defaultInConfig
+      .set("access_key_id", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
+      .set("secret_access_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
 
-    config.getNested("in")
-      .set("region", EMBULK_DYNAMODB_TEST_REGION)
-      .set("table", EMBULK_DYNAMODB_TEST_TABLE)
+    doTest(inConfig)
+  }
+
+  @deprecated(since = "0.3.0")
+  @Test
+  def setAuthMethod_Basic_deprecated(): Unit = {
+    Assume.assumeTrue(runAwsCredentialsTest)
+    val inConfig: ConfigSource = defaultInConfig
+      .set("auth_method", "basic")
       .set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
       .set("secret_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
 
-    doTest(config)
+    doTest(inConfig)
   }
 
-  @Test(expected = classOf[PartialExecutionException])
-  def setAuthMethod_Basic_NotSet() {
-    val config = embulk.newConfigLoader().fromYamlFile(
-      new File("src/test/resources/yaml/authMethodBasic_Error.yml"))
+  @Test
+  def setAuthMethod_Basic(): Unit = {
+    Assume.assumeTrue(runAwsCredentialsTest)
+    val inConfig: ConfigSource = defaultInConfig
+      .set("auth_method", "basic")
+      .set("access_key_id", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
+      .set("secret_access_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
+
+    doTest(inConfig)
+  }
 
-    config.getNested("in")
-      .set("region", EMBULK_DYNAMODB_TEST_REGION)
-      .set("table", EMBULK_DYNAMODB_TEST_TABLE)
+  @deprecated(since = "0.3.0")
+  @Test(expected = classOf[ConfigException])
+  def throwIfSetAccessKeyAndAccessKeyId(): Unit = {
+    Assume.assumeTrue(runAwsCredentialsTest)
+    val inConfig: ConfigSource = defaultInConfig
+      .set("auth_method", "basic")
+      .set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
+      .set("access_key_id", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
+      .set("secret_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
 
-    doTest(config)
+    doTest(inConfig)
   }
 
-  @Test
-  def setAuthMethod_Env() {
-    val config = embulk.newConfigLoader().fromYamlFile(
-      new File("src/test/resources/yaml/authMethodEnv.yml"))
+  @deprecated(since = "0.3.0")
+  @Test(expected = classOf[ConfigException])
+  def throwIfSetSecretKeyAndSecretAccessKeyId(): Unit = {
+    Assume.assumeTrue(runAwsCredentialsTest)
+    val inConfig: ConfigSource = defaultInConfig
+      .set("auth_method", "basic")
+      .set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
+      .set("secret_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
+      .set("secret_access_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
 
-    config.getNested("in")
-      .set("region", EMBULK_DYNAMODB_TEST_REGION)
-      .set("table", EMBULK_DYNAMODB_TEST_TABLE)
+    doTest(inConfig)
+  }
 
-    doTest(config)
+  @Test(expected = classOf[ConfigException])
+  def setAuthMethod_Basic_NotSet(): Unit = {
+    val inConfig: ConfigSource = defaultInConfig
+      .set("auth_method", "basic")
+
+    doTest(inConfig)
   }
 
   @Test
-  def setAuthMethod_Profile() {
-    val config = embulk.newConfigLoader().fromYamlFile(
-      new File("src/test/resources/yaml/authMethodProfile.yml"))
+  def setAuthMethod_Env(): Unit = {
+    Assume.assumeTrue(runAwsCredentialsTest)
+    // NOTE: Requires to set the env vars like 'AWS_ACCESS_KEY_ID' and so on when testing.
+    val inConfig: ConfigSource = defaultInConfig
+      .set("auth_method", "env")
+
+    doTest(inConfig)
+  }
 
-    config.getNested("in")
-      .set("region", EMBULK_DYNAMODB_TEST_REGION)
-      .set("table", EMBULK_DYNAMODB_TEST_TABLE)
+  @Test
+  def setAuthMethod_Profile(): Unit = {
+    Assume.assumeTrue(runAwsCredentialsTest)
+    // NOTE: Requires to set credentials to '~/.aws' when testing.
+    val inConfig: ConfigSource = defaultInConfig
+      .set("auth_method", "profile")
       .set("profile_name", EMBULK_DYNAMODB_TEST_PROFILE_NAME)
 
-    doTest(config)
+    doTest(inConfig)
+  }
+
+  @Test(expected = classOf[IllegalArgumentException])
+  def setAuthMethod_Profile_NotExistProfileName(): Unit = {
+    val inConfig: ConfigSource = defaultInConfig
+      .set("auth_method", "profile")
+      .set("profile_name", "DO_NOT_EXIST")
+
+    doTest(inConfig)
   }
 
-  @Test(expected = classOf[PartialExecutionException])
-  def setAuthMethod_Profile_NotExistProfileName() {
-    val config = embulk.newConfigLoader().fromYamlFile(
-      new File("src/test/resources/yaml/authMethodProfile.yml"))
+  @Test
+  def setAuthMethod_assume_role(): Unit = {
+    Assume.assumeTrue(runAwsCredentialsTest)
+    val inConfig: ConfigSource = defaultInConfig
+      .set("auth_method", "assume_role")
+      .set("role_arn", EMBULK_DYNAMODB_TEST_ASSUME_ROLE_ROLE_ARN)
+      .set("role_session_name", "dummy")
+
+    doTest(inConfig)
+  }
 
-    config.getNested("in")
-      .set("region", EMBULK_DYNAMODB_TEST_REGION)
-      .set("table", EMBULK_DYNAMODB_TEST_TABLE)
-      .set("profile_name", "NotExistName")
+  @Test(expected = classOf[AWSSecurityTokenServiceException])
+  def setAuthMethod_assume_role_NotExistRoleArn(): Unit = {
+    Assume.assumeTrue(runAwsCredentialsTest)
+    val inConfig: ConfigSource = defaultInConfig
+      .set("auth_method", "assume_role")
+      .set("role_arn", "DO_NOT_EXIST")
+      .set("role_session_name", "dummy")
 
-    doTest(config)
+    doTest(inConfig)
   }
 }

data/src/test/scala/org/embulk/input/dynamodb/DynamodbQueryOperationTest.scala

@@ -0,0 +1,188 @@
+package org.embulk.input.dynamodb
+
+import com.amazonaws.services.dynamodbv2.model.{
+  AttributeDefinition,
+  AttributeValue,
+  CreateTableRequest,
+  KeySchemaElement,
+  KeyType,
+  ProvisionedThroughput,
+  PutItemRequest,
+  ScalarAttributeType
+}
+import org.embulk.config.ConfigSource
+import org.embulk.input.dynamodb.testutil.EmbulkTestBase
+import org.embulk.spi.util.Pages
+import org.hamcrest.CoreMatchers._
+import org.hamcrest.MatcherAssert.assertThat
+import org.junit.Test
+import org.msgpack.value.Value
+
+import scala.jdk.CollectionConverters._
+
+class DynamodbQueryOperationTest extends EmbulkTestBase {
+
+  private def testBackwardCompatibility(embulkInConfig: ConfigSource): Unit = {
+    cleanupTable("EMBULK_DYNAMODB_TEST_TABLE")
+    withDynamodb { dynamodb =>
+      dynamodb.createTable(
+        new CreateTableRequest()
+          .withTableName("EMBULK_DYNAMODB_TEST_TABLE")
+          .withAttributeDefinitions(
+            new AttributeDefinition()
+              .withAttributeName("pri-key")
+              .withAttributeType(ScalarAttributeType.S),
+            new AttributeDefinition()
+              .withAttributeName("sort-key")
+              .withAttributeType(ScalarAttributeType.N)
+          )
+          .withKeySchema(
+            new KeySchemaElement()
+              .withAttributeName("pri-key")
+              .withKeyType(KeyType.HASH),
+            new KeySchemaElement()
+              .withAttributeName("sort-key")
+              .withKeyType(KeyType.RANGE)
+          )
+          .withProvisionedThroughput(
+            new ProvisionedThroughput()
+              .withReadCapacityUnits(5L)
+              .withWriteCapacityUnits(5L)
+          )
+      )
+
+      dynamodb.putItem(
+        new PutItemRequest()
+          .withTableName("EMBULK_DYNAMODB_TEST_TABLE")
+          .withItem(
+            Map
+              .newBuilder[String, AttributeValue]
+              .addOne("pri-key", new AttributeValue().withS("key-1"))
+              .addOne("sort-key", new AttributeValue().withN("0"))
+              .addOne("doubleValue", new AttributeValue().withN("42.195"))
+              .addOne("boolValue", new AttributeValue().withBOOL(true))
+              .addOne(
+                "listValue",
+                new AttributeValue().withL(
+                  new AttributeValue().withS("list-value"),
+                  new AttributeValue().withN("123")
+                )
+              )
+              .addOne(
+                "mapValue",
+                new AttributeValue().withM(
+                  Map
+                    .newBuilder[String, AttributeValue]
+                    .addOne(
+                      "map-key-1",
+                      new AttributeValue().withS("map-value-1")
+                    )
+                    .addOne("map-key-2", new AttributeValue().withN("456"))
+                    .result()
+                    .asJava
+                )
+              )
+              .result()
+              .asJava
+          )
+      )
+
+      val path = embulk.createTempFile("csv")
+      val result = embulk
+        .inputBuilder()
+        .in(embulkInConfig)
+        .outputPath(path)
+        .preview()
+
+      val pages = result.getPages
+      val head = Pages.toObjects(result.getSchema, pages.get(0)).get(0)
+
+      assertThat(head(0).toString, is("key-1"))
+      assertThat(head(1).asInstanceOf[Long], is(0L))
+      assertThat(head(2).asInstanceOf[Double], is(42.195))
+      assertThat(head(3).asInstanceOf[Boolean], is(true))
+
+      val arrayValue = head(4).asInstanceOf[Value].asArrayValue()
+      assertThat(arrayValue.size(), is(2))
+      assertThat(arrayValue.get(0).asStringValue().toString, is("list-value"))
+      assertThat(arrayValue.get(1).asIntegerValue().asLong(), is(123L))
+
+      val mapValue = head(5).asInstanceOf[Value].asMapValue()
+      assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-1"))
+      assertThat(
+        mapValue
+          .entrySet()
+          .asScala
+          .filter(_.getKey.toString.equals("map-key-1"))
+          .head
+          .getValue
+          .toString,
+        is("map-value-1")
+      )
+      assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-2"))
+      assertThat(
+        mapValue
+          .entrySet()
+          .asScala
+          .filter(_.getKey.toString.equals("map-key-2"))
+          .head
+          .getValue
+          .asIntegerValue()
+          .asLong(),
+        is(456L)
+      )
+
+    }
+  }
+
+  @Test
+  def deprecatedQueryOperationTest(): Unit = {
+    val inConfig: ConfigSource = embulk.configLoader().fromYamlString(s"""
+         |type: dynamodb
+         |end_point: http://${dynamoDBHost}:${dynamoDBPort}/
+         |table: EMBULK_DYNAMODB_TEST_TABLE
+         |auth_method: basic
+         |access_key: dummy
+         |secret_key: dummy
+         |operation: query
+         |filters:
+         |  - {name: pri-key, type: string, condition: EQ, value: key-1}
+         |columns:
+         |  - {name: pri-key,     type: string}
+         |  - {name: sort-key,    type: long}
+         |  - {name: doubleValue, type: double}
+         |  - {name: boolValue,   type: boolean}
+         |  - {name: listValue,   type: json}
+         |  - {name: mapValue,    type: json}
+         |""".stripMargin)
+
+    testBackwardCompatibility(inConfig)
+  }
+
+  @Test
+  def keepTheSameBehaviourAsDeprecatedQueryOperationTest(): Unit = {
+    val inConfig: ConfigSource = embulk.configLoader().fromYamlString(s"""
+         |type: dynamodb
+         |endpoint: http://${dynamoDBHost}:${dynamoDBPort}/
+         |table: EMBULK_DYNAMODB_TEST_TABLE
+         |auth_method: basic
+         |access_key: dummy
+         |secret_key: dummy
+         |query:
+         |  key_condition_expression: "#x = :v"
+         |  expression_attribute_names:
+         |    "#x": pri-key
+         |  expression_attribute_values:
+         |    ":v": {S: key-1}
+         |columns:
+         |  - {name: pri-key,     type: string}
+         |  - {name: sort-key,    type: long}
+         |  - {name: doubleValue, type: double}
+         |  - {name: boolValue,   type: boolean}
+         |  - {name: listValue,   type: json}
+         |  - {name: mapValue,    type: json}
+         |""".stripMargin)
+
+    testBackwardCompatibility(inConfig)
+  }
+}