embulk-input-dynamodb 0.2.0 → 0.3.0

data/src/test/scala/org/embulk/input/dynamodb/DynamodbScanOperationTest.scala

@@ -0,0 +1,181 @@
+package org.embulk.input.dynamodb
+
+import com.amazonaws.services.dynamodbv2.model.{
+  AttributeDefinition,
+  AttributeValue,
+  CreateTableRequest,
+  KeySchemaElement,
+  KeyType,
+  ProvisionedThroughput,
+  PutItemRequest,
+  ScalarAttributeType
+}
+import org.embulk.config.ConfigSource
+import org.embulk.input.dynamodb.testutil.EmbulkTestBase
+import org.embulk.spi.util.Pages
+import org.hamcrest.CoreMatchers._
+import org.hamcrest.MatcherAssert.assertThat
+import org.junit.Test
+import org.msgpack.value.Value
+
+import scala.jdk.CollectionConverters._
+
+class DynamodbScanOperationTest extends EmbulkTestBase {
+
+  private def testBackwardCompatibility(embulkInConfig: ConfigSource): Unit = {
+    cleanupTable("EMBULK_DYNAMODB_TEST_TABLE")
+    withDynamodb { dynamodb =>
+      dynamodb.createTable(
+        new CreateTableRequest()
+          .withTableName("EMBULK_DYNAMODB_TEST_TABLE")
+          .withAttributeDefinitions(
+            new AttributeDefinition()
+              .withAttributeName("pri-key")
+              .withAttributeType(ScalarAttributeType.S),
+            new AttributeDefinition()
+              .withAttributeName("sort-key")
+              .withAttributeType(ScalarAttributeType.N)
+          )
+          .withKeySchema(
+            new KeySchemaElement()
+              .withAttributeName("pri-key")
+              .withKeyType(KeyType.HASH),
+            new KeySchemaElement()
+              .withAttributeName("sort-key")
+              .withKeyType(KeyType.RANGE)
+          )
+          .withProvisionedThroughput(
+            new ProvisionedThroughput()
+              .withReadCapacityUnits(5L)
+              .withWriteCapacityUnits(5L)
+          )
+      )
+
+      dynamodb.putItem(
+        new PutItemRequest()
+          .withTableName("EMBULK_DYNAMODB_TEST_TABLE")
+          .withItem(
+            Map
+              .newBuilder[String, AttributeValue]
+              .addOne("pri-key", new AttributeValue().withS("key-1"))
+              .addOne("sort-key", new AttributeValue().withN("0"))
+              .addOne("doubleValue", new AttributeValue().withN("42.195"))
+              .addOne("boolValue", new AttributeValue().withBOOL(true))
+              .addOne(
+                "listValue",
+                new AttributeValue().withL(
+                  new AttributeValue().withS("list-value"),
+                  new AttributeValue().withN("123")
+                )
+              )
+              .addOne(
+                "mapValue",
+                new AttributeValue().withM(
+                  Map
+                    .newBuilder[String, AttributeValue]
+                    .addOne(
+                      "map-key-1",
+                      new AttributeValue().withS("map-value-1")
+                    )
+                    .addOne("map-key-2", new AttributeValue().withN("456"))
+                    .result()
+                    .asJava
+                )
+              )
+              .result()
+              .asJava
+          )
+      )
+    }
+
+    val path = embulk.createTempFile("csv")
+    val result = embulk
+      .inputBuilder()
+      .in(embulkInConfig)
+      .outputPath(path)
+      .preview()
+
+    val pages = result.getPages
+    val head = Pages.toObjects(result.getSchema, pages.get(0)).get(0)
+
+    assertThat(head(0).toString, is("key-1"))
+    assertThat(head(1).asInstanceOf[Long], is(0L))
+    assertThat(head(2).asInstanceOf[Double], is(42.195))
+    assertThat(head(3).asInstanceOf[Boolean], is(true))
+
+    val arrayValue = head(4).asInstanceOf[Value].asArrayValue()
+    assertThat(arrayValue.size(), is(2))
+    assertThat(arrayValue.get(0).asStringValue().toString, is("list-value"))
+    assertThat(arrayValue.get(1).asIntegerValue().asLong(), is(123L))
+
+    val mapValue = head(5).asInstanceOf[Value].asMapValue()
+    assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-1"))
+    assertThat(
+      mapValue
+        .entrySet()
+        .asScala
+        .filter(_.getKey.toString.equals("map-key-1"))
+        .head
+        .getValue
+        .toString,
+      is("map-value-1")
+    )
+    assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-2"))
+    assertThat(
+      mapValue
+        .entrySet()
+        .asScala
+        .filter(_.getKey.toString.equals("map-key-2"))
+        .head
+        .getValue
+        .asIntegerValue()
+        .asLong(),
+      is(456L)
+    )
+  }
+
+  @Test
+  def deprecatedScanOperationTest(): Unit = {
+
+    val inConfig: ConfigSource = embulk.configLoader().fromYamlString(s"""
+        |type: dynamodb
+        |end_point: http://${dynamoDBHost}:${dynamoDBPort}/
+        |table: EMBULK_DYNAMODB_TEST_TABLE
+        |auth_method: basic
+        |access_key: dummy
+        |secret_key: dummy
+        |operation: scan
+        |columns:
+        |  - {name: pri-key,     type: string}
+        |  - {name: sort-key,    type: long}
+        |  - {name: doubleValue, type: double}
+        |  - {name: boolValue,   type: boolean}
+        |  - {name: listValue,   type: json}
+        |  - {name: mapValue,    type: json}
+        |""".stripMargin)
+
+    testBackwardCompatibility(inConfig)
+  }
+
+  @Test
+  def keepTheSameBehaviourAsDeprecatedScanOperationTest(): Unit = {
+    val inConfig: ConfigSource = embulk.configLoader().fromYamlString(s"""
+        |type: dynamodb
+        |endpoint: http://${dynamoDBHost}:${dynamoDBPort}/
+        |table: EMBULK_DYNAMODB_TEST_TABLE
+        |auth_method: basic
+        |access_key: dummy
+        |secret_key: dummy
+        |scan: {}
+        |columns:
+        |  - {name: pri-key,     type: string}
+        |  - {name: sort-key,    type: long}
+        |  - {name: doubleValue, type: double}
+        |  - {name: boolValue,   type: boolean}
+        |  - {name: listValue,   type: json}
+        |  - {name: mapValue,    type: json}
+        |""".stripMargin)
+
+    testBackwardCompatibility(inConfig)
+  }
+}

data/src/test/scala/org/embulk/input/dynamodb/testutil/EmbulkTestBase.scala

@@ -0,0 +1,85 @@
+package org.embulk.input.dynamodb.testutil
+
+import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials}
+import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
+import com.amazonaws.services.dynamodbv2.{
+  AmazonDynamoDB,
+  AmazonDynamoDBClientBuilder
+}
+import org.embulk.input.dynamodb.DynamodbInputPlugin
+import org.embulk.spi.InputPlugin
+import org.embulk.test.TestingEmbulk
+import org.junit.{After, Rule}
+
+import scala.util.{Failure, Success, Try}
+import scala.util.chaining._
+
+trait EmbulkTestBase {
+  val dynamoDBHost: String = "localhost"
+  val dynamoDBPort: Int = 8000
+
+  def withDynamodb[A](f: AmazonDynamoDB => A): A = {
+    AmazonDynamoDBClientBuilder
+      .standard()
+      .withEndpointConfiguration(
+        new EndpointConfiguration(
+          s"http://$dynamoDBHost:$dynamoDBPort",
+          "us-east-1"
+        )
+      )
+      .withCredentials(
+        new AWSStaticCredentialsProvider(
+          new BasicAWSCredentials("dummy", "dummy")
+        )
+      )
+      .build()
+      .pipe { client =>
+        try f(client)
+        finally client.shutdown()
+      }
+  }
+
+  def cleanupTable(name: String): Unit = {
+    withDynamodb { dynamodb =>
+      Try(dynamodb.describeTable(name)) match {
+        case Success(_) => dynamodb.deleteTable(name)
+        case Failure(_) => // Do nothing.
+      }
+    }
+  }
+
+  @Rule
+  def embulk: TestingEmbulk =
+    TestingEmbulk
+      .builder()
+      .registerPlugin(
+        classOf[InputPlugin],
+        "dynamodb",
+        classOf[DynamodbInputPlugin]
+      )
+      .build()
+
+  @After
+  def destroyEmbulk(): Unit = {
+    embulk.destroy()
+  }
+
+  def getEnvironmentVariableOrShowErrorMessage(name: String): String = {
+    try {
+      Option(System.getenv(name)) match {
+        case Some(x) => x
+        case None =>
+          throw new IllegalStateException(
+            s"Please set the environment variable: $name"
+          )
+      }
+    }
+    catch {
+      case e: Throwable =>
+        throw new IllegalStateException(
+          s"Please set the environment variable: $name",
+          e
+        )
+    }
+  }
+}

metadata

@@ -1,19 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-dynamodb
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Daisuke Higashi
+- Civitaspo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-03 00:00:00.000000000 Z
+date: 2020-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   name: bundler
@@ -21,79 +22,105 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
   name: rake
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
 description: '"Loads records from Dynamodb."'
 email:
 - daisuke.develop@gmail.com
+- civitaspo@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".github/workflows/master.yml"
+- ".github/workflows/test.yml"
+- ".gitignore"
+- ".scalafmt.conf"
+- CHANGELOG.md
 - LICENSE
 - README.md
 - build.gradle
-- circle.yml
-- config/checkstyle/checkstyle.xml
-- config/checkstyle/default.xml
+- classpath/aws-java-sdk-core-1.11.711.jar
+- classpath/aws-java-sdk-dynamodb-1.11.711.jar
+- classpath/aws-java-sdk-kms-1.11.711.jar
+- classpath/aws-java-sdk-s3-1.11.711.jar
+- classpath/aws-java-sdk-sts-1.11.711.jar
+- classpath/commons-codec-1.11.jar
+- classpath/commons-logging-1.2.jar
+- classpath/embulk-input-dynamodb-0.3.0.jar
+- classpath/httpclient-4.5.9.jar
+- classpath/httpcore-4.4.11.jar
+- classpath/ion-java-1.0.2.jar
+- classpath/jackson-databind-2.6.7.3.jar
+- classpath/jackson-dataformat-cbor-2.6.7.jar
+- classpath/jmespath-java-1.11.711.jar
+- classpath/scala-library-2.13.1.jar
+- classpath/zio-macros-core_2.13-0.6.2.jar
+- classpath/zio-stacktracer_2.13-1.0.0-RC17.jar
+- classpath/zio_2.13-1.0.0-RC17.jar
+- example/config-deprecated.yml
+- example/config-query-as-json.yml
+- example/config-query.yml
+- example/config-scan.yml
+- example/prepare_dynamodb_table.sh
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
 - gradlew.bat
 - lib/embulk/input/dynamodb.rb
-- src/main/scala/org/embulk/input/dynamodb/AttributeValueHelper.scala
-- src/main/scala/org/embulk/input/dynamodb/AwsCredentials.scala
-- src/main/scala/org/embulk/input/dynamodb/DynamoDBClient.scala
+- run_dynamodb_local.sh
+- settings.gradle
+- src/main/scala/org/embulk/input/dynamodb/DeprecatedDynamodbInputPlugin.scala
 - src/main/scala/org/embulk/input/dynamodb/DynamodbInputPlugin.scala
-- src/main/scala/org/embulk/input/dynamodb/Filter.scala
-- src/main/scala/org/embulk/input/dynamodb/FilterConfig.scala
 - src/main/scala/org/embulk/input/dynamodb/PluginTask.scala
-- src/main/scala/org/embulk/input/dynamodb/ope/AbstractOperation.scala
-- src/main/scala/org/embulk/input/dynamodb/ope/QueryOperation.scala
-- src/main/scala/org/embulk/input/dynamodb/ope/ScanOperation.scala
+- src/main/scala/org/embulk/input/dynamodb/aws/Aws.scala
+- src/main/scala/org/embulk/input/dynamodb/aws/AwsClientConfiguration.scala
+- src/main/scala/org/embulk/input/dynamodb/aws/AwsCredentials.scala
+- src/main/scala/org/embulk/input/dynamodb/aws/AwsDynamodbConfiguration.scala
+- src/main/scala/org/embulk/input/dynamodb/aws/AwsEndpointConfiguration.scala
+- src/main/scala/org/embulk/input/dynamodb/aws/HttpProxy.scala
+- src/main/scala/org/embulk/input/dynamodb/deprecated/AttributeValueHelper.scala
+- src/main/scala/org/embulk/input/dynamodb/deprecated/Filter.scala
+- src/main/scala/org/embulk/input/dynamodb/deprecated/FilterConfig.scala
+- src/main/scala/org/embulk/input/dynamodb/deprecated/ope/AbstractOperation.scala
+- src/main/scala/org/embulk/input/dynamodb/deprecated/ope/QueryOperation.scala
+- src/main/scala/org/embulk/input/dynamodb/deprecated/ope/ScanOperation.scala
+- src/main/scala/org/embulk/input/dynamodb/item/DynamodbAttributeValue.scala
+- src/main/scala/org/embulk/input/dynamodb/item/DynamodbAttributeValueEmbulkTypeTransformable.scala
+- src/main/scala/org/embulk/input/dynamodb/item/DynamodbAttributeValueType.scala
+- src/main/scala/org/embulk/input/dynamodb/item/DynamodbItemColumnVisitor.scala
+- src/main/scala/org/embulk/input/dynamodb/item/DynamodbItemConsumer.scala
+- src/main/scala/org/embulk/input/dynamodb/item/DynamodbItemIterator.scala
+- src/main/scala/org/embulk/input/dynamodb/item/DynamodbItemReader.scala
+- src/main/scala/org/embulk/input/dynamodb/item/DynamodbItemSchema.scala
+- src/main/scala/org/embulk/input/dynamodb/operation/AbstractDynamodbOperation.scala
+- src/main/scala/org/embulk/input/dynamodb/operation/DynamodbOperationProxy.scala
+- src/main/scala/org/embulk/input/dynamodb/operation/DynamodbQueryOperation.scala
+- src/main/scala/org/embulk/input/dynamodb/operation/DynamodbScanOperation.scala
+- src/main/scala/org/embulk/input/dynamodb/operation/EmbulkDynamodbOperation.scala
 - src/main/scala/org/embulk/input/dynamodb/package.scala
 - src/test/resources/json/test.json
 - src/test/resources/json/test.template
-- src/test/resources/yaml/authMethodBasic.yml
-- src/test/resources/yaml/authMethodBasic_Error.yml
-- src/test/resources/yaml/authMethodEnv.yml
-- src/test/resources/yaml/authMethodProfile.yml
-- src/test/resources/yaml/dynamodb-local-query.yml
-- src/test/resources/yaml/dynamodb-local-scan.yml
-- src/test/resources/yaml/notSetAuthMethod.yml
 - src/test/scala/org/embulk/input/dynamodb/AttributeValueHelperTest.scala
 - src/test/scala/org/embulk/input/dynamodb/AwsCredentialsTest.scala
-- src/test/scala/org/embulk/input/dynamodb/ope/QueryOperationTest.scala
-- src/test/scala/org/embulk/input/dynamodb/ope/ScanOperationTest.scala
-- test/create_table.sh
-- test/put_items.sh
-- test/run_dynamodb_local.sh
-- classpath/aws-java-sdk-core-1.10.43.jar
-- classpath/aws-java-sdk-dynamodb-1.10.43.jar
-- classpath/aws-java-sdk-kms-1.10.43.jar
-- classpath/aws-java-sdk-s3-1.10.43.jar
-- classpath/commons-codec-1.6.jar
-- classpath/commons-logging-1.1.3.jar
-- classpath/embulk-input-dynamodb-0.2.0.jar
-- classpath/httpclient-4.3.6.jar
-- classpath/httpcore-4.3.3.jar
-- classpath/scala-library-2.11.8.jar
+- src/test/scala/org/embulk/input/dynamodb/DynamodbQueryOperationTest.scala
+- src/test/scala/org/embulk/input/dynamodb/DynamodbScanOperationTest.scala
+- src/test/scala/org/embulk/input/dynamodb/testutil/EmbulkTestBase.scala
 homepage: https://github.com/lulichn/embulk-input-dynamodb
 licenses:
 - MIT
@@ -104,21 +131,18 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.6.8
 signing_key:
 specification_version: 4
 summary: Dynamodb input plugin for Embulk
-test_files:
-- test/create_table.sh
-- test/put_items.sh
-- test/run_dynamodb_local.sh
+test_files: []