embulk-input-dynamodb 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fb06f7f31e946edcde0e6fde44443d11c20b2f1b
-  data.tar.gz: 83f448ad04d7f262c9a1438c79091ce1c6ac2765
+  metadata.gz: 603d3c6513d86bf773182a8214d70cb668fcf2a6
+  data.tar.gz: 60d9256bb5fac99e56f5424997037323e98f3868
 SHA512:
-  metadata.gz: 00544333da3be446d2e479fe1055f5599ea50655f0ba77d6ba465b0d4a0b58316675ac269a0299aaddd8469599b6d43dc02f05fb9b2d07b5daa25d68b4448f9f
-  data.tar.gz: 8960d06019a11c3e291f7724b0404e9dbd44df041a91cbe4bdaccb9b2e430a5213d60ff597aafab99ff9d6daa3409bead9b634126fa59a2712338e3084c6f318
+  metadata.gz: 55d4bdb5960bdf069804360dd0ca9b627bb7cfab0be6bb92c1012b14335019f3c1fd9dd31548fcf9b11b3e22d207efc35b8faf1172b7a5be531d62d39966d61e
+  data.tar.gz: 9ae148915f273eca767e740b0cd523f7d355b6f793644106320b592695124682769c47df41b643139d91111d9e541b65e692a32d31b9072a04ea1721b1fd5b35

data/README.md

@@ -9,7 +9,7 @@
 
 
 ## Configuration
-- **auth_method**: AWS Credential Type.
+- **auth_method**: AWS Credential Type.  
 Available values options are: `basic`, `env`, `instance`, `profile`, `properties`
   - **basic**: AWS access key and secret access key
   - **env**: Environment variables
@@ -21,13 +21,15 @@ Available values options are: `basic`, `env`, `instance`, `profile`, `properties
   - **secret_key**: AWS secret key (string, required)
 - If **auth_method** is set `profile`
   - **profile_name**: The name of a local configuration profile (string, optional)
-- **region**: Region Name (string, default: ap-northeast-1)
+- **region**: Region Name (string, optional)
+- **end_point**: EndPoint URL (string, optional)  
+`end_point` has priority when `region` and `end_point` are specified.
 - **table**: Table Name (string, required)
 - **scan_limit**: DynamoDB 1time Scan Query size limit (Int, optional) 
 - **record_limit**: Max Record Search limit (Long, optional) 
 - **columns**: a key-value pairs where key is a column name and value is options for the column (required)
   - **name**: Column name.
-  - **type**: Column values are converted to this embulk type.
+  - **type**: Column values are converted to this embulk type.  
   Available values options are: `boolean`, `long`, `double`, `string`, `json`
 - **filters**: query filter (optional)
 
@@ -46,7 +48,7 @@ in:
     - {name: ColumnB, type: double}
     - {name: ColumnC, type: string}
     - {name: ColumnD, type: boolean}
-    - {name: ColumnE, type: json}
+    - {name: ColumnE, type: json}  # DynamoDB Map, List and Set Column Type are json.
   filters:
     - {name: ColumnA, type: long, condition: BETWEEN, value: 10000, value2: 20000}
     - {name: ColumnC, type: string, condition: EQ, value: foobar}
@@ -55,6 +57,13 @@ out:
   type: stdout
 ```
 
+## Try
+
+```
+$ ./gradlew classpath
+$ embulk preview -I lib your-sample.yml
+```
+
 ## Build
 
 ```

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.1"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/circle.yml

@@ -1,8 +1,16 @@
+machine:
+  services:
+    - docker
+
 general:
   artifacts:
     - "~/embulk-input-dynamodb/build/reports/tests"
 
 test:
+  pre:
+    - sh ./test/run_dynamodb_local.sh; sleep 2
+    - sh ./test/create_table.sh
+    - sh ./test/put_items.sh
   post:
-  - mkdir -p $CIRCLE_TEST_REPORTS/junit/
-  - find . -type f -regex ".*/test-results/*/.*xml" -exec cp {} $CIRCLE_TEST_REPORTS/junit/ \;
+    - mkdir -p $CIRCLE_TEST_REPORTS/junit/
+    - find . -type f -regex ".*/test-results/*/.*xml" -exec cp {} $CIRCLE_TEST_REPORTS/junit/ \;

data/src/main/scala/org/embulk/input/dynamodb/AwsCredentials.scala

@@ -60,12 +60,4 @@ object AwsCredentials {
       }
     }
   }
-
-  private def require[A](value: Optional[A], message: String): A = {
-    if (value.isPresent) {
-      value.get()
-    } else {
-      throw new ConfigException("Required option is not set: " + message)
-    }
-  }
 }

data/src/main/scala/org/embulk/input/dynamodb/DynamoDBUtil.scala

@@ -6,20 +6,27 @@ import com.amazonaws.ClientConfiguration
 import com.amazonaws.regions.Regions
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient
 import com.amazonaws.services.dynamodbv2.model.{AttributeValue, Condition, ScanRequest, ScanResult}
+import org.embulk.config.ConfigException
 import org.embulk.spi._
 import org.embulk.spi.`type`.Types
-import org.msgpack.value.{ValueFactory, Value}
+import org.msgpack.value.{Value, ValueFactory}
 
-import scala.collection.JavaConversions._
 import scala.collection.JavaConverters._
 
 object DynamoDBUtil {
   def createClient(task: PluginTask): AmazonDynamoDBClient = {
-    new AmazonDynamoDBClient(
+    val client = new AmazonDynamoDBClient(
       AwsCredentials.getCredentialsProvider(task),
       new ClientConfiguration()
         .withMaxConnections(50))  // SDK Default Value
-      .withRegion(Regions.fromName(task.getRegion))
+
+    if (task.getEndPoint.isPresent) {
+      client.withEndpoint(task.getEndPoint.get())
+    } else if (task.getRegion.isPresent) {
+      client.withRegion(Regions.fromName(task.getRegion.get()))
+    } else {
+      throw new ConfigException("At least one of EndPoint or Region must be set")
+    }
   }
 
 
@@ -34,10 +41,10 @@ object DynamoDBUtil {
 
     val attributes: JList[String] = new JArrayList[String]()
 
-    schema.getColumns.foreach { column =>
+    schema.getColumns.asScala.foreach { column =>
       attributes.add(column.getName)
     }
-    val scanFilter: JMap[String, Condition] = createScanFilter(task)
+    val scanFilter: JMap[String, Condition] = createScanFilter(task).asJava
     var evaluateKey: JMap[String, AttributeValue] = null
 
     val scanLimit: Long   = task.getScanLimit
@@ -60,8 +67,8 @@ object DynamoDBUtil {
       val result: ScanResult = client.scan(request)
       evaluateKey = result.getLastEvaluatedKey
 
-      result.getItems.foreach { item =>
-        schema.getColumns.foreach { column =>
+      result.getItems.asScala.foreach { item =>
+        schema.getColumns.asScala.foreach { column =>
           val value = item.asScala.get(column.getName)
           column.getType match {
             case Types.STRING =>
@@ -97,7 +104,7 @@ object DynamoDBUtil {
     val filterMap = collection.mutable.HashMap[String, Condition]()
 
     Option(task.getFilters.orNull).map { filters =>
-      filters.getFilters.map { filter =>
+      filters.getFilters.asScala.map { filter =>
         val attributeValueList = collection.mutable.ArrayBuffer[AttributeValue]()
         attributeValueList += createAttributeValue(filter.getType, filter.getValue)
         Option(filter.getValue2).map { value2 =>
@@ -105,7 +112,7 @@ object DynamoDBUtil {
 
         filterMap += filter.getName -> new Condition()
           .withComparisonOperator(filter.getCondition)
-          .withAttributeValueList(attributeValueList)
+          .withAttributeValueList(attributeValueList.asJava)
       }
     }
 
@@ -138,7 +145,7 @@ object DynamoDBUtil {
     value.map(_.getN.toDouble).getOrElse(0D)
 
   implicit private def BooleanConvert(value: Option[AttributeValue]): Boolean =
-    value.exists(_.getS.toBoolean)
+    value.exists(_.getBOOL)
 
   implicit private def JsonConvert(value: Option[AttributeValue]): Value = {
     value.map { attr =>

data/src/main/scala/org/embulk/input/dynamodb/PluginTask.scala

@@ -22,8 +22,12 @@ trait PluginTask extends Task {
   def getProfileName: Optional[String]
 
   @Config("region")
-  @ConfigDefault("ap-northeast-1")
-  def getRegion: String
+  @ConfigDefault("null")
+  def getRegion: Optional[String]
+
+  @Config("end_point")
+  @ConfigDefault("null")
+  def getEndPoint: Optional[String]
 
   @Config("scan_limit")
   @ConfigDefault("0")

data/src/main/scala/org/embulk/input/dynamodb/package.scala

@@ -0,0 +1,14 @@
+package org.embulk.input
+
+import com.google.common.base.Optional
+import org.embulk.config.ConfigException
+
+package object dynamodb {
+  def require[A](value: Optional[A], message: String): A = {
+    if (value.isPresent) {
+      value.get()
+    } else {
+      throw new ConfigException("Required option is not set: " + message)
+    }
+  }
+}

data/src/test/resources/yaml/dynamodb-local.yml

@@ -0,0 +1,23 @@
+in:
+  type: dynamodb
+  end_point: http://localhost:8000/
+  table: ENV_VAR
+  auth_method: basic
+  access_key: dummy
+  secret_key: dummy
+  columns:
+    - {name: pri-key,     type: string}
+    - {name: sort-key,    type: long}
+    - {name: doubleValue, type: double}
+    - {name: boolValue,   type: boolean}
+    - {name: listValue,   type: json}
+    - {name: mapValue,    type: json}
+
+out:
+  type: file
+  path_prefix: dynamodb-local-result
+  file_ext: tsv
+  formatter:
+    type: csv
+    delimiter: "\t"
+    header_line: false

data/src/test/scala/org/embulk/input/dynamodb/AttributeValueHelperTest.scala

@@ -138,12 +138,18 @@ class AttributeValueHelperTest {
   }
 
   def attr[A](value: A)(implicit f: A=> AttributeValue): AttributeValue = f(value)
-  implicit def StringAttributeValue(value: String): AttributeValue   = new AttributeValue().withS(value)
-  implicit def IntegerAttributeValue(value: Int): AttributeValue     = new AttributeValue().withN(value.toString)
-  implicit def LongAttributeValue(value: Long): AttributeValue       = new AttributeValue().withN(value.toString)
-  implicit def FloatAttributeValue(value: Float): AttributeValue       = new AttributeValue().withN(value.toString)
-  implicit def DoubleAttributeValue(value: Double): AttributeValue       = new AttributeValue().withN(value.toString)
-  implicit def BooleanAttributeValue(value: Boolean): AttributeValue = new AttributeValue().withBOOL(value)
+  implicit def StringAttributeValue(value: String): AttributeValue
+    = new AttributeValue().withS(value)
+  implicit def IntegerAttributeValue(value: Int): AttributeValue
+    = new AttributeValue().withN(value.toString)
+  implicit def LongAttributeValue(value: Long): AttributeValue
+    = new AttributeValue().withN(value.toString)
+  implicit def FloatAttributeValue(value: Float): AttributeValue
+    = new AttributeValue().withN(value.toString)
+  implicit def DoubleAttributeValue(value: Double): AttributeValue
+    = new AttributeValue().withN(value.toString)
+  implicit def BooleanAttributeValue(value: Boolean): AttributeValue
+    = new AttributeValue().withBOOL(value)
   implicit def MapAttributeValue(value: Map[String, AttributeValue]): AttributeValue
     = new AttributeValue().withM(value.asJava)
   implicit def ListAttributeValue(value: List[AttributeValue]): AttributeValue

data/src/test/scala/org/embulk/input/dynamodb/DynamoDBUtilTest.scala

@@ -0,0 +1,81 @@
+package org.embulk.input.dynamodb
+
+import java.io.File
+import java.nio.charset.Charset
+import java.nio.file.{FileSystems, Files}
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.google.inject.{Binder, Module}
+import org.embulk.EmbulkEmbed
+import org.embulk.config.ConfigSource
+import org.embulk.plugin.InjectedPluginSource
+import org.embulk.spi.InputPlugin
+import org.hamcrest.CoreMatchers._
+import org.junit.Assert._
+import org.junit.{Before, Test}
+
+class DynamoDBUtilTest {
+  private var embulk: EmbulkEmbed = null
+
+  private var EMBULK_DYNAMODB_TEST_TABLE: String = null
+  private var mapper: ObjectMapper = null
+
+  @Before
+  def createResources() {
+    // Get Environments
+    EMBULK_DYNAMODB_TEST_TABLE = System.getenv("EMBULK_DYNAMODB_TEST_TABLE")
+
+    val bootstrap = new EmbulkEmbed.Bootstrap()
+    bootstrap.addModules(new Module {
+      def configure(binder: Binder): Unit = {
+        InjectedPluginSource.registerPluginTo(binder,
+          classOf[InputPlugin],
+          "dynamodb",
+          classOf[DynamodbInputPlugin])
+      }
+    })
+
+    embulk = bootstrap.initializeCloseable()
+
+    mapper = new ObjectMapper()
+  }
+
+
+  def doTest(config: ConfigSource) {
+    embulk.run(config)
+
+    val fs = FileSystems.getDefault
+    val lines = Files.readAllLines(fs.getPath("dynamodb-local-result000.00.tsv"), Charset.forName("UTF-8"))
+    assertEquals(lines.size, 1)
+
+    val head = lines.get(0)
+    val values = head.split("\t")
+
+    assertThat(values(0), is("key-1"))
+    assertThat(values(1), is("0"))
+    assertThat(values(2), is("42.195"))
+    assertThat(values(3), is("true"))
+
+    val listValue = mapper.readValue(values(4).replaceAll("\"(?!\")", ""), classOf[java.util.List[Object]])
+    assertThat(listValue.size(), is(2))
+    assertThat(listValue.get(0).asInstanceOf[String], is("list-value"))
+    assertThat(listValue.get(1).asInstanceOf[Int], is(123))
+
+    val mapValue = mapper.readValue(values(5).replaceAll("\"(?!\")", ""), classOf[java.util.Map[String, Object]])
+    assert(mapValue.containsKey("map-key-1"))
+    assertThat(mapValue.get("map-key-1").asInstanceOf[String], is("map-value-1"))
+    assert(mapValue.containsKey("map-key-2"))
+    assertThat(mapValue.get("map-key-2").asInstanceOf[Int], is(456))
+  }
+
+  @Test
+  def scanTest() {
+    val config = embulk.newConfigLoader().fromYamlFile(
+      new File("src/test/resources/yaml/dynamodb-local.yml"))
+
+    config.getNested("in")
+      .set("table", EMBULK_DYNAMODB_TEST_TABLE)
+
+    doTest(config)
+  }
+}

data/test/create_table.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+aws dynamodb create-table \
+	--table-name='EMBULK_DYNAMODB_TEST_TABLE' \
+	--attribute-definitions='[
+		{"AttributeName":"pri-key","AttributeType":"S"},
+		{"AttributeName":"sort-key","AttributeType":"N"}
+	]' \
+	--key-schema='[
+		{"AttributeName":"pri-key","KeyType":"HASH"},
+		{"AttributeName":"sort-key","KeyType":"RANGE"}
+	]' \
+	--provisioned-throughput='{"ReadCapacityUnits":5, "WriteCapacityUnits":5}' \
+	--endpoint-url http://localhost:8000 \
+	--region us-east-1
+

data/test/put_items.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+aws dynamodb put-item \
+	--table-name='EMBULK_DYNAMODB_TEST_TABLE' \
+	--item='{
+		"pri-key" : { "S" : "key-1" },
+		"sort-key" : { "N" : "0" },
+		"doubleValue" : { "N" : "42.195" },
+		"boolValue" : { "BOOL" : true },
+		"listValue" : { "L":
+			[
+				{ "S" : "list-value"},
+				{ "N" : "123"}
+			]
+		},
+		"mapValue" : { "M":
+			{
+				"map-key-1" : { "S" : "map-value-1" },
+				"map-key-2" : { "N" : "456" }
+			}
+		}
+	}' \
+	--endpoint-url http://localhost:8000 \
+	--region us-east-1
+

data/test/run_dynamodb_local.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+docker run -d \
+	-p 8000:8000 \
+	-v $PWD/dynamodb-local:/data \
+	--env DYNAMO_OPT='-dbPath /data -sharedDb' \
+	lulichn/dynamodb-local

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-dynamodb
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Daisuke Higashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-04 00:00:00.000000000 Z
+date: 2016-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -64,25 +64,31 @@ files:
 - src/main/scala/org/embulk/input/dynamodb/Filter.scala
 - src/main/scala/org/embulk/input/dynamodb/FilterConfig.scala
 - src/main/scala/org/embulk/input/dynamodb/PluginTask.scala
+- src/main/scala/org/embulk/input/dynamodb/package.scala
 - src/test/resources/json/test.json
 - src/test/resources/json/test.template
 - src/test/resources/yaml/authMethodBasic.yml
 - src/test/resources/yaml/authMethodBasic_Error.yml
 - src/test/resources/yaml/authMethodEnv.yml
 - src/test/resources/yaml/authMethodProfile.yml
+- src/test/resources/yaml/dynamodb-local.yml
 - src/test/resources/yaml/notSetAuthMethod.yml
 - src/test/scala/org/embulk/input/dynamodb/AttributeValueHelperTest.scala
 - src/test/scala/org/embulk/input/dynamodb/AwsCredentialsTest.scala
-- classpath/aws-java-sdk-core-1.10.43.jar
+- src/test/scala/org/embulk/input/dynamodb/DynamoDBUtilTest.scala
+- test/create_table.sh
+- test/put_items.sh
+- test/run_dynamodb_local.sh
+- classpath/commons-logging-1.1.3.jar
+- classpath/commons-codec-1.6.jar
+- classpath/scala-library-2.11.7.jar
 - classpath/aws-java-sdk-dynamodb-1.10.43.jar
+- classpath/embulk-input-dynamodb-0.1.1.jar
 - classpath/aws-java-sdk-kms-1.10.43.jar
-- classpath/aws-java-sdk-s3-1.10.43.jar
-- classpath/commons-codec-1.6.jar
-- classpath/commons-logging-1.1.3.jar
-- classpath/embulk-input-dynamodb-0.1.0.jar
-- classpath/httpclient-4.3.6.jar
 - classpath/httpcore-4.3.3.jar
-- classpath/scala-library-2.11.7.jar
+- classpath/httpclient-4.3.6.jar
+- classpath/aws-java-sdk-core-1.10.43.jar
+- classpath/aws-java-sdk-s3-1.10.43.jar
 homepage: https://github.com/lulichn/embulk-input-dynamodb
 licenses:
 - MIT
@@ -107,4 +113,7 @@ rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: Dynamodb input plugin for Embulk
-test_files: []
+test_files:
+- test/create_table.sh
+- test/put_items.sh
+- test/run_dynamodb_local.sh