embulk-input-dynamodb 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 603d3c6513d86bf773182a8214d70cb668fcf2a6
-  data.tar.gz: 60d9256bb5fac99e56f5424997037323e98f3868
+  metadata.gz: 324f33092c5bb362ecf9a804329e56eda0e545be
+  data.tar.gz: 00c350b4d52c76adf8291bd6abeee9d9b284eb10
 SHA512:
-  metadata.gz: 55d4bdb5960bdf069804360dd0ca9b627bb7cfab0be6bb92c1012b14335019f3c1fd9dd31548fcf9b11b3e22d207efc35b8faf1172b7a5be531d62d39966d61e
-  data.tar.gz: 9ae148915f273eca767e740b0cd523f7d355b6f793644106320b592695124682769c47df41b643139d91111d9e541b65e692a32d31b9072a04ea1721b1fd5b35
+  metadata.gz: 09103d2d4bdbc12d22f51318b7e6be74bba3420d9ce1e0003e78c55b77f242c03e5a74d5f9a95233be08539bb2c0cce8fbb4b8ad151104f5730c3f76f3edfd7d
+  data.tar.gz: c7d278c4eac6260264652ce74c6901d416dc3a4ebd287ce1162d16e911628f18430aad3b34205524b7e53e4b10ae32958cbe26872d359124b883bad67f78eb93

data/README.md

@@ -24,17 +24,27 @@ Available values options are: `basic`, `env`, `instance`, `profile`, `properties
 - **region**: Region Name (string, optional)
 - **end_point**: EndPoint URL (string, optional)  
 `end_point` has priority when `region` and `end_point` are specified.
+- **operation**: Operation Type (string, required)  
+Available types are: `scan`, `query`
 - **table**: Table Name (string, required)
-- **scan_limit**: DynamoDB 1time Scan Query size limit (Int, optional) 
-- **record_limit**: Max Record Search limit (Long, optional) 
+- **filters**: Query Filters  
+Required to `query` operation. Optional for `scan`.  
+  - **name**: Column name.
+  - **type**: Column type.
+  - **condition**: Comparison Operator.
+  - **value(s)**: Attribute Value(s).
+- **limit**: DynamoDB 1-time Scan/Query Operation size limit (Int, optional)
+- **scan_limit**: DynamoDB 1-time Scan Query size limit (Deprecated, Int, optional)
+- **record_limit**: Max Record Search limit (Long, optional)
 - **columns**: a key-value pairs where key is a column name and value is options for the column (required)
   - **name**: Column name.
   - **type**: Column values are converted to this embulk type.  
   Available values options are: `boolean`, `long`, `double`, `string`, `json`
-- **filters**: query filter (optional)
 
 ## Example
 
+- Scan Operation
+
 ```yaml
 in:
   type: dynamodb
@@ -42,6 +52,7 @@ in:
   access_key: YOUR_ACCESS_KEY
   secret_key: YOUR_SECRET_KEY
   region: ap-northeast-1
+  operation: scan
   table: YOUR_TABLE_NAME
   columns:
     - {name: ColumnA, type: long}
@@ -57,11 +68,33 @@ out:
   type: stdout
 ```
 
+- Query Operation
+
+```yaml
+in:
+  type: dynamodb
+  auth_method: env
+  region: ap-northeast-1
+  operation: query
+  table: YOUR_TABLE_NAME
+  columns:
+    - {name: ColumnA, type: long}
+    - {name: ColumnB, type: double}
+    - {name: ColumnC, type: string}
+    - {name: ColumnD, type: boolean}
+    - {name: ColumnE, type: json}
+  filters:
+    - {name: ColumnA, type: long, condition: EQ, value: 10000}
+
+out:
+  type: stdout
+```
+
 ## Try
 
 ```
 $ ./gradlew classpath
-$ embulk preview -I lib your-sample.yml
+$ embulk preview -I lib your-config.yml
 ```
 
 ## Build

data/build.gradle

@@ -14,22 +14,22 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.2.0"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile "org.scala-lang:scala-library:2.11.7"
+    compile "org.scala-lang:scala-library:2.11.8"
 
-    compile  "org.embulk:embulk-core:0.8.6"
-    provided "org.embulk:embulk-core:0.8.6"
+    compile  "org.embulk:embulk-core:0.8.13"
+    provided "org.embulk:embulk-core:0.8.13"
 
     compile "com.amazonaws:aws-java-sdk-dynamodb:1.10.43"
 
     testCompile "junit:junit:4.+"
-    testCompile "org.embulk:embulk-standards:0.8.6"
-    testCompile "org.embulk:embulk-core:0.8.6:tests"
+    testCompile "org.embulk:embulk-standards:0.8.13"
+    testCompile "org.embulk:embulk-core:0.8.13:tests"
 }
 
 compileScala {

data/src/main/scala/org/embulk/input/dynamodb/DynamoDBClient.scala

@@ -0,0 +1,23 @@
+package org.embulk.input.dynamodb
+
+import com.amazonaws.ClientConfiguration
+import com.amazonaws.regions.Regions
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient
+import org.embulk.config.ConfigException
+
+object DynamoDBClient {
+  def create(task: PluginTask): AmazonDynamoDBClient = {
+    val client = new AmazonDynamoDBClient(
+      AwsCredentials.getCredentialsProvider(task),
+      new ClientConfiguration()
+        .withMaxConnections(50))  // SDK Default Value
+
+    if (task.getEndPoint.isPresent) {
+      client.withEndpoint(task.getEndPoint.get())
+    } else if (task.getRegion.isPresent) {
+      client.withRegion(Regions.fromName(task.getRegion.get()))
+    } else {
+      throw new ConfigException("At least one of EndPoint or Region must be set")
+    }
+  }
+}

data/src/main/scala/org/embulk/input/dynamodb/DynamodbInputPlugin.scala

@@ -4,6 +4,7 @@ import java.util.{List => JList}
 
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient
 import org.embulk.config._
+import org.embulk.input.dynamodb.ope.{QueryOperation, ScanOperation}
 import org.embulk.spi._
 
 class DynamodbInputPlugin extends InputPlugin {
@@ -24,8 +25,13 @@ class DynamodbInputPlugin extends InputPlugin {
   def run(taskSource: TaskSource, schema: Schema, taskIndex: Int, output: PageOutput): TaskReport = {
     val task: PluginTask = taskSource.loadTask(classOf[PluginTask])
 
-    implicit val client: AmazonDynamoDBClient = DynamoDBUtil.createClient(task)
-    DynamoDBUtil.scan(task, schema, output)
+    val client: AmazonDynamoDBClient = DynamoDBClient.create(task)
+
+    val ope = task.getOperation.toLowerCase match {
+      case "scan"  => new ScanOperation(client)
+      case "query" => new QueryOperation(client)
+    }
+    ope.execute(task, schema, output)
 
     Exec.newTaskReport()
   }

data/src/main/scala/org/embulk/input/dynamodb/PluginTask.scala

@@ -29,6 +29,13 @@ trait PluginTask extends Task {
   @ConfigDefault("null")
   def getEndPoint: Optional[String]
 
+  @Config("operation")
+  def getOperation: String
+
+  @Config("limit")
+  @ConfigDefault("0")
+  def getLimit: Long
+
   @Config("scan_limit")
   @ConfigDefault("0")
   def getScanLimit: Long

data/src/main/scala/org/embulk/input/dynamodb/ope/AbstractOperation.scala

@@ -0,0 +1,101 @@
+package org.embulk.input.dynamodb.ope
+
+import com.amazonaws.services.dynamodbv2.model.{AttributeValue, Condition}
+import org.embulk.input.dynamodb.{AttributeValueHelper, PluginTask}
+import org.embulk.spi._
+import org.embulk.spi.`type`.Types
+import org.msgpack.value.{Value, ValueFactory}
+
+import scala.collection.JavaConverters._
+
+abstract class AbstractOperation {
+  def execute(task: PluginTask, schema: Schema, output: PageOutput): Unit
+
+  def getLimit(limit: Long, recordLimit: Long, recordCount: Long): Int = {
+    if (limit > 0 && recordLimit > 0) {
+      math.min(limit, recordLimit - recordCount).toInt
+    } else if (limit > 0 || recordLimit > 0) {
+      math.max(limit, recordLimit).toInt
+    } else { 0 }
+  }
+
+  def createFilters(task: PluginTask): Map[String, Condition] = {
+    val filterMap = collection.mutable.HashMap[String, Condition]()
+
+    Option(task.getFilters.orNull).map { filters =>
+      filters.getFilters.asScala.map { filter =>
+        val attributeValueList = collection.mutable.ArrayBuffer[AttributeValue]()
+        attributeValueList += createAttributeValue(filter.getType, filter.getValue)
+        Option(filter.getValue2).map { value2 =>
+          attributeValueList+= createAttributeValue(filter.getType, value2) }
+
+        filterMap += filter.getName -> new Condition()
+          .withComparisonOperator(filter.getCondition)
+          .withAttributeValueList(attributeValueList.asJava)
+      }
+    }
+
+    filterMap.toMap
+  }
+
+  def createAttributeValue(t: String, v: String): AttributeValue = {
+    t match {
+      case "string" =>
+        new AttributeValue().withS(v)
+      case "long" | "double" =>
+        new AttributeValue().withN(v)
+      case "boolean" =>
+        new AttributeValue().withBOOL(v.toBoolean)
+    }
+  }
+
+  def write(pageBuilder: PageBuilder, schema: Schema, items: Seq[Map[String, AttributeValue]]): Long = {
+    var count = 0
+
+    items.foreach { item =>
+      schema.getColumns.asScala.foreach { column =>
+        val value = item.get(column.getName)
+        column.getType match {
+          case Types.STRING =>
+            convert(column, value, pageBuilder.setString)
+          case Types.LONG =>
+            convert(column, value, pageBuilder.setLong)
+          case Types.DOUBLE =>
+            convert(column, value, pageBuilder.setDouble)
+          case Types.BOOLEAN =>
+            convert(column, value, pageBuilder.setBoolean)
+          case Types.JSON =>
+            convert(column, value, pageBuilder.setJson)
+          case _ => /* Do nothing */
+        }
+      }
+      pageBuilder.addRecord()
+      count += 1
+    }
+
+    count
+  }
+
+  def convert[A](column: Column,
+                   value: Option[AttributeValue],
+                   f: (Column, A) => Unit)(implicit f1: Option[AttributeValue] => A): Unit =
+    f(column, f1(value))
+
+  implicit def StringConvert(value: Option[AttributeValue]): String =
+    value.map(_.getS).getOrElse("")
+
+  implicit def LongConvert(value: Option[AttributeValue]): Long =
+    value.map(_.getN.toLong).getOrElse(0L)
+
+  implicit def DoubleConvert(value: Option[AttributeValue]): Double =
+    value.map(_.getN.toDouble).getOrElse(0D)
+
+  implicit def BooleanConvert(value: Option[AttributeValue]): Boolean =
+    value.exists(_.getBOOL)
+
+  implicit def JsonConvert(value: Option[AttributeValue]): Value = {
+    value.map { attr =>
+      AttributeValueHelper.decodeToValue(attr)
+    }.getOrElse(ValueFactory.newNil())
+  }
+}

data/src/main/scala/org/embulk/input/dynamodb/ope/QueryOperation.scala

@@ -0,0 +1,50 @@
+package org.embulk.input.dynamodb.ope
+
+import java.util.{List => JList, Map => JMap}
+
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient
+import com.amazonaws.services.dynamodbv2.model.{AttributeValue, Condition, QueryRequest, QueryResult}
+import org.embulk.input.dynamodb.PluginTask
+import org.embulk.spi.{BufferAllocator, PageBuilder, PageOutput, Schema}
+
+import scala.collection.JavaConverters._
+
+class QueryOperation(client: AmazonDynamoDBClient) extends AbstractOperation {
+  override def execute(task: PluginTask,
+                       schema: Schema,
+                       output: PageOutput): Unit =
+  {
+    val allocator: BufferAllocator = task.getBufferAllocator
+    val pageBuilder: PageBuilder = new PageBuilder(allocator, schema, output)
+
+    val attributes: JList[String] = schema.getColumns.asScala.map(_.getName).asJava
+    val conditions: JMap[String, Condition] = createFilters(task).asJava
+    var evaluateKey: JMap[String, AttributeValue] = null
+
+    val limit: Long   = math.max(task.getScanLimit, task.getLimit)
+    val recordLimit: Long = task.getRecordLimit
+    var recordCount: Long = 0
+
+    do {
+      val batchSize = getLimit(limit, recordLimit, recordCount)
+
+      val request: QueryRequest = new QueryRequest()
+        .withTableName(task.getTable)
+        .withAttributesToGet(attributes)
+        .withKeyConditions(conditions)
+        .withExclusiveStartKey(evaluateKey)
+
+      if (batchSize > 0) {
+        request.setLimit(batchSize)
+      }
+
+      val result: QueryResult = client.query(request)
+      evaluateKey = result.getLastEvaluatedKey
+
+      val items = result.getItems.asScala.map(_.asScala.toMap)
+      recordCount += write(pageBuilder, schema, items)
+    } while(evaluateKey != null && (recordLimit == 0 || recordLimit > recordCount))
+
+    pageBuilder.finish()
+  }
+}

data/src/main/scala/org/embulk/input/dynamodb/ope/ScanOperation.scala

@@ -0,0 +1,51 @@
+package org.embulk.input.dynamodb.ope
+
+import java.util.{List => JList, Map => JMap}
+
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient
+import com.amazonaws.services.dynamodbv2.model.{AttributeValue, Condition, ScanRequest, ScanResult}
+import org.embulk.input.dynamodb.PluginTask
+import org.embulk.spi.{BufferAllocator, PageBuilder, PageOutput, Schema}
+
+import scala.collection.JavaConverters._
+
+class ScanOperation(client: AmazonDynamoDBClient) extends AbstractOperation {
+  override def execute(
+            task: PluginTask,
+            schema: Schema,
+            output: PageOutput): Unit =
+  {
+    val allocator: BufferAllocator = task.getBufferAllocator
+    val pageBuilder: PageBuilder = new PageBuilder(allocator, schema, output)
+
+    val attributes: JList[String] = schema.getColumns.asScala.map(_.getName).asJava
+    val scanFilter: JMap[String, Condition] = createFilters(task).asJava
+    var evaluateKey: JMap[String, AttributeValue] = null
+
+    val scanLimit: Long   = task.getScanLimit
+    val recordLimit: Long = task.getRecordLimit
+    var recordCount: Long = 0
+
+    do {
+      val batchSize = getLimit(scanLimit, recordLimit, recordCount)
+
+      val request: ScanRequest = new ScanRequest()
+        .withTableName(task.getTable)
+        .withAttributesToGet(attributes)
+        .withScanFilter(scanFilter)
+        .withExclusiveStartKey(evaluateKey)
+
+      if (batchSize > 0) {
+        request.setLimit(batchSize)
+      }
+
+      val result: ScanResult = client.scan(request)
+      evaluateKey = result.getLastEvaluatedKey
+
+      val items = result.getItems.asScala.map(_.asScala.toMap)
+      recordCount += write(pageBuilder, schema, items)
+    } while(evaluateKey != null && (recordLimit == 0 || recordLimit > recordCount))
+
+    pageBuilder.finish()
+  }
+}

data/src/test/resources/yaml/authMethodBasic.yml

@@ -1,6 +1,7 @@
 in:
   type: dynamodb
   region: ENV_VAR
+  operation: scan
   table: ENV_VAR
   auth_method: basic
   access_key: ENV_VAR

data/src/test/resources/yaml/authMethodBasic_Error.yml

@@ -1,6 +1,7 @@
 in:
   type: dynamodb
   region: ENV_VAR
+  operation: scan
   table: ENV_VAR
   auth_method: basic
   columns:

data/src/test/resources/yaml/authMethodEnv.yml

@@ -1,6 +1,7 @@
 in:
   type: dynamodb
   region: ENV_VAR
+  operation: scan
   table: ENV_VAR
   auth_method: env
   columns:

data/src/test/resources/yaml/authMethodProfile.yml

@@ -1,6 +1,7 @@
 in:
   type: dynamodb
   region: ENV_VAR
+  operation: scan
   table: ENV_VAR
   auth_method: profile
   profile_name: ENV_VAR

data/src/test/resources/yaml/dynamodb-local-query.yml

@@ -0,0 +1,25 @@
+in:
+  type: dynamodb
+  end_point: http://localhost:8000/
+  table: ENV_VAR
+  auth_method: basic
+  access_key: dummy
+  secret_key: dummy
+  filters:
+    - {name: pri-key, type: string, condition: EQ, value: key-1}
+  columns:
+    - {name: pri-key,     type: string}
+    - {name: sort-key,    type: long}
+    - {name: doubleValue, type: double}
+    - {name: boolValue,   type: boolean}
+    - {name: listValue,   type: json}
+    - {name: mapValue,    type: json}
+
+out:
+  type: file
+  path_prefix: dynamodb-local-result
+  file_ext: tsv
+  formatter:
+    type: csv
+    delimiter: "\t"
+    header_line: false

data/src/test/resources/yaml/notSetAuthMethod.yml

@@ -1,6 +1,7 @@
 in:
   type: dynamodb
   region: ENV_VAR
+  operation: scan
   table: ENV_VAR
   access_key: ENV_VAR
   secret_key: ENV_VAR

data/src/test/scala/org/embulk/input/dynamodb/ope/QueryOperationTest.scala

@@ -0,0 +1,83 @@
+package org.embulk.input.dynamodb.ope
+
+import java.io.File
+import java.nio.charset.Charset
+import java.nio.file.{FileSystems, Files}
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.google.inject.{Binder, Module}
+import org.embulk.EmbulkEmbed
+import org.embulk.config.ConfigSource
+import org.embulk.input.dynamodb.DynamodbInputPlugin
+import org.embulk.plugin.InjectedPluginSource
+import org.embulk.spi.InputPlugin
+import org.hamcrest.CoreMatchers._
+import org.junit.Assert._
+import org.junit.{Before, Test}
+
+class QueryOperationTest {
+  private var embulk: EmbulkEmbed = null
+
+  private var EMBULK_DYNAMODB_TEST_TABLE: String = null
+  private var mapper: ObjectMapper = null
+
+  @Before
+  def createResources() {
+    // Get Environments
+    EMBULK_DYNAMODB_TEST_TABLE = System.getenv("EMBULK_DYNAMODB_TEST_TABLE")
+
+    val bootstrap = new EmbulkEmbed.Bootstrap()
+    bootstrap.addModules(new Module {
+      def configure(binder: Binder): Unit = {
+        InjectedPluginSource.registerPluginTo(binder,
+          classOf[InputPlugin],
+          "dynamodb",
+          classOf[DynamodbInputPlugin])
+      }
+    })
+
+    embulk = bootstrap.initializeCloseable()
+
+    mapper = new ObjectMapper()
+  }
+
+
+  def doTest(config: ConfigSource) {
+    embulk.run(config)
+
+    val fs = FileSystems.getDefault
+    val lines = Files.readAllLines(fs.getPath("dynamodb-local-result000.00.tsv"), Charset.forName("UTF-8"))
+    assertEquals(lines.size, 1)
+
+    val head = lines.get(0)
+    val values = head.split("\t")
+
+    assertThat(values(0), is("key-1"))
+    assertThat(values(1), is("0"))
+    assertThat(values(2), is("42.195"))
+    assertThat(values(3), is("true"))
+
+    val listValue = mapper.readValue(values(4).replaceAll("\"(?!\")", ""), classOf[java.util.List[Object]])
+    assertThat(listValue.size(), is(2))
+    assertThat(listValue.get(0).asInstanceOf[String], is("list-value"))
+    assertThat(listValue.get(1).asInstanceOf[Int], is(123))
+
+    val mapValue = mapper.readValue(values(5).replaceAll("\"(?!\")", ""), classOf[java.util.Map[String, Object]])
+    assert(mapValue.containsKey("map-key-1"))
+    assertThat(mapValue.get("map-key-1").asInstanceOf[String], is("map-value-1"))
+    assert(mapValue.containsKey("map-key-2"))
+    assertThat(mapValue.get("map-key-2").asInstanceOf[Int], is(456))
+  }
+
+  @Test
+  def queryTest() {
+    val config = embulk.newConfigLoader().fromYamlFile(
+      new File("src/test/resources/yaml/dynamodb-local-query.yml"))
+
+    config.getNested("in")
+      .set("operation", "query")
+      .set("table", EMBULK_DYNAMODB_TEST_TABLE)
+
+    doTest(config)
+  }
+}

data/src/test/scala/org/embulk/input/dynamodb/{DynamoDBUtilTest.scala → ope/ScanOperationTest.scala}

@@ -1,4 +1,4 @@
-package org.embulk.input.dynamodb
+package org.embulk.input.dynamodb.ope
 
 import java.io.File
 import java.nio.charset.Charset
@@ -8,13 +8,14 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import com.google.inject.{Binder, Module}
 import org.embulk.EmbulkEmbed
 import org.embulk.config.ConfigSource
+import org.embulk.input.dynamodb.DynamodbInputPlugin
 import org.embulk.plugin.InjectedPluginSource
 import org.embulk.spi.InputPlugin
 import org.hamcrest.CoreMatchers._
 import org.junit.Assert._
 import org.junit.{Before, Test}
 
-class DynamoDBUtilTest {
+class ScanOperationTest {
   private var embulk: EmbulkEmbed = null
 
   private var EMBULK_DYNAMODB_TEST_TABLE: String = null
@@ -71,9 +72,10 @@ class DynamoDBUtilTest {
   @Test
   def scanTest() {
     val config = embulk.newConfigLoader().fromYamlFile(
-      new File("src/test/resources/yaml/dynamodb-local.yml"))
+      new File("src/test/resources/yaml/dynamodb-local-scan.yml"))
 
     config.getNested("in")
+      .set("operation", "scan")
       .set("table", EMBULK_DYNAMODB_TEST_TABLE)
 
     doTest(config)

data/test/run_dynamodb_local.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
-docker run -d \
+docker run -i -t -d \
 	-p 8000:8000 \
-	-v $PWD/dynamodb-local:/data \
-	--env DYNAMO_OPT='-dbPath /data -sharedDb' \
-	lulichn/dynamodb-local
+	tray/dynamodb-local \
+	-inMemory -sharedDb -port 8000
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-dynamodb
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Daisuke Higashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-06 00:00:00.000000000 Z
+date: 2016-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -59,11 +59,14 @@ files:
 - lib/embulk/input/dynamodb.rb
 - src/main/scala/org/embulk/input/dynamodb/AttributeValueHelper.scala
 - src/main/scala/org/embulk/input/dynamodb/AwsCredentials.scala
-- src/main/scala/org/embulk/input/dynamodb/DynamoDBUtil.scala
+- src/main/scala/org/embulk/input/dynamodb/DynamoDBClient.scala
 - src/main/scala/org/embulk/input/dynamodb/DynamodbInputPlugin.scala
 - src/main/scala/org/embulk/input/dynamodb/Filter.scala
 - src/main/scala/org/embulk/input/dynamodb/FilterConfig.scala
 - src/main/scala/org/embulk/input/dynamodb/PluginTask.scala
+- src/main/scala/org/embulk/input/dynamodb/ope/AbstractOperation.scala
+- src/main/scala/org/embulk/input/dynamodb/ope/QueryOperation.scala
+- src/main/scala/org/embulk/input/dynamodb/ope/ScanOperation.scala
 - src/main/scala/org/embulk/input/dynamodb/package.scala
 - src/test/resources/json/test.json
 - src/test/resources/json/test.template
@@ -71,24 +74,26 @@ files:
 - src/test/resources/yaml/authMethodBasic_Error.yml
 - src/test/resources/yaml/authMethodEnv.yml
 - src/test/resources/yaml/authMethodProfile.yml
-- src/test/resources/yaml/dynamodb-local.yml
+- src/test/resources/yaml/dynamodb-local-query.yml
+- src/test/resources/yaml/dynamodb-local-scan.yml
 - src/test/resources/yaml/notSetAuthMethod.yml
 - src/test/scala/org/embulk/input/dynamodb/AttributeValueHelperTest.scala
 - src/test/scala/org/embulk/input/dynamodb/AwsCredentialsTest.scala
-- src/test/scala/org/embulk/input/dynamodb/DynamoDBUtilTest.scala
+- src/test/scala/org/embulk/input/dynamodb/ope/QueryOperationTest.scala
+- src/test/scala/org/embulk/input/dynamodb/ope/ScanOperationTest.scala
 - test/create_table.sh
 - test/put_items.sh
 - test/run_dynamodb_local.sh
-- classpath/commons-logging-1.1.3.jar
-- classpath/commons-codec-1.6.jar
-- classpath/scala-library-2.11.7.jar
+- classpath/aws-java-sdk-core-1.10.43.jar
 - classpath/aws-java-sdk-dynamodb-1.10.43.jar
-- classpath/embulk-input-dynamodb-0.1.1.jar
 - classpath/aws-java-sdk-kms-1.10.43.jar
-- classpath/httpcore-4.3.3.jar
-- classpath/httpclient-4.3.6.jar
-- classpath/aws-java-sdk-core-1.10.43.jar
 - classpath/aws-java-sdk-s3-1.10.43.jar
+- classpath/commons-codec-1.6.jar
+- classpath/commons-logging-1.1.3.jar
+- classpath/embulk-input-dynamodb-0.2.0.jar
+- classpath/httpclient-4.3.6.jar
+- classpath/httpcore-4.3.3.jar
+- classpath/scala-library-2.11.8.jar
 homepage: https://github.com/lulichn/embulk-input-dynamodb
 licenses:
 - MIT

data/src/main/scala/org/embulk/input/dynamodb/DynamoDBUtil.scala

@@ -1,155 +0,0 @@
-package org.embulk.input.dynamodb
-
-import java.util.{ArrayList => JArrayList, List => JList, Map => JMap}
-
-import com.amazonaws.ClientConfiguration
-import com.amazonaws.regions.Regions
-import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient
-import com.amazonaws.services.dynamodbv2.model.{AttributeValue, Condition, ScanRequest, ScanResult}
-import org.embulk.config.ConfigException
-import org.embulk.spi._
-import org.embulk.spi.`type`.Types
-import org.msgpack.value.{Value, ValueFactory}
-
-import scala.collection.JavaConverters._
-
-object DynamoDBUtil {
-  def createClient(task: PluginTask): AmazonDynamoDBClient = {
-    val client = new AmazonDynamoDBClient(
-      AwsCredentials.getCredentialsProvider(task),
-      new ClientConfiguration()
-        .withMaxConnections(50))  // SDK Default Value
-
-    if (task.getEndPoint.isPresent) {
-      client.withEndpoint(task.getEndPoint.get())
-    } else if (task.getRegion.isPresent) {
-      client.withRegion(Regions.fromName(task.getRegion.get()))
-    } else {
-      throw new ConfigException("At least one of EndPoint or Region must be set")
-    }
-  }
-
-
-  def scan(
-            task: PluginTask,
-            schema: Schema,
-            output: PageOutput)
-          (implicit client: AmazonDynamoDBClient): Unit =
-  {
-    val allocator: BufferAllocator = task.getBufferAllocator
-    val pageBuilder: PageBuilder = new PageBuilder(allocator, schema, output)
-
-    val attributes: JList[String] = new JArrayList[String]()
-
-    schema.getColumns.asScala.foreach { column =>
-      attributes.add(column.getName)
-    }
-    val scanFilter: JMap[String, Condition] = createScanFilter(task).asJava
-    var evaluateKey: JMap[String, AttributeValue] = null
-
-    val scanLimit: Long   = task.getScanLimit
-    val recordLimit: Long = task.getRecordLimit
-    var recordCount: Long = 0
-
-    do {
-      val batchSize = getScanLimit(scanLimit, recordLimit, recordCount)
-
-      val request: ScanRequest = new ScanRequest()
-        .withTableName(task.getTable)
-        .withAttributesToGet(attributes)
-        .withScanFilter(scanFilter)
-        .withExclusiveStartKey(evaluateKey)
-
-      if (batchSize > 0) {
-        request.setLimit(batchSize)
-      }
-
-      val result: ScanResult = client.scan(request)
-      evaluateKey = result.getLastEvaluatedKey
-
-      result.getItems.asScala.foreach { item =>
-        schema.getColumns.asScala.foreach { column =>
-          val value = item.asScala.get(column.getName)
-          column.getType match {
-            case Types.STRING =>
-              convert(column, value, pageBuilder.setString)
-            case Types.LONG =>
-              convert(column, value, pageBuilder.setLong)
-            case Types.DOUBLE =>
-              convert(column, value, pageBuilder.setDouble)
-            case Types.BOOLEAN =>
-              convert(column, value, pageBuilder.setBoolean)
-            case Types.JSON =>
-              convert(column, value, pageBuilder.setJson)
-            case _ => /* Do nothing */
-          }
-        }
-        pageBuilder.addRecord()
-        recordCount += 1
-      }
-    } while(evaluateKey != null && (recordLimit == 0 || recordLimit > recordCount))
-
-    pageBuilder.finish()
-  }
-
-  private def getScanLimit(scanLimit: Long, recordLimit: Long, recordCount: Long): Int = {
-    if (scanLimit > 0 && recordLimit > 0) {
-      math.min(scanLimit, recordLimit - recordCount).toInt
-    } else if (scanLimit > 0 || recordLimit > 0) {
-      math.max(scanLimit, recordLimit).toInt
-    } else { 0 }
-  }
-
-  private def createScanFilter(task: PluginTask): Map[String, Condition] = {
-    val filterMap = collection.mutable.HashMap[String, Condition]()
-
-    Option(task.getFilters.orNull).map { filters =>
-      filters.getFilters.asScala.map { filter =>
-        val attributeValueList = collection.mutable.ArrayBuffer[AttributeValue]()
-        attributeValueList += createAttributeValue(filter.getType, filter.getValue)
-        Option(filter.getValue2).map { value2 =>
-          attributeValueList+= createAttributeValue(filter.getType, value2) }
-
-        filterMap += filter.getName -> new Condition()
-          .withComparisonOperator(filter.getCondition)
-          .withAttributeValueList(attributeValueList.asJava)
-      }
-    }
-
-    filterMap.toMap
-  }
-
-  private def createAttributeValue(t: String, v: String): AttributeValue = {
-    t match {
-      case "string" =>
-        new AttributeValue().withS(v)
-      case "long" | "double" =>
-        new AttributeValue().withN(v)
-      case "boolean" =>
-        new AttributeValue().withBOOL(v.toBoolean)
-    }
-  }
-
-  private def convert[A](column: Column,
-                   value: Option[AttributeValue],
-                   f: (Column, A) => Unit)(implicit f1: Option[AttributeValue] => A): Unit =
-    f(column, f1(value))
-
-  implicit private def StringConvert(value: Option[AttributeValue]): String =
-    value.map(_.getS).getOrElse("")
-
-  implicit private def LongConvert(value: Option[AttributeValue]): Long =
-    value.map(_.getN.toLong).getOrElse(0L)
-
-  implicit private def DoubleConvert(value: Option[AttributeValue]): Double =
-    value.map(_.getN.toDouble).getOrElse(0D)
-
-  implicit private def BooleanConvert(value: Option[AttributeValue]): Boolean =
-    value.exists(_.getBOOL)
-
-  implicit private def JsonConvert(value: Option[AttributeValue]): Value = {
-    value.map { attr =>
-      AttributeValueHelper.decodeToValue(attr)
-    }.getOrElse(ValueFactory.newNil())
-  }
-}