embulk-input-dynamodb 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b499a6aa99576c0713943f0be90e2435b12ac017
-  data.tar.gz: ebfbba02b9e38d42c97049c7742d4245a0f368f5
+  metadata.gz: d5bff72dc25c3fe5ce57da31798ac74d2ac7e8bf
+  data.tar.gz: a35560d2b4a29e5d5d71d2d2656210da7653e19b
 SHA512:
-  metadata.gz: 220cdcde6ec63897c87a53b8cd656a1e2628e930410f8a7dca9873bf2d9ded244222ed867997282804a327f047c46eb51b9d7e0abf16666e96ebf6b1588ac149
-  data.tar.gz: 5d951db675c6166da1b70b65d9d80584eadf0e0aaeb28d710a07741dd4a401955b80eef88619a124739d328a32b8d3b5328228be59b4c06970f871f1060659e1
+  metadata.gz: 41fd0fdc599929929d81519aefb58c72cb250f66650f31b8305d08ed26f77d41fe08fb8df275ada2383f388c6629a7d55c6b4211a519f74382781e6667cc85a6
+  data.tar.gz: d0f608e76092fe34e150a4d3e4076c7a3f94e36d6990f64d721c2628702492456e7d48cd713bca5e8442cab958e4e82d7d695fab5901d0d3c75ac3d5f340cd1a

data/README.md

@@ -14,7 +14,8 @@
 If you don't specify keys, I'll use the profile configuration file for the default profile.
 - **region**: Region Name (string, default: ap-northeast-1)
 - **table**: Table Name (string, required)
-- **limit**: Scan Limit (integer, default: 100)
+- **scan_limit**: DynamoDB 1time Scan Query size limit (Int, optional) 
+- **record_limit**: Max Record Search limit (Long, optional) 
 
 ## Example
 
@@ -25,7 +26,6 @@ in:
   secret_key: YOUR_SECRET_KEY
   region: ap-northeast-1
   table: YOUR_TABLE_NAME
-  limit: 1000
   columns:
     - {name: ColumnA, type: long}
     - {name: ColumnB, type: double}

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.0.2"
+version = "0.0.3"
 
 dependencies {
     compile "org.scala-lang:scala-library:2.11.5"

data/src/main/scala/org/embulk/input/DynamoDBUtil.scala

@@ -1,17 +1,27 @@
 package org.embulk.input
 
-import com.amazonaws.ClientConfiguration
-import com.amazonaws.auth.{AWSCredentials, BasicAWSCredentials, AWSCredentialsProvider}
+import java.util.{ArrayList => JArrayList, List => JList, Map => JMap}
+
 import com.amazonaws.auth.profile.ProfileCredentialsProvider
+import com.amazonaws.auth.{AWSCredentials, AWSCredentialsProvider, BasicAWSCredentials}
 import com.amazonaws.regions.Regions
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient
 import com.amazonaws.services.dynamodbv2.model._
+import com.amazonaws.{AmazonClientException, ClientConfiguration}
 import org.embulk.spi._
 
-import java.util.{ArrayList => JArrayList, List => JList}
 import scala.collection.JavaConversions._
 
 object DynamoDBUtil {
+  def createClient(task: PluginTask): AmazonDynamoDBClient = {
+    try {
+      createClientUsingIAMRole(task)
+    } catch {
+      case e: AmazonClientException =>
+        createClientUsingCredentials(task)
+    }
+  }
+
   private def getCredentialsProvider(task: PluginTask): AWSCredentialsProvider = {
    {for {
       accessKey <- Option(task.getAccessKey.orNull)
@@ -23,18 +33,30 @@ object DynamoDBUtil {
          new BasicAWSCredentials(accessKey, secretKey)
        }
      }
-    }}.getOrElse{
+    }}.getOrElse {
       new ProfileCredentialsProvider()
     }
   }
 
-  def createClient(task: PluginTask): AmazonDynamoDBClient = {
+  private def createClientUsingIAMRole(task: PluginTask): AmazonDynamoDBClient = {
+    val client: AmazonDynamoDBClient = new AmazonDynamoDBClient(
+      new ClientConfiguration().withMaxConnections(10))
+      .withRegion(Regions.fromName(task.getRegion))
+
+    client.describeTable(task.getTable)   // FIXME
+
+    client
+  }
+
+  private def createClientUsingCredentials(task: PluginTask): AmazonDynamoDBClient = {
     val credentialsProvider: AWSCredentialsProvider = getCredentialsProvider(task)
     val client: AmazonDynamoDBClient = new AmazonDynamoDBClient(
       credentialsProvider,
       new ClientConfiguration().withMaxConnections(10))
       .withRegion(Regions.fromName(task.getRegion))
 
+    client.describeTable(task.getTable)  // FIXME
+
     client
   }
 
@@ -49,44 +71,68 @@ object DynamoDBUtil {
       attributes.add(column.getName)
     }
     val scanFilter: Map[String, Condition] = createScanFilter(task)
-    val limit: Int = task.getLimit
-
-    val request: ScanRequest = new ScanRequest()
-      .withTableName(task.getTable)
-      .withAttributesToGet(attributes)
-      .withScanFilter(scanFilter)
-      .withLimit(limit)
-
-    val result: ScanResult = client.scan(request)
-    result.getItems.foreach { item =>
-      schema.getColumns.foreach { column =>
-        val value = item.get(column.getName)
-        column.getType.getName match {
-          case "string" =>
-            pageBuilder.setString(column, Option(value) map { _.getS } getOrElse { "" })
-          case "long" =>
-            pageBuilder.setLong(column, Option(value) map { _.getN.toLong } getOrElse { 0L })
-          case "double" =>
-            pageBuilder.setDouble(column, Option(value) map { _.getN.toDouble } getOrElse { 0D })
-          case "boolean" =>
-            pageBuilder.setBoolean(column, Option(value) map { _.getBOOL == true } getOrElse { false })
-          case _ => /* Do nothing */
+    var evaluateKey: JMap[String, AttributeValue] = null
+
+    val scanLimit: Long   = task.getScanLimit
+    val recordLimit: Long = task.getRecordLimit
+    var recordCount: Long = 0
+
+    do {
+      val batchSize = getScanLimit(scanLimit, recordLimit, recordCount)
+
+      val request: ScanRequest = new ScanRequest()
+        .withTableName(task.getTable)
+        .withAttributesToGet(attributes)
+        .withScanFilter(scanFilter)
+        .withExclusiveStartKey(evaluateKey)
+
+      if (batchSize > 0) {
+        request.setLimit(batchSize)
+      }
+
+      val result: ScanResult = client.scan(request)
+      evaluateKey = result.getLastEvaluatedKey
+
+      result.getItems.foreach { item =>
+        schema.getColumns.foreach { column =>
+          val value = item.get(column.getName)
+          column.getType.getName match {
+            case "string" =>
+              pageBuilder.setString(column, Option(value) map { _.getS } getOrElse { "" })
+            case "long" =>
+              pageBuilder.setLong(column, Option(value) map { _.getN.toLong } getOrElse { 0L })
+            case "double" =>
+              pageBuilder.setDouble(column, Option(value) map { _.getN.toDouble } getOrElse { 0D })
+            case "boolean" =>
+              pageBuilder.setBoolean(column, Option(value) map { _.getBOOL == true } getOrElse { false })
+            case _ => /* Do nothing */
+          }
         }
+        pageBuilder.addRecord()
+        recordCount += 1
       }
-      pageBuilder.addRecord()
-    }
+      println(s"$recordLimit $recordLimit $recordCount")
+    } while(evaluateKey != null && (recordLimit == 0 || recordLimit > recordCount))
 
     pageBuilder.finish()
   }
 
+  private def getScanLimit(scanLimit: Long, recordLimit: Long, recordCount: Long): Int = {
+    if (scanLimit > 0 && recordLimit > 0) {
+      math.min(scanLimit, recordLimit - recordCount).toInt
+    } else if (scanLimit > 0 || recordLimit > 0) {
+      math.max(scanLimit, recordLimit).toInt
+    } else { 0 }
+  }
+
   private def createScanFilter(task: PluginTask): Map[String, Condition] = {
     val filterMap = collection.mutable.HashMap[String, Condition]()
 
     Option(task.getFilters.orNull).map { filters =>
       filters.getFilters.map { filter =>
         val attributeValueList = collection.mutable.ArrayBuffer[AttributeValue]()
-        attributeValueList += createAttrinuteValue(filter.getType, filter.getValue)
-        Option(filter.getValue2).map { value2 => attributeValueList += createAttrinuteValue(filter.getType, value2) }
+        attributeValueList += createAttributeValue(filter.getType, filter.getValue)
+        Option(filter.getValue2).map { value2 => attributeValueList += createAttributeValue(filter.getType, value2) }
 
         filterMap += filter.getName -> new Condition()
           .withComparisonOperator(filter.getCondition)
@@ -97,7 +143,7 @@ object DynamoDBUtil {
     filterMap.toMap
   }
 
-  private def createAttrinuteValue(t: String, v: String): AttributeValue = {
+  private def createAttributeValue(t: String, v: String): AttributeValue = {
     t match {
       case "string" =>
         new AttributeValue().withS(v)

data/src/main/scala/org/embulk/input/PluginTask.scala

@@ -17,13 +17,17 @@ trait PluginTask extends Task {
   @ConfigDefault("ap-northeast-1")
   def getRegion: String
 
+  @Config("scan_limit")
+  @ConfigDefault("0")
+  def getScanLimit: Long
+
+  @Config("record_limit")
+  @ConfigDefault("0")
+  def getRecordLimit: Long
+
   @Config("table")
   def getTable: String
 
-  @Config("limit")
-  @ConfigDefault("100")
-  def getLimit: Int
-
   @Config("columns")
   def getColumns: SchemaConfig
 

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-dynamodb
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Daisuke Higashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-03-16 00:00:00.000000000 Z
+date: 2015-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: bundler
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  name: rake
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
 description: '"Loads records from Dynamodb."'
 email:
 - daisuke.develop@gmail.com
@@ -65,9 +65,10 @@ files:
 - classpath/aws-java-sdk-s3-1.9.22.jar
 - classpath/commons-codec-1.6.jar
 - classpath/commons-logging-1.1.3.jar
-- classpath/embulk-input-dynamodb-0.0.2.jar
+- classpath/embulk-input-dynamodb-0.0.3.jar
 - classpath/httpclient-4.3.4.jar
 - classpath/httpcore-4.3.2.jar
+- classpath/joda-time-2.8.1.jar
 - classpath/scala-library-2.11.5.jar
 homepage: https://github.com/lulichn/embulk-input-dynamodb
 licenses: