embulk-input-dynamodb 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9806978e998d913f4154c2729c73fb3334fb0b63
-  data.tar.gz: 8af2ea9b4489a0cad2ad44d6cc04fc4d1d7a6035
+  metadata.gz: b499a6aa99576c0713943f0be90e2435b12ac017
+  data.tar.gz: ebfbba02b9e38d42c97049c7742d4245a0f368f5
 SHA512:
-  metadata.gz: a13f066a99e2dbdfecb67eab9150610e32b0c2ed21190094f7dabdf7c25be750fcbdbf3b6d04498106372e21fed1ddea7922d08c46321d7c5a0b8d89ff96cefa
-  data.tar.gz: de2117e3e1e4d9d2997f6bcb4606a86556c7379bce0b37db6fbb4a2062c3a4640f2739abba55fd95cae0f00bb8b734d1f31f041efbf28d68747018588fcc6b95
+  metadata.gz: 220cdcde6ec63897c87a53b8cd656a1e2628e930410f8a7dca9873bf2d9ded244222ed867997282804a327f047c46eb51b9d7e0abf16666e96ebf6b1588ac149
+  data.tar.gz: 5d951db675c6166da1b70b65d9d80584eadf0e0aaeb28d710a07741dd4a401955b80eef88619a124739d328a32b8d3b5328228be59b4c06970f871f1060659e1

data/LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015 lulichn
+Copyright (c) 2015 Daisuke Higashi
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -9,16 +9,20 @@
 
 
 ## Configuration
-
+- **access_key**: AWS access key (string, optional)
+- **secret_key**: AWS secret key (string, optional)  
+If you don't specify keys, I'll use the profile configuration file for the default profile.
 - **region**: Region Name (string, default: ap-northeast-1)
-- **table**: Table Name (string, required
-- **limit** Scan Limit (integer, default: 100)
+- **table**: Table Name (string, required)
+- **limit**: Scan Limit (integer, default: 100)
 
 ## Example
 
 ```yaml
 in:
   type: dynamodb
+  access_key: YOUR_ACCESS_KEY
+  secret_key: YOUR_SECRET_KEY
   region: ap-northeast-1
   table: YOUR_TABLE_NAME
   limit: 1000
@@ -27,6 +31,12 @@ in:
     - {name: ColumnB, type: double}
     - {name: ColumnC, type: string}
     - {name: ColumnD, type: boolean}
+  filters:
+    - {name: ColumnA, type: long, condition: BETWEEN, value: 10000, value2: 20000}
+    - {name: ColumnC, type: string, condition: EQ, value: foobar}
+
+out:
+  type: stdout
 ```
 
 ## Build

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.0.1"
+version = "0.0.2"
 
 dependencies {
     compile "org.scala-lang:scala-library:2.11.5"

data/src/main/scala/org/embulk/input/DynamoDBUtil.scala

@@ -1,43 +1,60 @@
 package org.embulk.input
 
 import com.amazonaws.ClientConfiguration
+import com.amazonaws.auth.{AWSCredentials, BasicAWSCredentials, AWSCredentialsProvider}
 import com.amazonaws.auth.profile.ProfileCredentialsProvider
 import com.amazonaws.regions.Regions
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient
-import com.amazonaws.services.dynamodbv2.model.{ScanResult, ListTablesResult, ScanRequest}
+import com.amazonaws.services.dynamodbv2.model._
 import org.embulk.spi._
 
 import java.util.{ArrayList => JArrayList, List => JList}
 import scala.collection.JavaConversions._
 
 object DynamoDBUtil {
+  private def getCredentialsProvider(task: PluginTask): AWSCredentialsProvider = {
+   {for {
+      accessKey <- Option(task.getAccessKey.orNull)
+      secretKey <- Option(task.getSecretKey.orNull)
+    } yield {
+     new AWSCredentialsProvider {
+       override def refresh(): Unit = { }
+       override def getCredentials: AWSCredentials = {
+         new BasicAWSCredentials(accessKey, secretKey)
+       }
+     }
+    }}.getOrElse{
+      new ProfileCredentialsProvider()
+    }
+  }
+
   def createClient(task: PluginTask): AmazonDynamoDBClient = {
+    val credentialsProvider: AWSCredentialsProvider = getCredentialsProvider(task)
     val client: AmazonDynamoDBClient = new AmazonDynamoDBClient(
-      new ProfileCredentialsProvider(),
+      credentialsProvider,
       new ClientConfiguration().withMaxConnections(10))
       .withRegion(Regions.fromName(task.getRegion))
 
     client
   }
 
-  def listTables(client: AmazonDynamoDBClient): JList[String] = {
-    val result: ListTablesResult = client.listTables()
-    result.getTableNames
-  }
 
   def scan(client: AmazonDynamoDBClient, task: PluginTask, schema: Schema, output: PageOutput): Unit = {
     val allocator: BufferAllocator = task.getBufferAllocator
     val pageBuilder: PageBuilder = new PageBuilder(allocator, schema, output)
 
-    val limit: Int = task.getLimit
     val attributes: JList[String] = new JArrayList[String]()
+
     schema.getColumns.foreach { column =>
       attributes.add(column.getName)
     }
+    val scanFilter: Map[String, Condition] = createScanFilter(task)
+    val limit: Int = task.getLimit
 
     val request: ScanRequest = new ScanRequest()
       .withTableName(task.getTable)
       .withAttributesToGet(attributes)
+      .withScanFilter(scanFilter)
       .withLimit(limit)
 
     val result: ScanResult = client.scan(request)
@@ -61,4 +78,33 @@ object DynamoDBUtil {
 
     pageBuilder.finish()
   }
+
+  private def createScanFilter(task: PluginTask): Map[String, Condition] = {
+    val filterMap = collection.mutable.HashMap[String, Condition]()
+
+    Option(task.getFilters.orNull).map { filters =>
+      filters.getFilters.map { filter =>
+        val attributeValueList = collection.mutable.ArrayBuffer[AttributeValue]()
+        attributeValueList += createAttrinuteValue(filter.getType, filter.getValue)
+        Option(filter.getValue2).map { value2 => attributeValueList += createAttrinuteValue(filter.getType, value2) }
+
+        filterMap += filter.getName -> new Condition()
+          .withComparisonOperator(filter.getCondition)
+          .withAttributeValueList(attributeValueList)
+      }
+    }
+
+    filterMap.toMap
+  }
+
+  private def createAttrinuteValue(t: String, v: String): AttributeValue = {
+    t match {
+      case "string" =>
+        new AttributeValue().withS(v)
+      case "long" | "double" =>
+        new AttributeValue().withN(v)
+      case "boolean" =>
+        new AttributeValue().withBOOL(v.toBoolean)
+    }
+  }
 }

data/src/main/scala/org/embulk/input/Filter.scala

@@ -0,0 +1,32 @@
+package org.embulk.input
+
+import java.util.{List => JList}
+
+import com.fasterxml.jackson.annotation.{JsonValue, JsonCreator}
+import com.google.common.base.Objects
+
+class Filter {
+  private var filters: JList[FilterConfig] = _
+
+  @JsonCreator
+  def this(filters: JList[FilterConfig]) {
+    this()
+    this.filters = filters
+  }
+
+  @JsonValue
+  def getFilters: JList[FilterConfig] = filters
+
+  override def equals(obj: Any): Boolean = {
+    if(this == obj) return true
+
+    if(!obj.isInstanceOf[Filter]) return false
+
+    val other: Filter = obj.asInstanceOf[Filter]
+    Objects.equal(filters, other.filters)
+  }
+
+  override def hashCode: Int = {
+    Objects.hashCode(filters)
+  }
+}

data/src/main/scala/org/embulk/input/FilterConfig.scala

@@ -0,0 +1,55 @@
+package org.embulk.input
+
+import com.fasterxml.jackson.annotation.JsonProperty
+import com.google.common.base.Objects
+
+class FilterConfig {
+  private var _name: String = _
+  private var _type: String = _
+  private var _condition: String = _
+  private var _value: String = _
+  private var _value2: String = _
+
+  def this(
+            @JsonProperty("name") _name: String,
+            @JsonProperty("type") _type: String,
+            @JsonProperty("condition") _condition: String,
+            @JsonProperty("value") _value: String,
+            @JsonProperty("value2") _value2: String)
+  {
+    this()
+    this._name = _name
+    this._type = _type
+    this._condition = _condition
+    this._value = _value
+    this._value2 = _value2
+  }
+
+  @JsonProperty("name")
+  def getName = _name
+
+  @JsonProperty("type")
+  def getType = _type
+
+  @JsonProperty("condition")
+  def getCondition = _condition
+
+  @JsonProperty("value")
+  def getValue = _value
+
+  @JsonProperty("value2")
+  def getValue2 = _value2
+
+  override def equals(obj: Any): Boolean = {
+    if(this == obj) return true
+
+    if(!obj.isInstanceOf[FilterConfig]) return false
+
+    val other: FilterConfig = obj.asInstanceOf[FilterConfig]
+    Objects.equal(this._name, other._name) &&
+      Objects.equal(this._type, other._type) &&
+      Objects.equal(this._condition, other._condition) &&
+      Objects.equal(this._value, other._value) &&
+      Objects.equal(this._value2, other._value2)
+  }
+}

data/src/main/scala/org/embulk/input/PluginTask.scala

@@ -1,9 +1,18 @@
 package org.embulk.input
 
+import com.google.common.base.Optional
 import org.embulk.config.{ConfigInject, ConfigDefault, Config, Task}
 import org.embulk.spi.{BufferAllocator, SchemaConfig}
 
 trait PluginTask extends Task {
+  @Config("access_key")
+  @ConfigDefault("null")
+  def getAccessKey: Optional[String]
+
+  @Config("secret_key")
+  @ConfigDefault("null")
+  def getSecretKey: Optional[String]
+
   @Config("region")
   @ConfigDefault("ap-northeast-1")
   def getRegion: String
@@ -18,6 +27,10 @@ trait PluginTask extends Task {
   @Config("columns")
   def getColumns: SchemaConfig
 
+  @Config("filters")
+  @ConfigDefault("null")
+  def getFilters: Optional[Filter]
+
   @ConfigInject
   def getBufferAllocator: BufferAllocator
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-dynamodb
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Daisuke Higashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-03-10 00:00:00.000000000 Z
+date: 2015-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -47,7 +47,6 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - LICENSE
-- LICENSE.txt
 - README.md
 - build.gradle
 - gradle/wrapper/gradle-wrapper.jar
@@ -57,6 +56,8 @@ files:
 - lib/embulk/input/dynamodb.rb
 - src/main/scala/org/embulk/input/DynamoDBUtil.scala
 - src/main/scala/org/embulk/input/DynamodbInputPlugin.scala
+- src/main/scala/org/embulk/input/Filter.scala
+- src/main/scala/org/embulk/input/FilterConfig.scala
 - src/main/scala/org/embulk/input/PluginTask.scala
 - classpath/aws-java-sdk-core-1.9.22.jar
 - classpath/aws-java-sdk-dynamodb-1.9.22.jar
@@ -64,7 +65,7 @@ files:
 - classpath/aws-java-sdk-s3-1.9.22.jar
 - classpath/commons-codec-1.6.jar
 - classpath/commons-logging-1.1.3.jar
-- classpath/embulk-input-dynamodb-0.0.1.jar
+- classpath/embulk-input-dynamodb-0.0.2.jar
 - classpath/httpclient-4.3.4.jar
 - classpath/httpcore-4.3.2.jar
 - classpath/scala-library-2.11.5.jar

data/LICENSE.txt

@@ -1,21 +0,0 @@
-
-MIT License
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.