embulk-input-dynamodb 0.0.3 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d5bff72dc25c3fe5ce57da31798ac74d2ac7e8bf
-  data.tar.gz: a35560d2b4a29e5d5d71d2d2656210da7653e19b
+  metadata.gz: fb06f7f31e946edcde0e6fde44443d11c20b2f1b
+  data.tar.gz: 83f448ad04d7f262c9a1438c79091ce1c6ac2765
 SHA512:
-  metadata.gz: 41fd0fdc599929929d81519aefb58c72cb250f66650f31b8305d08ed26f77d41fe08fb8df275ada2383f388c6629a7d55c6b4211a519f74382781e6667cc85a6
-  data.tar.gz: d0f608e76092fe34e150a4d3e4076c7a3f94e36d6990f64d721c2628702492456e7d48cd713bca5e8442cab958e4e82d7d695fab5901d0d3c75ac3d5f340cd1a
+  metadata.gz: 00544333da3be446d2e479fe1055f5599ea50655f0ba77d6ba465b0d4a0b58316675ac269a0299aaddd8469599b6d43dc02f05fb9b2d07b5daa25d68b4448f9f
+  data.tar.gz: 8960d06019a11c3e291f7724b0404e9dbd44df041a91cbe4bdaccb9b2e430a5213d60ff597aafab99ff9d6daa3409bead9b634126fa59a2712338e3084c6f318

data/README.md

@@ -9,19 +9,34 @@
 
 
 ## Configuration
-- **access_key**: AWS access key (string, optional)
-- **secret_key**: AWS secret key (string, optional)  
-If you don't specify keys, I'll use the profile configuration file for the default profile.
+- **auth_method**: AWS Credential Type.
+Available values options are: `basic`, `env`, `instance`, `profile`, `properties`
+  - **basic**: AWS access key and secret access key
+  - **env**: Environment variables
+  - **instance**: EC2 Instance Metadata Service
+  - **profile**: Profile configuration file
+  - **properties**: Java system properties
+- If **auth_method** is set `basic`
+  - **access_key**: AWS access key (string, required)
+  - **secret_key**: AWS secret key (string, required)
+- If **auth_method** is set `profile`
+  - **profile_name**: The name of a local configuration profile (string, optional)
 - **region**: Region Name (string, default: ap-northeast-1)
 - **table**: Table Name (string, required)
 - **scan_limit**: DynamoDB 1time Scan Query size limit (Int, optional) 
 - **record_limit**: Max Record Search limit (Long, optional) 
+- **columns**: a key-value pairs where key is a column name and value is options for the column (required)
+  - **name**: Column name.
+  - **type**: Column values are converted to this embulk type.
+  Available values options are: `boolean`, `long`, `double`, `string`, `json`
+- **filters**: query filter (optional)
 
 ## Example
 
 ```yaml
 in:
   type: dynamodb
+  auth_method: basic
   access_key: YOUR_ACCESS_KEY
   secret_key: YOUR_SECRET_KEY
   region: ap-northeast-1
@@ -31,6 +46,7 @@ in:
     - {name: ColumnB, type: double}
     - {name: ColumnC, type: string}
     - {name: ColumnD, type: boolean}
+    - {name: ColumnE, type: json}
   filters:
     - {name: ColumnA, type: long, condition: BETWEEN, value: 10000, value2: 20000}
     - {name: ColumnC, type: string, condition: EQ, value: foobar}

data/build.gradle

@@ -2,6 +2,7 @@ plugins {
     id "com.jfrog.bintray" version "1.1"
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "scala"
+    id "checkstyle"
 }
 
 import com.github.jrubygradle.JRubyExec
@@ -13,16 +14,22 @@ configurations {
     provided
 }
 
-version = "0.0.3"
+version = "0.1.0"
+
+sourceCompatibility = 1.7
+targetCompatibility = 1.7
 
 dependencies {
-    compile "org.scala-lang:scala-library:2.11.5"
+    compile "org.scala-lang:scala-library:2.11.7"
+
+    compile  "org.embulk:embulk-core:0.8.6"
+    provided "org.embulk:embulk-core:0.8.6"
 
-    compile  "org.embulk:embulk-core:0.4.10"
-    provided "org.embulk:embulk-core:0.4.10"
+    compile "com.amazonaws:aws-java-sdk-dynamodb:1.10.43"
 
-    compile "com.amazonaws:aws-java-sdk-dynamodb:1.9.22"
     testCompile "junit:junit:4.+"
+    testCompile "org.embulk:embulk-standards:0.8.6"
+    testCompile "org.embulk:embulk-core:0.8.6:tests"
 }
 
 compileScala {
@@ -48,6 +55,23 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
 }
 clean { delete 'classpath' }
 
+checkstyle {
+    configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
+    toolVersion = '6.14.1'
+}
+checkstyleMain {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+checkstyleTest {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+task checkstyle(type: Checkstyle) {
+    classpath = sourceSets.main.output + sourceSets.test.output
+    source = sourceSets.main.allJava + sourceSets.test.allJava
+}
+
 task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
     jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
     script "build/gemspec"

data/circle.yml

@@ -0,0 +1,8 @@
+general:
+  artifacts:
+    - "~/embulk-input-dynamodb/build/reports/tests"
+
+test:
+  post:
+  - mkdir -p $CIRCLE_TEST_REPORTS/junit/
+  - find . -type f -regex ".*/test-results/*/.*xml" -exec cp {} $CIRCLE_TEST_REPORTS/junit/ \;

data/config/checkstyle/checkstyle.xml

@@ -0,0 +1,128 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+    <!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
+    <module name="FileTabCharacter"/>
+    <module name="NewlineAtEndOfFile">
+        <property name="lineSeparator" value="lf"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\r"/>
+        <property name="message" value="Line contains carriage return"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value=" \n"/>
+        <property name="message" value="Line has trailing whitespace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\{\n\n"/>
+        <property name="message" value="Blank line after opening brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\s*\}"/>
+        <property name="message" value="Blank line before closing brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\n"/>
+        <property name="message" value="Multiple consecutive blank lines"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\Z"/>
+        <property name="message" value="Blank line before end of file"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="Preconditions\.checkNotNull"/>
+        <property name="message" value="Use of checkNotNull"/>
+    </module>
+
+    <module name="TreeWalker">
+        <module name="EmptyBlock">
+            <property name="option" value="text"/>
+            <property name="tokens" value="
+                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
+                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
+        </module>
+        <module name="EmptyStatement"/>
+        <module name="EmptyForInitializerPad"/>
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="MethodParamPad">
+            <property name="allowLineBreaks" value="true"/>
+            <property name="option" value="nospace"/>
+        </module>
+        <module name="ParenPad"/>
+        <module name="TypecastParenPad"/>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
+        </module>
+        <module name="LeftCurly">
+            <property name="option" value="eol"/>
+            <property name="tokens" value="
+                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
+                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
+        </module>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+        </module>
+        <module name="GenericWhitespace"/>
+        <module name="WhitespaceAfter"/>
+        <module name="NoWhitespaceBefore"/>
+
+        <module name="UpperEll"/>
+        <module name="DefaultComesLast"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ModifierOrder"/>
+        <module name="OneStatementPerLine"/>
+        <module name="StringLiteralEquality"/>
+        <module name="MutableException"/>
+        <module name="EqualsHashCode"/>
+        <module name="InnerAssignment"/>
+        <module name="InterfaceIsType"/>
+        <module name="HideUtilityClassConstructor"/>
+
+        <module name="MemberName"/>
+        <module name="LocalVariableName"/>
+        <module name="LocalFinalVariableName"/>
+        <module name="TypeName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName"/>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+
+        <module name="AvoidStarImport"/>
+        <module name="RedundantImport"/>
+        <module name="UnusedImports"/>
+        <module name="ImportOrder">
+            <property name="groups" value="*,javax,java"/>
+            <property name="separated" value="true"/>
+            <property name="option" value="bottom"/>
+            <property name="sortStaticImportsAlphabetically" value="true"/>
+        </module>
+
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="ignoreEnhancedForColon" value="false"/>
+            <property name="tokens" value="
+                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
+                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
+                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
+                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
+                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
+                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
+        </module>
+    </module>
+</module>

data/config/checkstyle/default.xml

@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<!--
+     This is a subset of ./checkstyle.xml which allows some loose styles
+-->
+<module name="Checker">
+    <module name="FileTabCharacter"/>
+    <module name="NewlineAtEndOfFile">
+        <property name="lineSeparator" value="lf"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\r"/>
+        <property name="message" value="Line contains carriage return"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value=" \n"/>
+        <property name="message" value="Line has trailing whitespace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\n"/>
+        <property name="message" value="Multiple consecutive blank lines"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\Z"/>
+        <property name="message" value="Blank line before end of file"/>
+    </module>
+
+    <module name="TreeWalker">
+        <module name="EmptyBlock">
+            <property name="option" value="text"/>
+            <property name="tokens" value="
+                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
+                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
+        </module>
+        <module name="EmptyStatement"/>
+        <module name="EmptyForInitializerPad"/>
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="MethodParamPad">
+            <property name="allowLineBreaks" value="true"/>
+            <property name="option" value="nospace"/>
+        </module>
+        <module name="ParenPad"/>
+        <module name="TypecastParenPad"/>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
+        </module>
+        <module name="LeftCurly">
+            <property name="option" value="eol"/>
+            <property name="tokens" value="
+                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
+                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
+        </module>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+        </module>
+        <module name="GenericWhitespace"/>
+        <module name="WhitespaceAfter"/>
+        <module name="NoWhitespaceBefore"/>
+
+        <module name="UpperEll"/>
+        <module name="DefaultComesLast"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ModifierOrder"/>
+        <module name="OneStatementPerLine"/>
+        <module name="StringLiteralEquality"/>
+        <module name="MutableException"/>
+        <module name="EqualsHashCode"/>
+        <module name="InnerAssignment"/>
+        <module name="InterfaceIsType"/>
+        <module name="HideUtilityClassConstructor"/>
+
+        <module name="MemberName"/>
+        <module name="LocalVariableName"/>
+        <module name="LocalFinalVariableName"/>
+        <module name="TypeName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName"/>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="ignoreEnhancedForColon" value="false"/>
+            <property name="tokens" value="
+                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
+                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
+                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
+                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
+                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
+                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
+        </module>
+    </module>
+</module>

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Mon Mar 09 00:45:24 JST 2015
+#Wed Jan 13 12:41:02 JST 2016
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.2.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip

data/lib/embulk/input/dynamodb.rb

@@ -1,3 +1,3 @@
 Embulk::JavaPlugin.register_input(
-  "dynamodb", "org.embulk.input.DynamodbInputPlugin",
+  "dynamodb", "org.embulk.input.dynamodb.DynamodbInputPlugin",
   File.expand_path('../../../../classpath', __FILE__))

data/src/main/scala/org/embulk/input/dynamodb/AttributeValueHelper.scala

@@ -0,0 +1,41 @@
+package org.embulk.input.dynamodb
+
+import com.amazonaws.services.dynamodbv2.model.AttributeValue
+import org.msgpack.value.{Value, ValueFactory}
+
+import scala.util.Try
+
+object AttributeValueHelper {
+
+  // referring aws-scala
+  def decodeToValue(value: AttributeValue): Value = {
+    import scala.collection.JavaConverters._
+
+    // FIXME: Need Encode?
+    lazy val _bin  = Option(value.getB).map(v => ValueFactory.newBinary(v.array))
+    lazy val _bool = Option(value.getBOOL).map(v => ValueFactory.newBoolean(v))
+    lazy val _num  = Option(value.getN).map(v =>
+      Try(v.toLong).map(ValueFactory.newInteger).getOrElse(ValueFactory.newFloat(v.toDouble)))
+    lazy val _str  = Option(value.getS).map(v => ValueFactory.newString(v))
+    lazy val _nil  = Option(value.getNULL).map(v => ValueFactory.newNil)
+
+    lazy val _list = Option(value.getL).map(l =>
+      ValueFactory.newArray(l.asScala.map(v => decodeToValue(v)).asJava))
+    lazy val _ss   = Option(value.getSS).map(l =>
+      ValueFactory.newArray(l.asScala.map(v => ValueFactory.newString(v)).asJava))
+    lazy val _ns   = Option(value.getNS).map(l =>
+      ValueFactory.newArray(l.asScala.map(v =>
+        Try(v.toLong).map(ValueFactory.newInteger).getOrElse(ValueFactory.newFloat(v.toDouble))).asJava))
+    // FIXME: Need Encode?
+    lazy val _bs   = Option(value.getBS).map(l =>
+      ValueFactory.newArray(l.asScala.map(v => ValueFactory.newBinary(v.array)).asJava))
+    lazy val _map  = Option(value.getM).map(m =>
+      ValueFactory.newMap(m.asScala.map(v => ValueFactory.newString(v._1) -> decodeToValue(v._2)).asJava))
+
+    _bin.orElse(_bool).orElse(_num).orElse(_str).orElse(_nil)
+      .orElse(_list).orElse(_ss).orElse(_ns).orElse(_bs).orElse(_map) match {
+      case None => ValueFactory.newNil
+      case Some(j) => j
+    }
+  }
+}

data/src/main/scala/org/embulk/input/dynamodb/AwsCredentials.scala

@@ -0,0 +1,71 @@
+package org.embulk.input.dynamodb
+
+import com.amazonaws.auth._
+import com.amazonaws.auth.profile.ProfileCredentialsProvider
+import com.google.common.base.Optional
+import org.embulk.config.ConfigException
+
+object AwsCredentials {
+  def getCredentialsProvider(task: PluginTask): AWSCredentialsProvider = {
+    if (!task.getAuthMethod.isPresent) {
+      // backward compatibility
+      if (task.getAccessKey.isPresent && task.getSecretKey.isPresent) {
+        new AWSCredentialsProvider {
+          override def refresh(): Unit = {}
+
+          override def getCredentials: AWSCredentials = {
+            new BasicAWSCredentials(
+              task.getAccessKey.get(),
+              task.getSecretKey.get())
+          }
+        }
+      } else {
+        new ProfileCredentialsProvider()
+      }
+    } else {
+      val cred = task.getAuthMethod.get() match {
+        case "basic" =>
+          val accessKey = require(task.getAccessKey, "'access_key'")
+          val secretKey = require(task.getSecretKey, "'secret_key'")
+
+          new BasicAWSCredentials(accessKey, secretKey)
+
+        case "env" =>
+          new EnvironmentVariableCredentialsProvider().getCredentials
+
+        case "instance" =>
+          new InstanceProfileCredentialsProvider().getCredentials
+
+        case "profile" =>
+          val profileName = task.getProfileName.or("default")
+
+          try {
+            new ProfileCredentialsProvider(profileName).getCredentials
+          } catch {
+            case e: IllegalArgumentException =>
+              throw new ConfigException(s"No AWS profile named $profileName")
+          }
+
+        case "properties" =>
+          new SystemPropertiesCredentialsProvider().getCredentials
+
+        case _ =>
+          throw new ConfigException(s"Unknown 'auth_method' ${task.getAuthMethod.get()}")
+      }
+
+      new AWSCredentialsProvider {
+        override def refresh(): Unit = {}
+
+        override def getCredentials: AWSCredentials = { cred }
+      }
+    }
+  }
+
+  private def require[A](value: Optional[A], message: String): A = {
+    if (value.isPresent) {
+      value.get()
+    } else {
+      throw new ConfigException("Required option is not set: " + message)
+    }
+  }
+}