embulk-input-dynamodb 0.2.0 → 0.3.0

data/circle.yml

@@ -1,16 +0,0 @@
-machine:
-  services:
-    - docker
-
-general:
-  artifacts:
-    - "~/embulk-input-dynamodb/build/reports/tests"
-
-test:
-  pre:
-    - sh ./test/run_dynamodb_local.sh; sleep 2
-    - sh ./test/create_table.sh
-    - sh ./test/put_items.sh
-  post:
-    - mkdir -p $CIRCLE_TEST_REPORTS/junit/
-    - find . -type f -regex ".*/test-results/*/.*xml" -exec cp {} $CIRCLE_TEST_REPORTS/junit/ \;

data/config/checkstyle/checkstyle.xml

@@ -1,128 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE module PUBLIC
-        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
-        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
-<module name="Checker">
-    <!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
-    <module name="FileTabCharacter"/>
-    <module name="NewlineAtEndOfFile">
-        <property name="lineSeparator" value="lf"/>
-    </module>
-    <module name="RegexpMultiline">
-        <property name="format" value="\r"/>
-        <property name="message" value="Line contains carriage return"/>
-    </module>
-    <module name="RegexpMultiline">
-        <property name="format" value=" \n"/>
-        <property name="message" value="Line has trailing whitespace"/>
-    </module>
-    <module name="RegexpMultiline">
-        <property name="format" value="\{\n\n"/>
-        <property name="message" value="Blank line after opening brace"/>
-    </module>
-    <module name="RegexpMultiline">
-        <property name="format" value="\n\n\s*\}"/>
-        <property name="message" value="Blank line before closing brace"/>
-    </module>
-    <module name="RegexpMultiline">
-        <property name="format" value="\n\n\n"/>
-        <property name="message" value="Multiple consecutive blank lines"/>
-    </module>
-    <module name="RegexpMultiline">
-        <property name="format" value="\n\n\Z"/>
-        <property name="message" value="Blank line before end of file"/>
-    </module>
-    <module name="RegexpMultiline">
-        <property name="format" value="Preconditions\.checkNotNull"/>
-        <property name="message" value="Use of checkNotNull"/>
-    </module>
-
-    <module name="TreeWalker">
-        <module name="EmptyBlock">
-            <property name="option" value="text"/>
-            <property name="tokens" value="
-                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
-                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
-        </module>
-        <module name="EmptyStatement"/>
-        <module name="EmptyForInitializerPad"/>
-        <module name="EmptyForIteratorPad">
-            <property name="option" value="space"/>
-        </module>
-        <module name="MethodParamPad">
-            <property name="allowLineBreaks" value="true"/>
-            <property name="option" value="nospace"/>
-        </module>
-        <module name="ParenPad"/>
-        <module name="TypecastParenPad"/>
-        <module name="NeedBraces"/>
-        <module name="LeftCurly">
-            <property name="option" value="nl"/>
-            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
-        </module>
-        <module name="LeftCurly">
-            <property name="option" value="eol"/>
-            <property name="tokens" value="
-                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
-                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
-        </module>
-        <module name="RightCurly">
-            <property name="option" value="alone"/>
-        </module>
-        <module name="GenericWhitespace"/>
-        <module name="WhitespaceAfter"/>
-        <module name="NoWhitespaceBefore"/>
-
-        <module name="UpperEll"/>
-        <module name="DefaultComesLast"/>
-        <module name="ArrayTypeStyle"/>
-        <module name="MultipleVariableDeclarations"/>
-        <module name="ModifierOrder"/>
-        <module name="OneStatementPerLine"/>
-        <module name="StringLiteralEquality"/>
-        <module name="MutableException"/>
-        <module name="EqualsHashCode"/>
-        <module name="InnerAssignment"/>
-        <module name="InterfaceIsType"/>
-        <module name="HideUtilityClassConstructor"/>
-
-        <module name="MemberName"/>
-        <module name="LocalVariableName"/>
-        <module name="LocalFinalVariableName"/>
-        <module name="TypeName"/>
-        <module name="PackageName"/>
-        <module name="ParameterName"/>
-        <module name="StaticVariableName"/>
-        <module name="ClassTypeParameterName">
-            <property name="format" value="^[A-Z][0-9]?$"/>
-        </module>
-        <module name="MethodTypeParameterName">
-            <property name="format" value="^[A-Z][0-9]?$"/>
-        </module>
-
-        <module name="AvoidStarImport"/>
-        <module name="RedundantImport"/>
-        <module name="UnusedImports"/>
-        <module name="ImportOrder">
-            <property name="groups" value="*,javax,java"/>
-            <property name="separated" value="true"/>
-            <property name="option" value="bottom"/>
-            <property name="sortStaticImportsAlphabetically" value="true"/>
-        </module>
-
-        <module name="WhitespaceAround">
-            <property name="allowEmptyConstructors" value="true"/>
-            <property name="allowEmptyMethods" value="true"/>
-            <property name="ignoreEnhancedForColon" value="false"/>
-            <property name="tokens" value="
-                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
-                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
-                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
-                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
-                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
-                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
-                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
-                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
-        </module>
-    </module>
-</module>

data/config/checkstyle/default.xml

@@ -1,108 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE module PUBLIC
-        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
-        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
-<!--
-     This is a subset of ./checkstyle.xml which allows some loose styles
--->
-<module name="Checker">
-    <module name="FileTabCharacter"/>
-    <module name="NewlineAtEndOfFile">
-        <property name="lineSeparator" value="lf"/>
-    </module>
-    <module name="RegexpMultiline">
-        <property name="format" value="\r"/>
-        <property name="message" value="Line contains carriage return"/>
-    </module>
-    <module name="RegexpMultiline">
-        <property name="format" value=" \n"/>
-        <property name="message" value="Line has trailing whitespace"/>
-    </module>
-    <module name="RegexpMultiline">
-        <property name="format" value="\n\n\n"/>
-        <property name="message" value="Multiple consecutive blank lines"/>
-    </module>
-    <module name="RegexpMultiline">
-        <property name="format" value="\n\n\Z"/>
-        <property name="message" value="Blank line before end of file"/>
-    </module>
-
-    <module name="TreeWalker">
-        <module name="EmptyBlock">
-            <property name="option" value="text"/>
-            <property name="tokens" value="
-                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
-                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
-        </module>
-        <module name="EmptyStatement"/>
-        <module name="EmptyForInitializerPad"/>
-        <module name="EmptyForIteratorPad">
-            <property name="option" value="space"/>
-        </module>
-        <module name="MethodParamPad">
-            <property name="allowLineBreaks" value="true"/>
-            <property name="option" value="nospace"/>
-        </module>
-        <module name="ParenPad"/>
-        <module name="TypecastParenPad"/>
-        <module name="NeedBraces"/>
-        <module name="LeftCurly">
-            <property name="option" value="nl"/>
-            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
-        </module>
-        <module name="LeftCurly">
-            <property name="option" value="eol"/>
-            <property name="tokens" value="
-                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
-                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
-        </module>
-        <module name="RightCurly">
-            <property name="option" value="alone"/>
-        </module>
-        <module name="GenericWhitespace"/>
-        <module name="WhitespaceAfter"/>
-        <module name="NoWhitespaceBefore"/>
-
-        <module name="UpperEll"/>
-        <module name="DefaultComesLast"/>
-        <module name="ArrayTypeStyle"/>
-        <module name="MultipleVariableDeclarations"/>
-        <module name="ModifierOrder"/>
-        <module name="OneStatementPerLine"/>
-        <module name="StringLiteralEquality"/>
-        <module name="MutableException"/>
-        <module name="EqualsHashCode"/>
-        <module name="InnerAssignment"/>
-        <module name="InterfaceIsType"/>
-        <module name="HideUtilityClassConstructor"/>
-
-        <module name="MemberName"/>
-        <module name="LocalVariableName"/>
-        <module name="LocalFinalVariableName"/>
-        <module name="TypeName"/>
-        <module name="PackageName"/>
-        <module name="ParameterName"/>
-        <module name="StaticVariableName"/>
-        <module name="ClassTypeParameterName">
-            <property name="format" value="^[A-Z][0-9]?$"/>
-        </module>
-        <module name="MethodTypeParameterName">
-            <property name="format" value="^[A-Z][0-9]?$"/>
-        </module>
-
-        <module name="WhitespaceAround">
-            <property name="allowEmptyConstructors" value="true"/>
-            <property name="allowEmptyMethods" value="true"/>
-            <property name="ignoreEnhancedForColon" value="false"/>
-            <property name="tokens" value="
-                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
-                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
-                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
-                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
-                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
-                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
-                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
-                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
-        </module>
-    </module>
-</module>

data/src/main/scala/org/embulk/input/dynamodb/AttributeValueHelper.scala

@@ -1,41 +0,0 @@
-package org.embulk.input.dynamodb
-
-import com.amazonaws.services.dynamodbv2.model.AttributeValue
-import org.msgpack.value.{Value, ValueFactory}
-
-import scala.util.Try
-
-object AttributeValueHelper {
-
-  // referring aws-scala
-  def decodeToValue(value: AttributeValue): Value = {
-    import scala.collection.JavaConverters._
-
-    // FIXME: Need Encode?
-    lazy val _bin  = Option(value.getB).map(v => ValueFactory.newBinary(v.array))
-    lazy val _bool = Option(value.getBOOL).map(v => ValueFactory.newBoolean(v))
-    lazy val _num  = Option(value.getN).map(v =>
-      Try(v.toLong).map(ValueFactory.newInteger).getOrElse(ValueFactory.newFloat(v.toDouble)))
-    lazy val _str  = Option(value.getS).map(v => ValueFactory.newString(v))
-    lazy val _nil  = Option(value.getNULL).map(v => ValueFactory.newNil)
-
-    lazy val _list = Option(value.getL).map(l =>
-      ValueFactory.newArray(l.asScala.map(v => decodeToValue(v)).asJava))
-    lazy val _ss   = Option(value.getSS).map(l =>
-      ValueFactory.newArray(l.asScala.map(v => ValueFactory.newString(v)).asJava))
-    lazy val _ns   = Option(value.getNS).map(l =>
-      ValueFactory.newArray(l.asScala.map(v =>
-        Try(v.toLong).map(ValueFactory.newInteger).getOrElse(ValueFactory.newFloat(v.toDouble))).asJava))
-    // FIXME: Need Encode?
-    lazy val _bs   = Option(value.getBS).map(l =>
-      ValueFactory.newArray(l.asScala.map(v => ValueFactory.newBinary(v.array)).asJava))
-    lazy val _map  = Option(value.getM).map(m =>
-      ValueFactory.newMap(m.asScala.map(v => ValueFactory.newString(v._1) -> decodeToValue(v._2)).asJava))
-
-    _bin.orElse(_bool).orElse(_num).orElse(_str).orElse(_nil)
-      .orElse(_list).orElse(_ss).orElse(_ns).orElse(_bs).orElse(_map) match {
-      case None => ValueFactory.newNil
-      case Some(j) => j
-    }
-  }
-}

data/src/main/scala/org/embulk/input/dynamodb/AwsCredentials.scala

@@ -1,63 +0,0 @@
-package org.embulk.input.dynamodb
-
-import com.amazonaws.auth._
-import com.amazonaws.auth.profile.ProfileCredentialsProvider
-import com.google.common.base.Optional
-import org.embulk.config.ConfigException
-
-object AwsCredentials {
-  def getCredentialsProvider(task: PluginTask): AWSCredentialsProvider = {
-    if (!task.getAuthMethod.isPresent) {
-      // backward compatibility
-      if (task.getAccessKey.isPresent && task.getSecretKey.isPresent) {
-        new AWSCredentialsProvider {
-          override def refresh(): Unit = {}
-
-          override def getCredentials: AWSCredentials = {
-            new BasicAWSCredentials(
-              task.getAccessKey.get(),
-              task.getSecretKey.get())
-          }
-        }
-      } else {
-        new ProfileCredentialsProvider()
-      }
-    } else {
-      val cred = task.getAuthMethod.get() match {
-        case "basic" =>
-          val accessKey = require(task.getAccessKey, "'access_key'")
-          val secretKey = require(task.getSecretKey, "'secret_key'")
-
-          new BasicAWSCredentials(accessKey, secretKey)
-
-        case "env" =>
-          new EnvironmentVariableCredentialsProvider().getCredentials
-
-        case "instance" =>
-          new InstanceProfileCredentialsProvider().getCredentials
-
-        case "profile" =>
-          val profileName = task.getProfileName.or("default")
-
-          try {
-            new ProfileCredentialsProvider(profileName).getCredentials
-          } catch {
-            case e: IllegalArgumentException =>
-              throw new ConfigException(s"No AWS profile named $profileName")
-          }
-
-        case "properties" =>
-          new SystemPropertiesCredentialsProvider().getCredentials
-
-        case _ =>
-          throw new ConfigException(s"Unknown 'auth_method' ${task.getAuthMethod.get()}")
-      }
-
-      new AWSCredentialsProvider {
-        override def refresh(): Unit = {}
-
-        override def getCredentials: AWSCredentials = { cred }
-      }
-    }
-  }
-}

data/src/main/scala/org/embulk/input/dynamodb/DynamoDBClient.scala

@@ -1,23 +0,0 @@
-package org.embulk.input.dynamodb
-
-import com.amazonaws.ClientConfiguration
-import com.amazonaws.regions.Regions
-import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient
-import org.embulk.config.ConfigException
-
-object DynamoDBClient {
-  def create(task: PluginTask): AmazonDynamoDBClient = {
-    val client = new AmazonDynamoDBClient(
-      AwsCredentials.getCredentialsProvider(task),
-      new ClientConfiguration()
-        .withMaxConnections(50))  // SDK Default Value
-
-    if (task.getEndPoint.isPresent) {
-      client.withEndpoint(task.getEndPoint.get())
-    } else if (task.getRegion.isPresent) {
-      client.withRegion(Regions.fromName(task.getRegion.get()))
-    } else {
-      throw new ConfigException("At least one of EndPoint or Region must be set")
-    }
-  }
-}

data/src/test/resources/yaml/authMethodBasic.yml

@@ -1,21 +0,0 @@
-in:
-  type: dynamodb
-  region: ENV_VAR
-  operation: scan
-  table: ENV_VAR
-  auth_method: basic
-  access_key: ENV_VAR
-  secret_key: ENV_VAR
-  columns:
-    - {name: pri-key,  type: string}
-    - {name: sort-key, type: long}
-    - {name: value, type: string}
-
-out:
-  type: file
-  path_prefix: result
-  file_ext: tsv
-  formatter:
-    type: csv
-    delimiter: "\t"
-    header_line: false

data/src/test/resources/yaml/authMethodBasic_Error.yml

@@ -1,19 +0,0 @@
-in:
-  type: dynamodb
-  region: ENV_VAR
-  operation: scan
-  table: ENV_VAR
-  auth_method: basic
-  columns:
-    - {name: pri-key,  type: string}
-    - {name: sort-key, type: long}
-    - {name: value, type: string}
-
-out:
-  type: file
-  path_prefix: result
-  file_ext: tsv
-  formatter:
-    type: csv
-    delimiter: "\t"
-    header_line: false

data/src/test/resources/yaml/authMethodEnv.yml

@@ -1,19 +0,0 @@
-in:
-  type: dynamodb
-  region: ENV_VAR
-  operation: scan
-  table: ENV_VAR
-  auth_method: env
-  columns:
-    - {name: pri-key,  type: string}
-    - {name: sort-key, type: long}
-    - {name: value, type: string}
-
-out:
-  type: file
-  path_prefix: result
-  file_ext: tsv
-  formatter:
-    type: csv
-    delimiter: "\t"
-    header_line: false