embulk-input-dynamodb 0.2.0 → 0.3.0

data/src/main/scala/org/embulk/input/dynamodb/aws/AwsDynamodbConfiguration.scala

@@ -0,0 +1,35 @@
+package org.embulk.input.dynamodb.aws
+
+import java.util.Optional
+
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder
+import org.embulk.config.{Config, ConfigDefault}
+import org.embulk.input.dynamodb.aws.AwsDynamodbConfiguration.Task
+
+object AwsDynamodbConfiguration {
+
+  trait Task {
+
+    @Config("enable_endpoint_discovery")
+    @ConfigDefault("null")
+    def getEnableEndpointDiscovery: Optional[Boolean]
+
+  }
+
+  def apply(task: Task): AwsDynamodbConfiguration = {
+    new AwsDynamodbConfiguration(task)
+  }
+}
+
+class AwsDynamodbConfiguration(task: Task) {
+
+  def configureAmazonDynamoDBClientBuilder(
+      builder: AmazonDynamoDBClientBuilder
+  ): Unit = {
+    task.getEnableEndpointDiscovery.ifPresent { v =>
+      if (v) builder.enableEndpointDiscovery()
+      else builder.disableEndpointDiscovery()
+    }
+  }
+
+}

data/src/main/scala/org/embulk/input/dynamodb/aws/AwsEndpointConfiguration.scala

@@ -0,0 +1,79 @@
+package org.embulk.input.dynamodb.aws
+
+import java.util.Optional
+
+import com.amazonaws.client.builder.AwsClientBuilder
+import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
+import com.amazonaws.regions.{DefaultAwsRegionProviderChain, Regions}
+import org.embulk.config.{Config, ConfigDefault, ConfigException}
+import org.embulk.input.dynamodb.aws.AwsEndpointConfiguration.Task
+import org.embulk.input.dynamodb.logger
+import zio.macros.annotation.delegate
+
+import scala.util.Try
+
+object AwsEndpointConfiguration {
+
+  trait Task {
+
+    @deprecated(message = "Use #getEndpoint() instead.", since = "0.3.0")
+    @Config("end_point")
+    @ConfigDefault("null")
+    def getEndPoint: Optional[String]
+
+    @Config("endpoint")
+    @ConfigDefault("null")
+    def getEndpoint: Optional[String]
+
+    @Config("region")
+    @ConfigDefault("null")
+    def getRegion: Optional[String]
+  }
+
+  def apply(task: Task): AwsEndpointConfiguration = {
+    new AwsEndpointConfiguration(AwsEndpointConfigurationTaskCompat(task))
+  }
+}
+
+case class AwsEndpointConfigurationTaskCompat(@delegate task: Task)
+    extends Task {
+  override def getEndPoint: Optional[String] = throw new NotImplementedError()
+
+  override def getEndpoint: Optional[String] = {
+    if (task.getEndpoint.isPresent && task.getEndPoint.isPresent)
+      throw new ConfigException(
+        "You cannot use both \"endpoint\" option and \"end_point\" option. Use \"endpoint\" option."
+      )
+    if (task.getEndPoint.isPresent) {
+      logger.warn(
+        "[Deprecated] \"end_point\" option is deprecated. Use \"endpoint\" option instead."
+      )
+      return task.getEndPoint
+    }
+    task.getEndpoint
+  }
+}
+
+class AwsEndpointConfiguration(task: Task) {
+
+  def configureAwsClientBuilder[S <: AwsClientBuilder[S, T], T](
+      builder: AwsClientBuilder[S, T]
+  ): Unit = {
+    if (task.getRegion.isPresent && task.getEndpoint.isPresent) {
+      val ec =
+        new EndpointConfiguration(task.getEndpoint.get, task.getRegion.get)
+      builder.setEndpointConfiguration(ec)
+    }
+    else if (task.getRegion.isPresent && !task.getEndpoint.isPresent) {
+      builder.setRegion(task.getRegion.get)
+    }
+    else if (!task.getRegion.isPresent && task.getEndpoint.isPresent) {
+      val r: String = Try(new DefaultAwsRegionProviderChain().getRegion)
+        .getOrElse(Regions.DEFAULT_REGION.getName)
+      val e: String = task.getEndpoint.get
+      val ec = new EndpointConfiguration(e, r)
+      builder.setEndpointConfiguration(ec)
+    }
+  }
+
+}

data/src/main/scala/org/embulk/input/dynamodb/aws/HttpProxy.scala

@@ -0,0 +1,61 @@
+package org.embulk.input.dynamodb.aws
+
+import java.util.Optional
+
+import com.amazonaws.{ClientConfiguration, Protocol}
+import org.embulk.config.{Config, ConfigDefault, ConfigException}
+import org.embulk.input.dynamodb.aws.HttpProxy.Task
+
+object HttpProxy {
+
+  trait Task {
+
+    @Config("host")
+    @ConfigDefault("null")
+    def getHost: Optional[String]
+
+    @Config("port")
+    @ConfigDefault("null")
+    def getPort: Optional[Int]
+
+    @Config("protocol")
+    @ConfigDefault("\"https\"")
+    def getProtocol: String
+
+    @Config("user")
+    @ConfigDefault("null")
+    def getUser: Optional[String]
+
+    @Config("password")
+    @ConfigDefault("null")
+    def getPassword: Optional[String]
+
+  }
+
+  def apply(task: Task): HttpProxy = {
+    new HttpProxy(task)
+  }
+
+}
+
+class HttpProxy(task: Task) {
+
+  def configureClientConfiguration(cc: ClientConfiguration): Unit = {
+    task.getHost.ifPresent(v => cc.setProxyHost(v))
+    task.getPort.ifPresent(v => cc.setProxyPort(v))
+
+    Protocol.values.find(p => p.name().equals(task.getProtocol)) match {
+      case Some(v) =>
+        cc.setProtocol(v)
+      case None =>
+        throw new ConfigException(
+          s"'${task.getProtocol}' is unsupported: `protocol` must be one of [${Protocol.values
+            .map(v => s"'$v'")
+            .mkString(", ")}]."
+        )
+    }
+
+    task.getUser.ifPresent(v => cc.setProxyUsername(v))
+    task.getPassword.ifPresent(v => cc.setProxyPassword(v))
+  }
+}

data/src/main/scala/org/embulk/input/dynamodb/deprecated/AttributeValueHelper.scala

@@ -0,0 +1,72 @@
+package org.embulk.input.dynamodb.deprecated
+
+import com.amazonaws.services.dynamodbv2.model.AttributeValue
+import org.msgpack.value.{Value, ValueFactory}
+
+import scala.util.Try
+
+object AttributeValueHelper {
+
+  // referring aws-scala
+  def decodeToValue(value: AttributeValue): Value = {
+    import scala.jdk.CollectionConverters._
+
+    // FIXME: Need Encode?
+    lazy val _bin = Option(value.getB).map(v => ValueFactory.newBinary(v.array))
+    lazy val _bool = Option(value.getBOOL).map(v => ValueFactory.newBoolean(v))
+    lazy val _num = Option(value.getN).map(v =>
+      Try(v.toLong)
+        .map(ValueFactory.newInteger)
+        .getOrElse(ValueFactory.newFloat(v.toDouble))
+    )
+    lazy val _str = Option(value.getS).map(v => ValueFactory.newString(v))
+    lazy val _nil = Option(value.getNULL).map(v => ValueFactory.newNil)
+
+    lazy val _list = Option(value.getL).map(l =>
+      ValueFactory.newArray(l.asScala.map(v => decodeToValue(v)).asJava)
+    )
+    lazy val _ss = Option(value.getSS).map(l =>
+      ValueFactory.newArray(
+        l.asScala.map(v => ValueFactory.newString(v)).asJava
+      )
+    )
+    lazy val _ns = Option(value.getNS).map(l =>
+      ValueFactory.newArray(
+        l.asScala
+          .map(v =>
+            Try(v.toLong)
+              .map(ValueFactory.newInteger)
+              .getOrElse(ValueFactory.newFloat(v.toDouble))
+          )
+          .asJava
+      )
+    )
+    // FIXME: Need Encode?
+    lazy val _bs = Option(value.getBS).map(l =>
+      ValueFactory.newArray(
+        l.asScala.map(v => ValueFactory.newBinary(v.array)).asJava
+      )
+    )
+    lazy val _map = Option(value.getM).map(m =>
+      ValueFactory.newMap(
+        m.asScala
+          .map(v => ValueFactory.newString(v._1) -> decodeToValue(v._2))
+          .asJava
+      )
+    )
+
+    _bin
+      .orElse(_bool)
+      .orElse(_num)
+      .orElse(_str)
+      .orElse(_nil)
+      .orElse(_list)
+      .orElse(_ss)
+      .orElse(_ns)
+      .orElse(_bs)
+      .orElse(_map) match {
+      case None    => ValueFactory.newNil
+      case Some(j) => j
+    }
+  }
+}

data/src/main/scala/org/embulk/input/dynamodb/{Filter.scala → deprecated/Filter.scala}

@@ -1,4 +1,4 @@
-package org.embulk.input.dynamodb
+package org.embulk.input.dynamodb.deprecated
 
 import java.util.{List => JList}
 
@@ -18,9 +18,9 @@ class Filter {
   def getFilters: JList[FilterConfig] = filters
 
   override def equals(obj: Any): Boolean = {
-    if(this == obj) return true
+    if (this == obj) return true
 
-    if(!obj.isInstanceOf[Filter]) return false
+    if (!obj.isInstanceOf[Filter]) return false
 
     val other: Filter = obj.asInstanceOf[Filter]
     Objects.equal(filters, other.filters)

data/src/main/scala/org/embulk/input/dynamodb/{FilterConfig.scala → deprecated/FilterConfig.scala}

@@ -1,4 +1,4 @@
-package org.embulk.input.dynamodb
+package org.embulk.input.dynamodb.deprecated
 
 import com.fasterxml.jackson.annotation.JsonProperty
 import com.google.common.base.Objects
@@ -11,12 +11,12 @@ class FilterConfig {
   private var _value2: String = _
 
   def this(
-            @JsonProperty("name") _name: String,
-            @JsonProperty("type") _type: String,
-            @JsonProperty("condition") _condition: String,
-            @JsonProperty("value") _value: String,
-            @JsonProperty("value2") _value2: String)
-  {
+      @JsonProperty("name") _name: String,
+      @JsonProperty("type") _type: String,
+      @JsonProperty("condition") _condition: String,
+      @JsonProperty("value") _value: String,
+      @JsonProperty("value2") _value2: String
+  ) {
     this()
     this._name = _name
     this._type = _type
@@ -41,15 +41,15 @@ class FilterConfig {
   def getValue2 = _value2
 
   override def equals(obj: Any): Boolean = {
-    if(this == obj) return true
+    if (this == obj) return true
 
-    if(!obj.isInstanceOf[FilterConfig]) return false
+    if (!obj.isInstanceOf[FilterConfig]) return false
 
     val other: FilterConfig = obj.asInstanceOf[FilterConfig]
     Objects.equal(this._name, other._name) &&
-      Objects.equal(this._type, other._type) &&
-      Objects.equal(this._condition, other._condition) &&
-      Objects.equal(this._value, other._value) &&
-      Objects.equal(this._value2, other._value2)
+    Objects.equal(this._type, other._type) &&
+    Objects.equal(this._condition, other._condition) &&
+    Objects.equal(this._value, other._value) &&
+    Objects.equal(this._value2, other._value2)
   }
 }

data/src/main/scala/org/embulk/input/dynamodb/{ope → deprecated/ope}/AbstractOperation.scala

@@ -1,12 +1,13 @@
-package org.embulk.input.dynamodb.ope
+package org.embulk.input.dynamodb.deprecated.ope
 
 import com.amazonaws.services.dynamodbv2.model.{AttributeValue, Condition}
-import org.embulk.input.dynamodb.{AttributeValueHelper, PluginTask}
+import org.embulk.input.dynamodb.PluginTask
+import org.embulk.input.dynamodb.deprecated.AttributeValueHelper
 import org.embulk.spi._
 import org.embulk.spi.`type`.Types
 import org.msgpack.value.{Value, ValueFactory}
 
-import scala.collection.JavaConverters._
+import scala.jdk.CollectionConverters._
 
 abstract class AbstractOperation {
   def execute(task: PluginTask, schema: Schema, output: PageOutput): Unit
@@ -14,20 +15,29 @@ abstract class AbstractOperation {
   def getLimit(limit: Long, recordLimit: Long, recordCount: Long): Int = {
     if (limit > 0 && recordLimit > 0) {
       math.min(limit, recordLimit - recordCount).toInt
-    } else if (limit > 0 || recordLimit > 0) {
+    }
+    else if (limit > 0 || recordLimit > 0) {
       math.max(limit, recordLimit).toInt
-    } else { 0 }
+    }
+    else {
+      0
+    }
   }
 
   def createFilters(task: PluginTask): Map[String, Condition] = {
     val filterMap = collection.mutable.HashMap[String, Condition]()
 
-    Option(task.getFilters.orNull).map { filters =>
+    Option(task.getFilters.orElse(null)).map { filters =>
       filters.getFilters.asScala.map { filter =>
-        val attributeValueList = collection.mutable.ArrayBuffer[AttributeValue]()
-        attributeValueList += createAttributeValue(filter.getType, filter.getValue)
+        val attributeValueList =
+          collection.mutable.ArrayBuffer[AttributeValue]()
+        attributeValueList += createAttributeValue(
+          filter.getType,
+          filter.getValue
+        )
         Option(filter.getValue2).map { value2 =>
-          attributeValueList+= createAttributeValue(filter.getType, value2) }
+          attributeValueList += createAttributeValue(filter.getType, value2)
+        }
 
         filterMap += filter.getName -> new Condition()
           .withComparisonOperator(filter.getCondition)
@@ -49,7 +59,11 @@ abstract class AbstractOperation {
     }
   }
 
-  def write(pageBuilder: PageBuilder, schema: Schema, items: Seq[Map[String, AttributeValue]]): Long = {
+  def write(
+      pageBuilder: PageBuilder,
+      schema: Schema,
+      items: Seq[Map[String, AttributeValue]]
+  ): Long = {
     var count = 0
 
     items.foreach { item =>
@@ -76,26 +90,30 @@ abstract class AbstractOperation {
     count
   }
 
-  def convert[A](column: Column,
-                   value: Option[AttributeValue],
-                   f: (Column, A) => Unit)(implicit f1: Option[AttributeValue] => A): Unit =
+  def convert[A](
+      column: Column,
+      value: Option[AttributeValue],
+      f: (Column, A) => Unit
+  )(implicit f1: Option[AttributeValue] => A): Unit =
     f(column, f1(value))
 
   implicit def StringConvert(value: Option[AttributeValue]): String =
     value.map(_.getS).getOrElse("")
 
   implicit def LongConvert(value: Option[AttributeValue]): Long =
-    value.map(_.getN.toLong).getOrElse(0L)
+    value.map(_.getN).flatMap(Option(_)).map(_.toLong).getOrElse(0L)
 
   implicit def DoubleConvert(value: Option[AttributeValue]): Double =
-    value.map(_.getN.toDouble).getOrElse(0D)
+    value.map(_.getN).flatMap(Option(_)).map(_.toDouble).getOrElse(0d)
 
   implicit def BooleanConvert(value: Option[AttributeValue]): Boolean =
     value.exists(_.getBOOL)
 
   implicit def JsonConvert(value: Option[AttributeValue]): Value = {
-    value.map { attr =>
-      AttributeValueHelper.decodeToValue(attr)
-    }.getOrElse(ValueFactory.newNil())
+    value
+      .map { attr =>
+        AttributeValueHelper.decodeToValue(attr)
+      }
+      .getOrElse(ValueFactory.newNil())
   }
 }

data/src/main/scala/org/embulk/input/dynamodb/{ope → deprecated/ope}/QueryOperation.scala

@@ -1,27 +1,35 @@
-package org.embulk.input.dynamodb.ope
+package org.embulk.input.dynamodb.deprecated.ope
 
 import java.util.{List => JList, Map => JMap}
 
-import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient
-import com.amazonaws.services.dynamodbv2.model.{AttributeValue, Condition, QueryRequest, QueryResult}
+import com.amazonaws.services.dynamodbv2.{AmazonDynamoDB, AmazonDynamoDBClient}
+import com.amazonaws.services.dynamodbv2.model.{
+  AttributeValue,
+  Condition,
+  QueryRequest,
+  QueryResult
+}
 import org.embulk.input.dynamodb.PluginTask
 import org.embulk.spi.{BufferAllocator, PageBuilder, PageOutput, Schema}
 
-import scala.collection.JavaConverters._
+import scala.jdk.CollectionConverters._
+
+class QueryOperation(client: AmazonDynamoDB) extends AbstractOperation {
 
-class QueryOperation(client: AmazonDynamoDBClient) extends AbstractOperation {
-  override def execute(task: PluginTask,
-                       schema: Schema,
-                       output: PageOutput): Unit =
-  {
+  override def execute(
+      task: PluginTask,
+      schema: Schema,
+      output: PageOutput
+  ): Unit = {
     val allocator: BufferAllocator = task.getBufferAllocator
     val pageBuilder: PageBuilder = new PageBuilder(allocator, schema, output)
 
-    val attributes: JList[String] = schema.getColumns.asScala.map(_.getName).asJava
+    val attributes: JList[String] =
+      schema.getColumns.asScala.map(_.getName).asJava
     val conditions: JMap[String, Condition] = createFilters(task).asJava
     var evaluateKey: JMap[String, AttributeValue] = null
 
-    val limit: Long   = math.max(task.getScanLimit, task.getLimit)
+    val limit: Long = math.max(task.getScanLimit, task.getLimit)
     val recordLimit: Long = task.getRecordLimit
     var recordCount: Long = 0
 
@@ -41,9 +49,9 @@ class QueryOperation(client: AmazonDynamoDBClient) extends AbstractOperation {
       val result: QueryResult = client.query(request)
       evaluateKey = result.getLastEvaluatedKey
 
-      val items = result.getItems.asScala.map(_.asScala.toMap)
+      val items = result.getItems.asScala.map(_.asScala.toMap).toSeq
       recordCount += write(pageBuilder, schema, items)
-    } while(evaluateKey != null && (recordLimit == 0 || recordLimit > recordCount))
+    } while (evaluateKey != null && (recordLimit == 0 || recordLimit > recordCount))
 
     pageBuilder.finish()
   }