embulk-input-dynamodb 0.2.0 → 0.3.0

data/src/main/scala/org/embulk/input/dynamodb/DynamodbInputPlugin.scala

@@ -2,46 +2,97 @@ package org.embulk.input.dynamodb
 
 import java.util.{List => JList}
 
-import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient
-import org.embulk.config._
-import org.embulk.input.dynamodb.ope.{QueryOperation, ScanOperation}
-import org.embulk.spi._
+import org.embulk.config.{ConfigDiff, ConfigSource, TaskReport, TaskSource}
+import org.embulk.input.dynamodb.aws.Aws
+import org.embulk.input.dynamodb.item.DynamodbItemSchema
+import org.embulk.input.dynamodb.operation.DynamodbOperationProxy
+import org.embulk.spi.{Exec, InputPlugin, PageBuilder, PageOutput, Schema}
 
 class DynamodbInputPlugin extends InputPlugin {
-  def transaction(config: ConfigSource, control: InputPlugin.Control): ConfigDiff = {
-    val task: PluginTask = config.loadConfig(classOf[PluginTask])
 
-    val schema: Schema = task.getColumns.toSchema
-    val taskCount: Int = 1
+  override def transaction(
+      config: ConfigSource,
+      control: InputPlugin.Control
+  ): ConfigDiff = {
+    val task: PluginTask = PluginTask.load(config)
+    if (isDeprecatedOperationRequired(task))
+      return DeprecatedDynamodbInputPlugin.transaction(config, control)
 
-    resume(task.dump(), schema, taskCount, control)
-  }
+    val schema: Schema = DynamodbItemSchema(task).getEmbulkSchema
+    val taskCount: Int = DynamodbOperationProxy(task).getEmbulkTaskCount
 
-  def resume(taskSource: TaskSource, schema: Schema, taskCount: Int, control: InputPlugin.Control): ConfigDiff = {
-    control.run(taskSource, schema, taskCount)
+    control.run(task.dump(), schema, taskCount)
     Exec.newConfigDiff()
   }
 
-  def run(taskSource: TaskSource, schema: Schema, taskIndex: Int, output: PageOutput): TaskReport = {
-    val task: PluginTask = taskSource.loadTask(classOf[PluginTask])
+  override def resume(
+      taskSource: TaskSource,
+      schema: Schema,
+      taskCount: Int,
+      control: InputPlugin.Control
+  ): ConfigDiff = {
+    val task: PluginTask = PluginTask.load(taskSource)
+    if (isDeprecatedOperationRequired(task))
+      return DeprecatedDynamodbInputPlugin.resume(
+        taskSource,
+        schema,
+        taskCount,
+        control
+      )
+    throw new UnsupportedOperationException
+  }
 
-    val client: AmazonDynamoDBClient = DynamoDBClient.create(task)
+  override def run(
+      taskSource: TaskSource,
+      schema: Schema,
+      taskIndex: Int,
+      output: PageOutput
+  ): TaskReport = {
+    val task: PluginTask = PluginTask.load(taskSource)
+    if (isDeprecatedOperationRequired(task))
+      return DeprecatedDynamodbInputPlugin.run(
+        taskSource,
+        schema,
+        taskIndex,
+        output
+      )
 
-    val ope = task.getOperation.toLowerCase match {
-      case "scan"  => new ScanOperation(client)
-      case "query" => new QueryOperation(client)
-    }
-    ope.execute(task, schema, output)
+    val pageBuilder = new PageBuilder(task.getBufferAllocator, schema, output)
 
+    Aws(task).withDynamodb { dynamodb =>
+      DynamodbOperationProxy(task).run(
+        dynamodb,
+        taskIndex,
+        DynamodbItemSchema(task).getItemsConsumer(pageBuilder)
+      )
+    }
+    pageBuilder.finish()
     Exec.newTaskReport()
   }
 
-  def cleanup(taskSource: TaskSource, schema: Schema, taskCount: Int, successTaskReports: JList[TaskReport]): Unit = {
-    // TODO
+  override def cleanup(
+      taskSource: TaskSource,
+      schema: Schema,
+      taskCount: Int,
+      successTaskReports: JList[TaskReport]
+  ): Unit = {
+    val task: PluginTask = PluginTask.load(taskSource)
+    if (isDeprecatedOperationRequired(task))
+      DeprecatedDynamodbInputPlugin.cleanup(
+        taskSource,
+        schema,
+        taskCount,
+        successTaskReports
+      )
   }
 
-  def guess(config: ConfigSource): ConfigDiff = {
-    // TODO
-    null
+  override def guess(config: ConfigSource): ConfigDiff = {
+    val task: PluginTask = PluginTask.load(config)
+    if (isDeprecatedOperationRequired(task))
+      return DeprecatedDynamodbInputPlugin.guess(config)
+    throw new UnsupportedOperationException
   }
+
+  private def isDeprecatedOperationRequired(task: PluginTask): Boolean =
+    task.getOperation.isPresent
 }

data/src/main/scala/org/embulk/input/dynamodb/PluginTask.scala

@@ -1,55 +1,71 @@
 package org.embulk.input.dynamodb
 
-import com.google.common.base.Optional
-import org.embulk.config.{Config, ConfigDefault, ConfigInject, Task}
-import org.embulk.spi.{BufferAllocator, SchemaConfig}
+import java.util.Optional
 
-trait PluginTask extends Task {
-  @Config("auth_method")
-  @ConfigDefault("null")
-  def getAuthMethod: Optional[String]
-
-  @Config("access_key")
-  @ConfigDefault("null")
-  def getAccessKey: Optional[String]
-
-  @Config("secret_key")
-  @ConfigDefault("null")
-  def getSecretKey: Optional[String]
-
-  @Config("profile_name")
-  @ConfigDefault("null")
-  def getProfileName: Optional[String]
+import org.embulk.config.{
+  Config,
+  ConfigDefault,
+  ConfigException,
+  ConfigInject,
+  ConfigSource,
+  Task,
+  TaskSource
+}
+import org.embulk.input.dynamodb.aws.Aws
+import org.embulk.input.dynamodb.deprecated.Filter
+import org.embulk.input.dynamodb.item.DynamodbItemSchema
+import org.embulk.input.dynamodb.operation.DynamodbOperationProxy
+import org.embulk.spi.BufferAllocator
+import zio.macros.annotation.delegate
 
-  @Config("region")
-  @ConfigDefault("null")
-  def getRegion: Optional[String]
+import scala.util.chaining._
 
-  @Config("end_point")
-  @ConfigDefault("null")
-  def getEndPoint: Optional[String]
+trait PluginTask
+    extends Task
+    with Aws.Task
+    with DynamodbItemSchema.Task
+    with DynamodbOperationProxy.Task {
 
+  @deprecated(
+    message = "Use #getScan() or #getQuery() instead.",
+    since = "0.3.0"
+  )
   @Config("operation")
-  def getOperation: String
+  @ConfigDefault("null")
+  def getOperation: Optional[String]
 
+  @deprecated(
+    message =
+      "Use DynamodbQueryOperation.Task#getBatchSize() or DynamodbScanOperation.Task#getBatchSize() instead.",
+    since = "0.3.0"
+  )
   @Config("limit")
   @ConfigDefault("0")
   def getLimit: Long
 
+  @deprecated(
+    message =
+      "Use DynamodbQueryOperation.Task#getBatchSize() or DynamodbScanOperation.Task#getBatchSize() instead.",
+    since = "0.3.0"
+  )
   @Config("scan_limit")
   @ConfigDefault("0")
   def getScanLimit: Long
 
+  @deprecated(
+    message =
+      "Use DynamodbQueryOperation.Task#getLimit() or DynamodbScanOperation.Task#getLimit() instead.",
+    since = "0.3.0"
+  )
   @Config("record_limit")
   @ConfigDefault("0")
   def getRecordLimit: Long
 
-  @Config("table")
-  def getTable: String
-
-  @Config("columns")
-  def getColumns: SchemaConfig
-
+  @deprecated(
+    message =
+      "Use DynamodbQueryOperation.Task#getFilterExpression() or DynamodbScanOperation.Task#getFilterExpression() instead.",
+    since = "0.3.0"
+  )
   @Config("filters")
   @ConfigDefault("null")
   def getFilters: Optional[Filter]
@@ -57,3 +73,87 @@ trait PluginTask extends Task {
   @ConfigInject
   def getBufferAllocator: BufferAllocator
 }
+
+case class PluginTaskCompat(@delegate task: PluginTask) extends PluginTask {
+
+  override def getOperation: Optional[String] = {
+    task.getOperation.ifPresent { op =>
+      logger.warn(
+        "[Deprecated] The option \"operation\" is deprecated. Use \"scan\" or \"query\" option instead."
+      )
+
+      op.toLowerCase match {
+        case "scan" | "query" => // do nothing
+        case x =>
+          throw new ConfigException(
+            s"Operation '$x' is unsupported. Available values are 'scan' or 'query'."
+          )
+      }
+
+      if (getScan.isPresent || getQuery.isPresent)
+        throw new ConfigException(
+          "The option \"operation\" must not be used together with either \"scan\" or \"query\" options."
+        )
+    }
+    task.getOperation
+  }
+
+  override def getLimit: Long = {
+    logger.warn(
+      "[Deprecated] The option \"limit\" is deprecated. Use \"query.batch_size\" or \"scan.batch_size\" instead."
+    )
+    task.getLimit
+  }
+
+  override def getScanLimit: Long = {
+    logger.warn(
+      "[Deprecated] The option \"scan_limit\" is deprecated. Use \"query.batch_size\" or \"scan.batch_size\" instead."
+    )
+    task.getScanLimit
+  }
+
+  override def getRecordLimit: Long = {
+    logger.warn(
+      "[Deprecated] The option \"record_limit\" is deprecated. Use \"query.limit\" or \"scan.limit\" instead."
+    )
+    task.getRecordLimit
+  }
+
+  override def getFilters: Optional[Filter] = {
+    logger.warn(
+      "[Deprecated] The option \"filters\" is deprecated. Use \"query.filter_expression\" or \"scan.filter_expression\" instead."
+    )
+    task.getFilters
+  }
+}
+
+object PluginTask {
+
+  def load(configSource: ConfigSource): PluginTask = {
+    configSource
+      .loadConfig(classOf[PluginTask])
+      .pipe(PluginTaskCompat)
+      .tap(configure)
+  }
+
+  def load(taskSource: TaskSource): PluginTask = {
+    taskSource
+      .loadTask(classOf[PluginTask])
+      .pipe(PluginTaskCompat)
+      .tap(configure)
+  }
+
+  private def configure(task: PluginTask): Unit = {
+    if (!task.getOperation.isPresent && !task.getScan.isPresent && !task.getQuery.isPresent) {
+      // NOTE: "operation" option is deprecated, so this is not shown the message.
+      throw new ConfigException(
+        "Either \"scan\" or \"query\" option is required."
+      )
+    }
+    if (task.getOperation.isPresent && (task.getScan.isPresent || task.getQuery.isPresent)) {
+      throw new ConfigException(
+        "[Deprecated] You must not use \"scan\" or \"query\" option with \"operation\" option."
+      )
+    }
+  }
+}

data/src/main/scala/org/embulk/input/dynamodb/aws/Aws.scala

@@ -0,0 +1,44 @@
+package org.embulk.input.dynamodb.aws
+
+import com.amazonaws.client.builder.AwsClientBuilder
+import com.amazonaws.services.dynamodbv2.{
+  AmazonDynamoDB,
+  AmazonDynamoDBClientBuilder
+}
+
+object Aws {
+
+  trait Task
+      extends AwsCredentials.Task
+      with AwsEndpointConfiguration.Task
+      with AwsClientConfiguration.Task
+      with AwsDynamodbConfiguration.Task
+
+  def apply(task: Task): Aws = {
+    new Aws(task)
+  }
+
+}
+
+class Aws(task: Aws.Task) {
+
+  def withDynamodb[A](f: AmazonDynamoDB => A): A = {
+    val builder: AmazonDynamoDBClientBuilder =
+      AmazonDynamoDBClientBuilder.standard()
+    AwsDynamodbConfiguration(task).configureAmazonDynamoDBClientBuilder(builder)
+    val svc = createService(builder)
+    try f(svc)
+    finally svc.shutdown()
+
+  }
+
+  def createService[S <: AwsClientBuilder[S, T], T](
+      builder: AwsClientBuilder[S, T]
+  ): T = {
+    AwsEndpointConfiguration(task).configureAwsClientBuilder(builder)
+    AwsClientConfiguration(task).configureAwsClientBuilder(builder)
+    builder.setCredentials(AwsCredentials(task).createAwsCredentialsProvider)
+
+    builder.build()
+  }
+}

data/src/main/scala/org/embulk/input/dynamodb/aws/AwsClientConfiguration.scala

@@ -0,0 +1,37 @@
+package org.embulk.input.dynamodb.aws
+
+import java.util.Optional
+
+import com.amazonaws.ClientConfiguration
+import com.amazonaws.client.builder.AwsClientBuilder
+import org.embulk.config.{Config, ConfigDefault}
+import org.embulk.input.dynamodb.aws.AwsClientConfiguration.Task
+
+object AwsClientConfiguration {
+
+  trait Task {
+
+    @Config("http_proxy")
+    @ConfigDefault("null")
+    def getHttpProxy: Optional[HttpProxy.Task]
+
+  }
+
+  def apply(task: Task): AwsClientConfiguration = {
+    new AwsClientConfiguration(task)
+  }
+}
+
+class AwsClientConfiguration(task: Task) {
+
+  def configureAwsClientBuilder[S <: AwsClientBuilder[S, T], T](
+      builder: AwsClientBuilder[S, T]
+  ): Unit = {
+    task.getHttpProxy.ifPresent { v =>
+      val cc = new ClientConfiguration
+      HttpProxy(v).configureClientConfiguration(cc)
+      builder.setClientConfiguration(cc)
+    }
+  }
+
+}

data/src/main/scala/org/embulk/input/dynamodb/aws/AwsCredentials.scala

@@ -0,0 +1,240 @@
+package org.embulk.input.dynamodb.aws
+
+import java.util.Optional
+
+import com.amazonaws.auth.{
+  AnonymousAWSCredentials,
+  AWSCredentialsProvider,
+  AWSStaticCredentialsProvider,
+  BasicAWSCredentials,
+  BasicSessionCredentials,
+  DefaultAWSCredentialsProviderChain,
+  EC2ContainerCredentialsProviderWrapper,
+  EnvironmentVariableCredentialsProvider,
+  STSAssumeRoleSessionCredentialsProvider,
+  SystemPropertiesCredentialsProvider,
+  WebIdentityTokenCredentialsProvider
+}
+import com.amazonaws.auth.profile.{
+  ProfileCredentialsProvider,
+  ProfilesConfigFile
+}
+import org.embulk.config.{Config, ConfigDefault, ConfigException}
+import org.embulk.input.dynamodb.aws.AwsCredentials.Task
+import org.embulk.input.dynamodb.logger
+import org.embulk.spi.unit.LocalFile
+import zio.macros.annotation.delegate
+
+object AwsCredentials {
+
+  trait Task {
+
+    @Config("auth_method")
+    @ConfigDefault("\"default\"")
+    def getAuthMethod: String
+
+    @deprecated(message = "Use #getAccessKeyId() instead.", since = "0.3.0")
+    @Config("access_key")
+    @ConfigDefault("null")
+    def getAccessKey: Optional[String]
+
+    @Config("access_key_id")
+    @ConfigDefault("null")
+    def getAccessKeyId: Optional[String]
+
+    @deprecated(message = "Use #getSecretAccessKey() instead.", since = "0.3.0")
+    @Config("secret_key")
+    @ConfigDefault("null")
+    def getSecretKey: Optional[String]
+
+    @Config("secret_access_key")
+    @ConfigDefault("null")
+    def getSecretAccessKey: Optional[String]
+
+    @Config("session_token")
+    @ConfigDefault("null")
+    def getSessionToken: Optional[String]
+
+    @Config("profile_file")
+    @ConfigDefault("null")
+    def getProfileFile: Optional[LocalFile]
+
+    @Config("profile_name")
+    @ConfigDefault("\"default\"")
+    def getProfileName: String
+
+    @Config("role_arn")
+    @ConfigDefault("null")
+    def getRoleArn: Optional[String]
+
+    @Config("role_session_name")
+    @ConfigDefault("null")
+    def getRoleSessionName: Optional[String]
+
+    @Config("role_external_id")
+    @ConfigDefault("null")
+    def getRoleExternalId: Optional[String]
+
+    @Config("role_session_duration_seconds")
+    @ConfigDefault("null")
+    def getRoleSessionDurationSeconds: Optional[Int]
+
+    @Config("scope_down_policy")
+    @ConfigDefault("null")
+    def getScopeDownPolicy: Optional[String]
+
+    @Config("web_identity_token_file")
+    @ConfigDefault("null")
+    def getWebIdentityTokenFile: Optional[String]
+  }
+
+  def apply(task: Task): AwsCredentials = {
+    new AwsCredentials(AwsCredentialsTaskCompat(task))
+  }
+}
+
+case class AwsCredentialsTaskCompat(@delegate task: Task) extends Task {
+
+  override def getAccessKey: Optional[String] = {
+    throw new NotImplementedError()
+  }
+
+  override def getSecretKey: Optional[String] = {
+    throw new NotImplementedError()
+  }
+
+  override def getAuthMethod: String = {
+    if (getAccessKeyId.isPresent && getSecretAccessKey.isPresent) {
+      if (task.getAuthMethod != "basic") {
+        logger.warn(
+          "[Deprecated] The default value of \"auth_method\" option is \"default\", " +
+            "but currently use \"basic\" auth_method for backward compatibility " +
+            "because you set \"access_key_id\" and \"secret_access_key\" options. " +
+            "Please set \"basic\" to \"auth_method\" option expressly."
+        )
+        return "basic"
+      }
+    }
+    task.getAuthMethod
+  }
+
+  override def getAccessKeyId: Optional[String] = {
+    if (task.getAccessKeyId.isPresent && task.getAccessKey.isPresent)
+      throw new ConfigException(
+        "You cannot use both \"access_key_id\" option and \"access_key\" option. Use \"access_key_id\" option."
+      )
+    if (task.getAccessKey.isPresent) {
+      logger.warn(
+        "[Deprecated] \"access_key\" option is deprecated. Use \"access_key_id\" option instead."
+      )
+      return task.getAccessKey
+    }
+    task.getAccessKeyId
+  }
+
+  override def getSecretAccessKey: Optional[String] = {
+    if (task.getSecretAccessKey.isPresent && task.getSecretKey.isPresent)
+      throw new ConfigException(
+        "You cannot use both \"secret_access_key\" option and \"secret_key\" option. Use \"secret_access_key\" option."
+      )
+    if (task.getSecretKey.isPresent) {
+      logger.warn(
+        "[Deprecated] \"secret_key\" option is deprecated. Use \"secret_access_key\" option instead."
+      )
+      return task.getSecretKey
+    }
+    task.getSecretAccessKey
+  }
+}
+
+class AwsCredentials(task: Task) {
+
+  def createAwsCredentialsProvider: AWSCredentialsProvider = {
+    task.getAuthMethod match {
+      case "basic" =>
+        new AWSStaticCredentialsProvider(
+          new BasicAWSCredentials(
+            getRequiredOption(task.getAccessKeyId, "access_key_id"),
+            getRequiredOption(task.getSecretAccessKey, "secret_access_key")
+          )
+        )
+
+      case "env" =>
+        new EnvironmentVariableCredentialsProvider
+
+      case "instance" =>
+        // NOTE: combination of InstanceProfileCredentialsProvider and ContainerCredentialsProvider
+        new EC2ContainerCredentialsProviderWrapper
+
+      case "profile" =>
+        if (task.getProfileFile.isPresent) {
+          val pf: ProfilesConfigFile = new ProfilesConfigFile(
+            task.getProfileFile.get().getFile
+          )
+          new ProfileCredentialsProvider(pf, task.getProfileName)
+        }
+        else new ProfileCredentialsProvider(task.getProfileName)
+
+      case "properties" =>
+        new SystemPropertiesCredentialsProvider
+
+      case "anonymous" =>
+        new AWSStaticCredentialsProvider(new AnonymousAWSCredentials)
+
+      case "session" =>
+        new AWSStaticCredentialsProvider(
+          new BasicSessionCredentials(
+            getRequiredOption(task.getAccessKeyId, "access_key_id"),
+            getRequiredOption(task.getSecretAccessKey, "secret_access_key"),
+            getRequiredOption(task.getSessionToken, "session_token")
+          )
+        )
+
+      case "assume_role" =>
+        // NOTE: Are http_proxy, endpoint, region required when assuming role?
+        val builder = new STSAssumeRoleSessionCredentialsProvider.Builder(
+          getRequiredOption(task.getRoleArn, "role_arn"),
+          getRequiredOption(task.getRoleSessionName, "role_session_name")
+        )
+        task.getRoleExternalId.ifPresent(v => builder.withExternalId(v))
+        task.getRoleSessionDurationSeconds.ifPresent(v =>
+          builder.withRoleSessionDurationSeconds(v)
+        )
+        task.getScopeDownPolicy.ifPresent(v => builder.withScopeDownPolicy(v))
+
+        builder.build()
+
+      case "web_identity_token" =>
+        WebIdentityTokenCredentialsProvider
+          .builder()
+          .roleArn(getRequiredOption(task.getRoleArn, "role_arn"))
+          .roleSessionName(
+            getRequiredOption(task.getRoleSessionName, "role_session_name")
+          )
+          .webIdentityTokenFile(
+            getRequiredOption(
+              task.getWebIdentityTokenFile,
+              "web_identity_token_file"
+            )
+          )
+          .build()
+
+      case "default" =>
+        new DefaultAWSCredentialsProviderChain
+
+      case am =>
+        throw new ConfigException(
+          s"'$am' is unsupported: `auth_method` must be one of ['basic', 'env', 'instance', 'profile', 'properties', 'anonymous', 'session', 'assume_role', 'default']."
+        )
+    }
+  }
+
+  private def getRequiredOption[A](o: Optional[A], name: String): A = {
+    o.orElseThrow(() =>
+      new ConfigException(
+        s"`$name` must be set when `auth_method` is ${task.getAuthMethod}."
+      )
+    )
+  }
+
+}