embulk-output-s3_parquet 0.1.0 → 0.2.0

data/src/main/scala/org/embulk/output/s3_parquet/aws/AwsEndpointConfiguration.scala

@@ -1,6 +1,5 @@
 package org.embulk.output.s3_parquet.aws
 
-
 import java.util.Optional
 
 import com.amazonaws.client.builder.AwsClientBuilder
@@ -11,47 +10,45 @@ import org.embulk.output.s3_parquet.aws.AwsEndpointConfiguration.Task
 
 import scala.util.Try
 
+object AwsEndpointConfiguration {
 
-object AwsEndpointConfiguration
-{
-
-    trait Task
-    {
+  trait Task {
 
-        @Config("endpoint")
-        @ConfigDefault("null")
-        def getEndpoint: Optional[String]
+    @Config("endpoint")
+    @ConfigDefault("null")
+    def getEndpoint: Optional[String]
 
-        @Config("region")
-        @ConfigDefault("null")
-        def getRegion: Optional[String]
+    @Config("region")
+    @ConfigDefault("null")
+    def getRegion: Optional[String]
 
-    }
+  }
 
-    def apply(task: Task): AwsEndpointConfiguration =
-    {
-        new AwsEndpointConfiguration(task)
-    }
+  def apply(task: Task): AwsEndpointConfiguration = {
+    new AwsEndpointConfiguration(task)
+  }
 }
 
-class AwsEndpointConfiguration(task: Task)
-{
-
-    def configureAwsClientBuilder[S <: AwsClientBuilder[S, T], T](builder: AwsClientBuilder[S, T]): Unit =
-    {
-        if (task.getRegion.isPresent && task.getEndpoint.isPresent) {
-            val ec = new EndpointConfiguration(task.getEndpoint.get, task.getRegion.get)
-            builder.setEndpointConfiguration(ec)
-        }
-        else if (task.getRegion.isPresent && !task.getEndpoint.isPresent) {
-            builder.setRegion(task.getRegion.get)
-        }
-        else if (!task.getRegion.isPresent && task.getEndpoint.isPresent) {
-            val r: String = Try(new DefaultAwsRegionProviderChain().getRegion).getOrElse(Regions.DEFAULT_REGION.getName)
-            val e: String = task.getEndpoint.get
-            val ec = new EndpointConfiguration(e, r)
-            builder.setEndpointConfiguration(ec)
-        }
+class AwsEndpointConfiguration(task: Task) {
+
+  def configureAwsClientBuilder[S <: AwsClientBuilder[S, T], T](
+      builder: AwsClientBuilder[S, T]
+  ): Unit = {
+    if (task.getRegion.isPresent && task.getEndpoint.isPresent) {
+      val ec =
+        new EndpointConfiguration(task.getEndpoint.get, task.getRegion.get)
+      builder.setEndpointConfiguration(ec)
+    }
+    else if (task.getRegion.isPresent && !task.getEndpoint.isPresent) {
+      builder.setRegion(task.getRegion.get)
+    }
+    else if (!task.getRegion.isPresent && task.getEndpoint.isPresent) {
+      val r: String = Try(new DefaultAwsRegionProviderChain().getRegion)
+        .getOrElse(Regions.DEFAULT_REGION.getName)
+      val e: String = task.getEndpoint.get
+      val ec = new EndpointConfiguration(e, r)
+      builder.setEndpointConfiguration(ec)
     }
+  }
 
 }

data/src/main/scala/org/embulk/output/s3_parquet/aws/AwsS3Configuration.scala

@@ -1,64 +1,68 @@
 package org.embulk.output.s3_parquet.aws
 
-
 import java.util.Optional
 
 import com.amazonaws.services.s3.AmazonS3ClientBuilder
 import org.embulk.config.{Config, ConfigDefault}
 import org.embulk.output.s3_parquet.aws.AwsS3Configuration.Task
 
-
 /*
  * These are advanced settings, so write no documentation.
  */
-object AwsS3Configuration
-{
-    trait Task
-    {
+object AwsS3Configuration {
 
-        @Config("accelerate_mode_enabled")
-        @ConfigDefault("null")
-        def getAccelerateModeEnabled: Optional[Boolean]
+  trait Task {
 
-        @Config("chunked_encoding_disabled")
-        @ConfigDefault("null")
-        def getChunkedEncodingDisabled: Optional[Boolean]
+    @Config("accelerate_mode_enabled")
+    @ConfigDefault("null")
+    def getAccelerateModeEnabled: Optional[Boolean]
 
-        @Config("dualstack_enabled")
-        @ConfigDefault("null")
-        def getDualstackEnabled: Optional[Boolean]
+    @Config("chunked_encoding_disabled")
+    @ConfigDefault("null")
+    def getChunkedEncodingDisabled: Optional[Boolean]
 
-        @Config("force_global_bucket_access_enabled")
-        @ConfigDefault("null")
-        def getForceGlobalBucketAccessEnabled: Optional[Boolean]
+    @Config("dualstack_enabled")
+    @ConfigDefault("null")
+    def getDualstackEnabled: Optional[Boolean]
 
-        @Config("path_style_access_enabled")
-        @ConfigDefault("null")
-        def getPathStyleAccessEnabled: Optional[Boolean]
+    @Config("force_global_bucket_access_enabled")
+    @ConfigDefault("null")
+    def getForceGlobalBucketAccessEnabled: Optional[Boolean]
 
-        @Config("payload_signing_enabled")
-        @ConfigDefault("null")
-        def getPayloadSigningEnabled: Optional[Boolean]
+    @Config("path_style_access_enabled")
+    @ConfigDefault("null")
+    def getPathStyleAccessEnabled: Optional[Boolean]
 
-    }
+    @Config("payload_signing_enabled")
+    @ConfigDefault("null")
+    def getPayloadSigningEnabled: Optional[Boolean]
 
-    def apply(task: Task): AwsS3Configuration =
-    {
-        new AwsS3Configuration(task)
-    }
-}
+  }
 
-class AwsS3Configuration(task: Task)
-{
+  def apply(task: Task): AwsS3Configuration = {
+    new AwsS3Configuration(task)
+  }
+}
 
-    def configureAmazonS3ClientBuilder(builder: AmazonS3ClientBuilder): Unit =
-    {
-        task.getAccelerateModeEnabled.ifPresent(v => builder.setAccelerateModeEnabled(v))
-        task.getChunkedEncodingDisabled.ifPresent(v => builder.setChunkedEncodingDisabled(v))
-        task.getDualstackEnabled.ifPresent(v => builder.setDualstackEnabled(v))
-        task.getForceGlobalBucketAccessEnabled.ifPresent(v => builder.setForceGlobalBucketAccessEnabled(v))
-        task.getPathStyleAccessEnabled.ifPresent(v => builder.setPathStyleAccessEnabled(v))
-        task.getPayloadSigningEnabled.ifPresent(v => builder.setPayloadSigningEnabled(v))
-    }
+class AwsS3Configuration(task: Task) {
+
+  def configureAmazonS3ClientBuilder(builder: AmazonS3ClientBuilder): Unit = {
+    task.getAccelerateModeEnabled.ifPresent(v =>
+      builder.setAccelerateModeEnabled(v)
+    )
+    task.getChunkedEncodingDisabled.ifPresent(v =>
+      builder.setChunkedEncodingDisabled(v)
+    )
+    task.getDualstackEnabled.ifPresent(v => builder.setDualstackEnabled(v))
+    task.getForceGlobalBucketAccessEnabled.ifPresent(v =>
+      builder.setForceGlobalBucketAccessEnabled(v)
+    )
+    task.getPathStyleAccessEnabled.ifPresent(v =>
+      builder.setPathStyleAccessEnabled(v)
+    )
+    task.getPayloadSigningEnabled.ifPresent(v =>
+      builder.setPayloadSigningEnabled(v)
+    )
+  }
 
 }

data/src/main/scala/org/embulk/output/s3_parquet/aws/HttpProxy.scala

@@ -1,64 +1,61 @@
 package org.embulk.output.s3_parquet.aws
 
-
 import java.util.Optional
 
 import com.amazonaws.{ClientConfiguration, Protocol}
 import org.embulk.config.{Config, ConfigDefault, ConfigException}
 import org.embulk.output.s3_parquet.aws.HttpProxy.Task
 
+object HttpProxy {
 
-object HttpProxy
-{
-
-    trait Task
-    {
+  trait Task {
 
-        @Config("host")
-        @ConfigDefault("null")
-        def getHost: Optional[String]
+    @Config("host")
+    @ConfigDefault("null")
+    def getHost: Optional[String]
 
-        @Config("port")
-        @ConfigDefault("null")
-        def getPort: Optional[Int]
+    @Config("port")
+    @ConfigDefault("null")
+    def getPort: Optional[Int]
 
-        @Config("protocol")
-        @ConfigDefault("\"https\"")
-        def getProtocol: String
+    @Config("protocol")
+    @ConfigDefault("\"https\"")
+    def getProtocol: String
 
-        @Config("user")
-        @ConfigDefault("null")
-        def getUser: Optional[String]
+    @Config("user")
+    @ConfigDefault("null")
+    def getUser: Optional[String]
 
-        @Config("password")
-        @ConfigDefault("null")
-        def getPassword: Optional[String]
+    @Config("password")
+    @ConfigDefault("null")
+    def getPassword: Optional[String]
 
-    }
+  }
 
-    def apply(task: Task): HttpProxy =
-    {
-        new HttpProxy(task)
-    }
+  def apply(task: Task): HttpProxy = {
+    new HttpProxy(task)
+  }
 
 }
 
-class HttpProxy(task: Task)
-{
-
-    def configureClientConfiguration(cc: ClientConfiguration): Unit =
-    {
-        task.getHost.ifPresent(v => cc.setProxyHost(v))
-        task.getPort.ifPresent(v => cc.setProxyPort(v))
-
-        Protocol.values.find(p => p.name().equals(task.getProtocol)) match {
-            case Some(v) =>
-                cc.setProtocol(v)
-            case None    =>
-                throw new ConfigException(s"'${task.getProtocol}' is unsupported: `protocol` must be one of [${Protocol.values.map(v => s"'$v'").mkString(", ")}].")
-        }
-
-        task.getUser.ifPresent(v => cc.setProxyUsername(v))
-        task.getPassword.ifPresent(v => cc.setProxyPassword(v))
+class HttpProxy(task: Task) {
+
+  def configureClientConfiguration(cc: ClientConfiguration): Unit = {
+    task.getHost.ifPresent(v => cc.setProxyHost(v))
+    task.getPort.ifPresent(v => cc.setProxyPort(v))
+
+    Protocol.values.find(p => p.name().equals(task.getProtocol)) match {
+      case Some(v) =>
+        cc.setProtocol(v)
+      case None =>
+        throw new ConfigException(
+          s"'${task.getProtocol}' is unsupported: `protocol` must be one of [${Protocol.values
+            .map(v => s"'$v'")
+            .mkString(", ")}]."
+        )
     }
+
+    task.getUser.ifPresent(v => cc.setProxyUsername(v))
+    task.getPassword.ifPresent(v => cc.setProxyPassword(v))
+  }
 }

data/src/main/scala/org/embulk/output/s3_parquet/parquet/EmbulkMessageType.scala

@@ -1,107 +1,153 @@
 package org.embulk.output.s3_parquet.parquet
 
-
 import com.google.common.collect.ImmutableList
-import org.apache.parquet.schema.{MessageType, OriginalType, PrimitiveType, Type}
+import org.apache.parquet.schema.{
+  MessageType,
+  OriginalType,
+  PrimitiveType,
+  Type
+}
 import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
 import org.embulk.spi.{Column, ColumnVisitor, Schema}
 
+object EmbulkMessageType {
+
+  def builder(): Builder = {
+    Builder()
+  }
+
+  case class Builder(
+      name: String = "embulk",
+      schema: Schema = Schema.builder().build(),
+      logicalTypeHandlers: LogicalTypeHandlerStore =
+        LogicalTypeHandlerStore.empty
+  ) {
+
+    def withName(name: String): Builder = {
+      Builder(
+        name = name,
+        schema = schema,
+        logicalTypeHandlers = logicalTypeHandlers
+      )
+    }
+
+    def withSchema(schema: Schema): Builder = {
+      Builder(
+        name = name,
+        schema = schema,
+        logicalTypeHandlers = logicalTypeHandlers
+      )
+    }
+
+    def withLogicalTypeHandlers(
+        logicalTypeHandlers: LogicalTypeHandlerStore
+    ): Builder = {
+      Builder(
+        name = name,
+        schema = schema,
+        logicalTypeHandlers = logicalTypeHandlers
+      )
+    }
+
+    def build(): MessageType = {
+      val builder: ImmutableList.Builder[Type] = ImmutableList.builder[Type]()
+      schema.visitColumns(
+        EmbulkMessageTypeColumnVisitor(builder, logicalTypeHandlers)
+      )
+      new MessageType("embulk", builder.build())
+    }
+
+  }
+
+  private case class EmbulkMessageTypeColumnVisitor(
+      builder: ImmutableList.Builder[Type],
+      logicalTypeHandlers: LogicalTypeHandlerStore =
+        LogicalTypeHandlerStore.empty
+  ) extends ColumnVisitor {
+
+    override def booleanColumn(column: Column): Unit = {
+      builder.add(
+        new PrimitiveType(
+          Type.Repetition.OPTIONAL,
+          PrimitiveTypeName.BOOLEAN,
+          column.getName
+        )
+      )
+    }
 
-object EmbulkMessageType
-{
+    override def longColumn(column: Column): Unit = {
+      val name = column.getName
+      val et = column.getType
+
+      val t = logicalTypeHandlers.get(name, et) match {
+        case Some(h) if h.isConvertible(et) => h.newSchemaFieldType(name)
+        case _ =>
+          new PrimitiveType(
+            Type.Repetition.OPTIONAL,
+            PrimitiveTypeName.INT64,
+            column.getName
+          )
+      }
+
+      builder.add(t)
+    }
 
-    def builder(): Builder =
-    {
-        Builder()
+    override def doubleColumn(column: Column): Unit = {
+      builder.add(
+        new PrimitiveType(
+          Type.Repetition.OPTIONAL,
+          PrimitiveTypeName.DOUBLE,
+          column.getName
+        )
+      )
     }
 
-    case class Builder(name: String = "embulk",
-                       schema: Schema = Schema.builder().build(),
-                       logicalTypeHandlers: LogicalTypeHandlerStore = LogicalTypeHandlerStore.empty)
-    {
-
-        def withName(name: String): Builder =
-        {
-            Builder(name = name, schema = schema, logicalTypeHandlers = logicalTypeHandlers)
-        }
-
-        def withSchema(schema: Schema): Builder =
-        {
-            Builder(name = name, schema = schema, logicalTypeHandlers = logicalTypeHandlers)
-        }
-
-        def withLogicalTypeHandlers(logicalTypeHandlers: LogicalTypeHandlerStore): Builder =
-        {
-            Builder(name = name, schema = schema, logicalTypeHandlers = logicalTypeHandlers)
-        }
-
-        def build(): MessageType =
-        {
-            val builder: ImmutableList.Builder[Type] = ImmutableList.builder[Type]()
-            schema.visitColumns(EmbulkMessageTypeColumnVisitor(builder, logicalTypeHandlers))
-            new MessageType("embulk", builder.build())
-        }
+    override def stringColumn(column: Column): Unit = {
+      builder.add(
+        new PrimitiveType(
+          Type.Repetition.OPTIONAL,
+          PrimitiveTypeName.BINARY,
+          column.getName,
+          OriginalType.UTF8
+        )
+      )
+    }
 
+    override def timestampColumn(column: Column): Unit = {
+      val name = column.getName
+      val et = column.getType
+
+      val t = logicalTypeHandlers.get(name, et) match {
+        case Some(h) if h.isConvertible(et) => h.newSchemaFieldType(name)
+        case _ =>
+          new PrimitiveType(
+            Type.Repetition.OPTIONAL,
+            PrimitiveTypeName.BINARY,
+            name,
+            OriginalType.UTF8
+          )
+      }
+
+      builder.add(t)
     }
 
-    private case class EmbulkMessageTypeColumnVisitor(builder: ImmutableList.Builder[Type],
-                                                      logicalTypeHandlers: LogicalTypeHandlerStore = LogicalTypeHandlerStore.empty)
-        extends ColumnVisitor
-    {
-
-        override def booleanColumn(column: Column): Unit =
-        {
-            builder.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BOOLEAN, column.getName))
-        }
-
-        override def longColumn(column: Column): Unit =
-        {
-            val name = column.getName
-            val et = column.getType
-
-            val t = logicalTypeHandlers.get(name, et) match {
-                case Some(h) if h.isConvertible(et) => h.newSchemaFieldType(name)
-                case _                              => new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.INT64, column.getName)
-            }
-
-            builder.add(t)
-        }
-
-        override def doubleColumn(column: Column): Unit =
-        {
-            builder.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.DOUBLE, column.getName))
-        }
-
-        override def stringColumn(column: Column): Unit =
-        {
-            builder.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BINARY, column.getName, OriginalType.UTF8))
-        }
-
-        override def timestampColumn(column: Column): Unit =
-        {
-            val name = column.getName
-            val et = column.getType
-
-            val t = logicalTypeHandlers.get(name, et) match {
-                case Some(h) if h.isConvertible(et) => h.newSchemaFieldType(name)
-                case _                              => new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BINARY, name, OriginalType.UTF8)
-            }
-
-            builder.add(t)
-        }
-
-        override def jsonColumn(column: Column): Unit =
-        {
-            val name = column.getName
-            val et = column.getType
-
-            val t = logicalTypeHandlers.get(name, et) match {
-                case Some(h) if h.isConvertible(et) => h.newSchemaFieldType(name)
-                case _                              => new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BINARY, name, OriginalType.UTF8)
-            }
-
-            builder.add(t)
-        }
+    override def jsonColumn(column: Column): Unit = {
+      val name = column.getName
+      val et = column.getType
+
+      val t = logicalTypeHandlers.get(name, et) match {
+        case Some(h) if h.isConvertible(et) => h.newSchemaFieldType(name)
+        case _ =>
+          new PrimitiveType(
+            Type.Repetition.OPTIONAL,
+            PrimitiveTypeName.BINARY,
+            name,
+            OriginalType.UTF8
+          )
+      }
+
+      builder.add(t)
     }
+  }
 
-}
+}