embulk-output-s3_parquet 0.0.2 → 0.0.3

data/src/main/scala/org/embulk/output/s3_parquet/aws/AwsEndpointConfiguration.scala

@@ -11,39 +11,47 @@ import org.embulk.output.s3_parquet.aws.AwsEndpointConfiguration.Task
 
 import scala.util.Try
 
-object AwsEndpointConfiguration {
 
-  trait Task {
+object AwsEndpointConfiguration
+{
 
-    @Config("endpoint")
-    @ConfigDefault("null")
-    def getEndpoint: Optional[String]
+    trait Task
+    {
 
-    @Config("region")
-    @ConfigDefault("null")
-    def getRegion: Optional[String]
+        @Config("endpoint")
+        @ConfigDefault("null")
+        def getEndpoint: Optional[String]
 
-  }
+        @Config("region")
+        @ConfigDefault("null")
+        def getRegion: Optional[String]
 
-  def apply(task: Task): AwsEndpointConfiguration = new AwsEndpointConfiguration(task)
-}
-
-class AwsEndpointConfiguration(task: Task) {
-
-  def configureAwsClientBuilder[S <: AwsClientBuilder[S, T], T](builder: AwsClientBuilder[S, T]): Unit = {
-    if (task.getRegion.isPresent && task.getEndpoint.isPresent) {
-      val ec = new EndpointConfiguration(task.getEndpoint.get, task.getRegion.get)
-      builder.setEndpointConfiguration(ec)
     }
-    else if (task.getRegion.isPresent && !task.getEndpoint.isPresent) {
-      builder.setRegion(task.getRegion.get)
+
+    def apply(task: Task): AwsEndpointConfiguration =
+    {
+        new AwsEndpointConfiguration(task)
     }
-    else if (!task.getRegion.isPresent && task.getEndpoint.isPresent) {
-      val r: String = Try(new DefaultAwsRegionProviderChain().getRegion).getOrElse(Regions.DEFAULT_REGION.getName)
-      val e: String = task.getEndpoint.get
-      val ec = new EndpointConfiguration(e, r)
-      builder.setEndpointConfiguration(ec)
+}
+
+class AwsEndpointConfiguration(task: Task)
+{
+
+    def configureAwsClientBuilder[S <: AwsClientBuilder[S, T], T](builder: AwsClientBuilder[S, T]): Unit =
+    {
+        if (task.getRegion.isPresent && task.getEndpoint.isPresent) {
+            val ec = new EndpointConfiguration(task.getEndpoint.get, task.getRegion.get)
+            builder.setEndpointConfiguration(ec)
+        }
+        else if (task.getRegion.isPresent && !task.getEndpoint.isPresent) {
+            builder.setRegion(task.getRegion.get)
+        }
+        else if (!task.getRegion.isPresent && task.getEndpoint.isPresent) {
+            val r: String = Try(new DefaultAwsRegionProviderChain().getRegion).getOrElse(Regions.DEFAULT_REGION.getName)
+            val e: String = task.getEndpoint.get
+            val ec = new EndpointConfiguration(e, r)
+            builder.setEndpointConfiguration(ec)
+        }
     }
-  }
 
 }

data/src/main/scala/org/embulk/output/s3_parquet/aws/AwsS3Configuration.scala

@@ -7,50 +7,58 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder
 import org.embulk.config.{Config, ConfigDefault}
 import org.embulk.output.s3_parquet.aws.AwsS3Configuration.Task
 
+
 /*
  * These are advanced settings, so write no documentation.
  */
-object AwsS3Configuration {
-  trait Task {
+object AwsS3Configuration
+{
+    trait Task
+    {
 
-    @Config("accelerate_mode_enabled")
-    @ConfigDefault("null")
-    def getAccelerateModeEnabled: Optional[Boolean]
+        @Config("accelerate_mode_enabled")
+        @ConfigDefault("null")
+        def getAccelerateModeEnabled: Optional[Boolean]
 
-    @Config("chunked_encoding_disabled")
-    @ConfigDefault("null")
-    def getChunkedEncodingDisabled: Optional[Boolean]
+        @Config("chunked_encoding_disabled")
+        @ConfigDefault("null")
+        def getChunkedEncodingDisabled: Optional[Boolean]
 
-    @Config("dualstack_enabled")
-    @ConfigDefault("null")
-    def getDualstackEnabled: Optional[Boolean]
+        @Config("dualstack_enabled")
+        @ConfigDefault("null")
+        def getDualstackEnabled: Optional[Boolean]
 
-    @Config("force_global_bucket_access_enabled")
-    @ConfigDefault("null")
-    def getForceGlobalBucketAccessEnabled: Optional[Boolean]
+        @Config("force_global_bucket_access_enabled")
+        @ConfigDefault("null")
+        def getForceGlobalBucketAccessEnabled: Optional[Boolean]
 
-    @Config("path_style_access_enabled")
-    @ConfigDefault("null")
-    def getPathStyleAccessEnabled: Optional[Boolean]
+        @Config("path_style_access_enabled")
+        @ConfigDefault("null")
+        def getPathStyleAccessEnabled: Optional[Boolean]
 
-    @Config("payload_signing_enabled")
-    @ConfigDefault("null")
-    def getPayloadSigningEnabled: Optional[Boolean]
+        @Config("payload_signing_enabled")
+        @ConfigDefault("null")
+        def getPayloadSigningEnabled: Optional[Boolean]
 
-  }
+    }
 
-  def apply(task: Task): AwsS3Configuration = new AwsS3Configuration(task)
+    def apply(task: Task): AwsS3Configuration =
+    {
+        new AwsS3Configuration(task)
+    }
 }
 
-class AwsS3Configuration(task: Task) {
+class AwsS3Configuration(task: Task)
+{
 
-  def configureAmazonS3ClientBuilder(builder: AmazonS3ClientBuilder): Unit = {
-    task.getAccelerateModeEnabled.ifPresent(v => builder.setAccelerateModeEnabled(v))
-    task.getChunkedEncodingDisabled.ifPresent(v => builder.setChunkedEncodingDisabled(v))
-    task.getDualstackEnabled.ifPresent(v => builder.setDualstackEnabled(v))
-    task.getForceGlobalBucketAccessEnabled.ifPresent(v => builder.setForceGlobalBucketAccessEnabled(v))
-    task.getPathStyleAccessEnabled.ifPresent(v => builder.setPathStyleAccessEnabled(v))
-    task.getPayloadSigningEnabled.ifPresent(v => builder.setPayloadSigningEnabled(v))
-  }
+    def configureAmazonS3ClientBuilder(builder: AmazonS3ClientBuilder): Unit =
+    {
+        task.getAccelerateModeEnabled.ifPresent(v => builder.setAccelerateModeEnabled(v))
+        task.getChunkedEncodingDisabled.ifPresent(v => builder.setChunkedEncodingDisabled(v))
+        task.getDualstackEnabled.ifPresent(v => builder.setDualstackEnabled(v))
+        task.getForceGlobalBucketAccessEnabled.ifPresent(v => builder.setForceGlobalBucketAccessEnabled(v))
+        task.getPathStyleAccessEnabled.ifPresent(v => builder.setPathStyleAccessEnabled(v))
+        task.getPayloadSigningEnabled.ifPresent(v => builder.setPayloadSigningEnabled(v))
+    }
 
 }

data/src/main/scala/org/embulk/output/s3_parquet/aws/HttpProxy.scala

@@ -7,50 +7,58 @@ import com.amazonaws.{ClientConfiguration, Protocol}
 import org.embulk.config.{Config, ConfigDefault, ConfigException}
 import org.embulk.output.s3_parquet.aws.HttpProxy.Task
 
-object HttpProxy {
 
-  trait Task {
+object HttpProxy
+{
 
-    @Config("host")
-    @ConfigDefault("null")
-    def getHost: Optional[String]
+    trait Task
+    {
 
-    @Config("port")
-    @ConfigDefault("null")
-    def getPort: Optional[Int]
+        @Config("host")
+        @ConfigDefault("null")
+        def getHost: Optional[String]
 
-    @Config("protocol")
-    @ConfigDefault("\"https\"")
-    def getProtocol: String
+        @Config("port")
+        @ConfigDefault("null")
+        def getPort: Optional[Int]
 
-    @Config("user")
-    @ConfigDefault("null")
-    def getUser: Optional[String]
+        @Config("protocol")
+        @ConfigDefault("\"https\"")
+        def getProtocol: String
 
-    @Config("password")
-    @ConfigDefault("null")
-    def getPassword: Optional[String]
+        @Config("user")
+        @ConfigDefault("null")
+        def getUser: Optional[String]
 
-  }
+        @Config("password")
+        @ConfigDefault("null")
+        def getPassword: Optional[String]
 
-  def apply(task: Task): HttpProxy = new HttpProxy(task)
+    }
+
+    def apply(task: Task): HttpProxy =
+    {
+        new HttpProxy(task)
+    }
 
 }
 
-class HttpProxy(task: Task) {
+class HttpProxy(task: Task)
+{
 
-  def configureClientConfiguration(cc: ClientConfiguration): Unit = {
-    task.getHost.ifPresent(v => cc.setProxyHost(v))
-    task.getPort.ifPresent(v => cc.setProxyPort(v))
+    def configureClientConfiguration(cc: ClientConfiguration): Unit =
+    {
+        task.getHost.ifPresent(v => cc.setProxyHost(v))
+        task.getPort.ifPresent(v => cc.setProxyPort(v))
 
-    Protocol.values.find(p => p.name().equals(task.getProtocol)) match {
-      case Some(v) =>
-        cc.setProtocol(v)
-      case None =>
-        throw new ConfigException(s"'${task.getProtocol}' is unsupported: `protocol` must be one of [${Protocol.values.map(v => s"'$v'").mkString(", ")}].")
-    }
+        Protocol.values.find(p => p.name().equals(task.getProtocol)) match {
+            case Some(v) =>
+                cc.setProtocol(v)
+            case None    =>
+                throw new ConfigException(s"'${task.getProtocol}' is unsupported: `protocol` must be one of [${Protocol.values.map(v => s"'$v'").mkString(", ")}].")
+        }
 
-    task.getUser.ifPresent(v => cc.setProxyUsername(v))
-    task.getPassword.ifPresent(v => cc.setProxyPassword(v))
-  }
+        task.getUser.ifPresent(v => cc.setProxyUsername(v))
+        task.getPassword.ifPresent(v => cc.setProxyPassword(v))
+    }
 }

data/src/main/scala/org/embulk/output/s3_parquet/parquet/EmbulkMessageType.scala

@@ -6,54 +6,74 @@ import org.apache.parquet.schema.{MessageType, OriginalType, PrimitiveType, Type
 import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
 import org.embulk.spi.{Column, ColumnVisitor, Schema}
 
-object EmbulkMessageType {
 
-  def builder(): Builder = Builder()
-
-  case class Builder(name: String = "embulk",
-                     schema: Schema = Schema.builder().build()) {
-
-    def withName(name: String): Builder = Builder(name = name, schema = schema)
-
-    def withSchema(schema: Schema): Builder = Builder(name = name, schema = schema)
-
-    def build(): MessageType = {
-      val builder: ImmutableList.Builder[Type] = ImmutableList.builder[Type]()
-      schema.visitColumns(EmbulkMessageTypeColumnVisitor(builder))
-      new MessageType("embulk", builder.build())
+object EmbulkMessageType
+{
 
+    def builder(): Builder =
+    {
+        Builder()
     }
 
-  }
+    case class Builder(name: String = "embulk",
+                       schema: Schema = Schema.builder().build())
+    {
 
-  private case class EmbulkMessageTypeColumnVisitor(builder: ImmutableList.Builder[Type])
-    extends ColumnVisitor {
+        def withName(name: String): Builder =
+        {
+            Builder(name = name, schema = schema)
+        }
 
-    override def booleanColumn(column: Column): Unit = {
-      builder.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BOOLEAN, column.getName))
-    }
+        def withSchema(schema: Schema): Builder =
+        {
+            Builder(name = name, schema = schema)
+        }
 
-    override def longColumn(column: Column): Unit = {
-      builder.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.INT64, column.getName))
-    }
-
-    override def doubleColumn(column: Column): Unit = {
-      builder.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.DOUBLE, column.getName))
-    }
+        def build(): MessageType =
+        {
+            val builder: ImmutableList.Builder[Type] = ImmutableList.builder[Type]()
+            schema.visitColumns(EmbulkMessageTypeColumnVisitor(builder))
+            new MessageType("embulk", builder.build())
 
-    override def stringColumn(column: Column): Unit = {
-      builder.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BINARY, column.getName, OriginalType.UTF8))
-    }
+        }
 
-    override def timestampColumn(column: Column): Unit = {
-      // TODO: Support OriginalType.TIME* ?
-      builder.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BINARY, column.getName, OriginalType.UTF8))
     }
 
-    override def jsonColumn(column: Column): Unit = {
-      // TODO: does this work?
-      builder.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BINARY, column.getName, OriginalType.UTF8))
+    private case class EmbulkMessageTypeColumnVisitor(builder: ImmutableList.Builder[Type])
+        extends ColumnVisitor
+    {
+
+        override def booleanColumn(column: Column): Unit =
+        {
+            builder.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BOOLEAN, column.getName))
+        }
+
+        override def longColumn(column: Column): Unit =
+        {
+            builder.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.INT64, column.getName))
+        }
+
+        override def doubleColumn(column: Column): Unit =
+        {
+            builder.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.DOUBLE, column.getName))
+        }
+
+        override def stringColumn(column: Column): Unit =
+        {
+            builder.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BINARY, column.getName, OriginalType.UTF8))
+        }
+
+        override def timestampColumn(column: Column): Unit =
+        {
+            // TODO: Support OriginalType.TIME* ?
+            builder.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BINARY, column.getName, OriginalType.UTF8))
+        }
+
+        override def jsonColumn(column: Column): Unit =
+        {
+            // TODO: does this work?
+            builder.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BINARY, column.getName, OriginalType.UTF8))
+        }
     }
-  }
 
 }

data/src/main/scala/org/embulk/output/s3_parquet/parquet/ParquetFileWriteSupport.scala

@@ -9,25 +9,32 @@ import org.apache.parquet.schema.MessageType
 import org.embulk.spi.{PageReader, Schema}
 import org.embulk.spi.time.TimestampFormatter
 
-import scala.collection.JavaConverters._
+import scala.jdk.CollectionConverters._
 
-private[parquet] case class ParquetFileWriteSupport(schema: Schema,
-                              timestampFormatters: Seq[TimestampFormatter])
-  extends WriteSupport[PageReader] {
-
-  private var currentParquetFileWriter: ParquetFileWriter = _
-
-  override def init(configuration: Configuration): WriteContext = {
-    val messageType: MessageType = EmbulkMessageType.builder()
-      .withSchema(schema)
-      .build()
-    val metadata: Map[String, String] = Map.empty // NOTE: When is this used?
-    new WriteContext(messageType, metadata.asJava)
-  }
 
-  override def prepareForWrite(recordConsumer: RecordConsumer): Unit = {
-    currentParquetFileWriter = ParquetFileWriter(recordConsumer, schema, timestampFormatters)
-  }
-
-  override def write(record: PageReader): Unit = currentParquetFileWriter.write(record)
+private[parquet] case class ParquetFileWriteSupport(schema: Schema,
+                                                    timestampFormatters: Seq[TimestampFormatter])
+    extends WriteSupport[PageReader]
+{
+
+    private var currentParquetFileWriter: ParquetFileWriter = _
+
+    override def init(configuration: Configuration): WriteContext =
+    {
+        val messageType: MessageType = EmbulkMessageType.builder()
+            .withSchema(schema)
+            .build()
+        val metadata: Map[String, String] = Map.empty // NOTE: When is this used?
+        new WriteContext(messageType, metadata.asJava)
+    }
+
+    override def prepareForWrite(recordConsumer: RecordConsumer): Unit =
+    {
+        currentParquetFileWriter = ParquetFileWriter(recordConsumer, schema, timestampFormatters)
+    }
+
+    override def write(record: PageReader): Unit =
+    {
+        currentParquetFileWriter.write(record)
+    }
 }

data/src/main/scala/org/embulk/output/s3_parquet/parquet/ParquetFileWriter.scala

@@ -9,117 +9,151 @@ import org.apache.parquet.io.api.{Binary, RecordConsumer}
 import org.embulk.spi.{Column, ColumnVisitor, PageReader, Schema}
 import org.embulk.spi.time.TimestampFormatter
 
-object ParquetFileWriter {
 
-  case class Builder(path: Path = null,
-                     schema: Schema = null,
-                     timestampFormatters: Seq[TimestampFormatter] = null)
-    extends ParquetWriter.Builder[PageReader, Builder](path) {
+object ParquetFileWriter
+{
 
-    def withPath(path: Path): Builder = copy(path = path)
+    case class Builder(path: Path = null,
+                       schema: Schema = null,
+                       timestampFormatters: Seq[TimestampFormatter] = null)
+        extends ParquetWriter.Builder[PageReader, Builder](path)
+    {
 
-    def withPath(pathString: String): Builder = copy(path = new Path(pathString))
+      def withPath(path: Path): Builder =
+      {
+        copy(path = path)
+      }
 
-    def withSchema(schema: Schema): Builder = copy(schema = schema)
+      def withPath(pathString: String): Builder =
+      {
+        copy(path = new Path(pathString))
+      }
 
-    def withTimestampFormatters(timestampFormatters: Seq[TimestampFormatter]): Builder = copy(timestampFormatters = timestampFormatters)
+      def withSchema(schema: Schema): Builder =
+      {
+        copy(schema = schema)
+      }
 
-    override def self(): Builder = this
+      def withTimestampFormatters(timestampFormatters: Seq[TimestampFormatter]): Builder =
+      {
+        copy(timestampFormatters = timestampFormatters)
+      }
 
-    override def getWriteSupport(conf: Configuration): WriteSupport[PageReader] = {
-      ParquetFileWriteSupport(schema, timestampFormatters)
+      override def self(): Builder =
+      {
+        this
+      }
+
+        override def getWriteSupport(conf: Configuration): WriteSupport[PageReader] =
+        {
+            ParquetFileWriteSupport(schema, timestampFormatters)
+        }
     }
-  }
 
-  def builder(): Builder = Builder()
+  def builder(): Builder =
+  {
+    Builder()
+  }
 
 }
 
 
 private[parquet] case class ParquetFileWriter(recordConsumer: RecordConsumer,
-                             schema: Schema,
-                             timestampFormatters: Seq[TimestampFormatter]) {
-
-  def write(record: PageReader): Unit = {
-    recordConsumer.startMessage()
-    writeRecord(record)
-    recordConsumer.endMessage()
-  }
-
-  private def writeRecord(record: PageReader): Unit = {
-
-    schema.visitColumns(new ColumnVisitor() {
-
-      override def booleanColumn(column: Column): Unit = {
-        nullOr(column, {
-          withWriteFieldContext(column, {
-            recordConsumer.addBoolean(record.getBoolean(column))
-          })
-        })
-      }
-
-      override def longColumn(column: Column): Unit = {
-        nullOr(column, {
-          withWriteFieldContext(column, {
-            recordConsumer.addLong(record.getLong(column))
-          })
-        })
-      }
+                                              schema: Schema,
+                                              timestampFormatters: Seq[TimestampFormatter])
+{
+
+    def write(record: PageReader): Unit =
+    {
+        recordConsumer.startMessage()
+        writeRecord(record)
+        recordConsumer.endMessage()
+    }
 
-      override def doubleColumn(column: Column): Unit = {
-        nullOr(column, {
-          withWriteFieldContext(column, {
-            recordConsumer.addDouble(record.getDouble(column))
-          })
-        })
-      }
+    private def writeRecord(record: PageReader): Unit =
+    {
+
+        schema.visitColumns(new ColumnVisitor()
+        {
+
+            override def booleanColumn(column: Column): Unit =
+            {
+                nullOr(column, {
+                    withWriteFieldContext(column, {
+                        recordConsumer.addBoolean(record.getBoolean(column))
+                    })
+                })
+            }
+
+            override def longColumn(column: Column): Unit =
+            {
+                nullOr(column, {
+                    withWriteFieldContext(column, {
+                        recordConsumer.addLong(record.getLong(column))
+                    })
+                })
+            }
+
+            override def doubleColumn(column: Column): Unit =
+            {
+                nullOr(column, {
+                    withWriteFieldContext(column, {
+                        recordConsumer.addDouble(record.getDouble(column))
+                    })
+                })
+            }
+
+            override def stringColumn(column: Column): Unit =
+            {
+                nullOr(column, {
+                    withWriteFieldContext(column, {
+                        val bin = Binary.fromString(record.getString(column))
+                        recordConsumer.addBinary(bin)
+                    })
+                })
+            }
+
+            override def timestampColumn(column: Column): Unit =
+            {
+                nullOr(column, {
+                    withWriteFieldContext(column, {
+                        // TODO: is a correct way to convert for parquet ?
+                        val t = record.getTimestamp(column)
+                        val ft = timestampFormatters(column.getIndex).format(t)
+                        val bin = Binary.fromString(ft)
+                        recordConsumer.addBinary(bin)
+                    })
+                })
+            }
+
+            override def jsonColumn(column: Column): Unit =
+            {
+                nullOr(column, {
+                    withWriteFieldContext(column, {
+                        // TODO: is a correct way to convert for parquet ?
+                        val msgPack = record.getJson(column)
+                        val bin = Binary.fromString(msgPack.toJson)
+                        recordConsumer.addBinary(bin)
+                    })
+                })
+            }
+
+            private def nullOr(column: Column,
+                               f: => Unit): Unit =
+            {
+                if (!record.isNull(column)) f
+            }
+
+            private def withWriteFieldContext(column: Column,
+                                              f: => Unit): Unit =
+            {
+                recordConsumer.startField(column.getName, column.getIndex)
+                f
+                recordConsumer.endField(column.getName, column.getIndex)
+            }
 
-      override def stringColumn(column: Column): Unit = {
-        nullOr(column, {
-          withWriteFieldContext(column, {
-            val bin = Binary.fromString(record.getString(column))
-            recordConsumer.addBinary(bin)
-          })
         })
-      }
 
-      override def timestampColumn(column: Column): Unit = {
-        nullOr(column, {
-          withWriteFieldContext(column, {
-            // TODO: is a correct way to convert for parquet ?
-            val t = record.getTimestamp(column)
-            val ft = timestampFormatters(column.getIndex).format(t)
-            val bin = Binary.fromString(ft)
-            recordConsumer.addBinary(bin)
-          })
-        })
-      }
-
-      override def jsonColumn(column: Column): Unit = {
-        nullOr(column, {
-          withWriteFieldContext(column, {
-            // TODO: is a correct way to convert for parquet ?
-            val msgPack = record.getJson(column)
-            val bin = Binary.fromString(msgPack.toJson)
-            recordConsumer.addBinary(bin)
-          })
-        })
-      }
-
-      private def nullOr(column: Column,
-                         f: => Unit): Unit = {
-        if (!record.isNull(column)) f
-      }
-
-      private def withWriteFieldContext(column: Column,
-                                        f: => Unit): Unit = {
-        recordConsumer.startField(column.getName, column.getIndex)
-        f
-        recordConsumer.endField(column.getName, column.getIndex)
-      }
-
-    })
-
-  }
+    }
 
 }