embulk-output-s3_parquet 0.1.0 → 0.2.0

data/src/main/scala/org/embulk/output/s3_parquet/parquet/LogicalTypeHandler.scala

@@ -1,6 +1,5 @@
 package org.embulk.output.s3_parquet.parquet
 
-
 import org.apache.parquet.io.api.{Binary, RecordConsumer}
 import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
 import org.apache.parquet.schema.{Type => PType}
@@ -11,135 +10,151 @@ import org.embulk.spi.`type`.Types
 import org.embulk.spi.time.Timestamp
 import org.msgpack.value.Value
 
-
 /**
- * Handle Apache Parquet 'Logical Types' on schema/value conversion.
- * ref. https://github.com/apache/parquet-format/blob/master/LogicalTypes.md
- *
- * It focuses on only older representation because newer supported since 1.11 is not used actually yet.
- * TODO Support both of older and newer representation after 1.11+ is published and other middleware supports it.
- *
- */
-sealed trait LogicalTypeHandler
-{
-    def isConvertible(t: EType): Boolean
-
-    def newSchemaFieldType(name: String): PrimitiveType
-
-    def consume(orig: Any,
-                recordConsumer: RecordConsumer): Unit
+  * Handle Apache Parquet 'Logical Types' on schema/value conversion.
+  * ref. https://github.com/apache/parquet-format/blob/master/LogicalTypes.md
+  *
+  * It focuses on only older representation because newer supported since 1.11 is not used actually yet.
+  * TODO Support both of older and newer representation after 1.11+ is published and other middleware supports it.
+  *
+  */
+sealed trait LogicalTypeHandler {
+  def isConvertible(t: EType): Boolean
+
+  def newSchemaFieldType(name: String): PrimitiveType
+
+  def consume(orig: Any, recordConsumer: RecordConsumer): Unit
 }
 
 abstract class IntLogicalTypeHandler(ot: OriginalType)
-    extends LogicalTypeHandler
-{
-    override def isConvertible(t: EType): Boolean =
-    {
-        t == Types.LONG
-    }
-
-    override def newSchemaFieldType(name: String): PrimitiveType =
-    {
-        new PrimitiveType(PType.Repetition.OPTIONAL, PrimitiveTypeName.INT64, name, ot)
-    }
-
-    override def consume(orig: Any,
-                         recordConsumer: RecordConsumer): Unit =
-    {
-        orig match {
-            case v: Long => recordConsumer.addLong(v)
-            case _       => throw new DataException("given mismatched type value; expected type is long")
-        }
+    extends LogicalTypeHandler {
+
+  override def isConvertible(t: EType): Boolean = {
+    t == Types.LONG
+  }
+
+  override def newSchemaFieldType(name: String): PrimitiveType = {
+    new PrimitiveType(
+      PType.Repetition.OPTIONAL,
+      PrimitiveTypeName.INT64,
+      name,
+      ot
+    )
+  }
+
+  override def consume(orig: Any, recordConsumer: RecordConsumer): Unit = {
+    orig match {
+      case v: Long => recordConsumer.addLong(v)
+      case _ =>
+        throw new DataException(
+          "given mismatched type value; expected type is long"
+        )
     }
+  }
 }
 
-object TimestampMillisLogicalTypeHandler
-    extends LogicalTypeHandler
-{
-    override def isConvertible(t: EType): Boolean =
-    {
-        t == Types.TIMESTAMP
-    }
-
-    override def newSchemaFieldType(name: String): PrimitiveType =
-    {
-        new PrimitiveType(PType.Repetition.OPTIONAL, PrimitiveTypeName.INT64, name, OriginalType.TIMESTAMP_MILLIS)
-    }
-
-    override def consume(orig: Any,
-                         recordConsumer: RecordConsumer): Unit =
-    {
-        orig match {
-            case ts: Timestamp => recordConsumer.addLong(ts.toEpochMilli)
-            case _             => throw new DataException("given mismatched type value; expected type is timestamp")
-        }
+object TimestampMillisLogicalTypeHandler extends LogicalTypeHandler {
+
+  override def isConvertible(t: EType): Boolean = {
+    t == Types.TIMESTAMP
+  }
+
+  override def newSchemaFieldType(name: String): PrimitiveType = {
+    new PrimitiveType(
+      PType.Repetition.OPTIONAL,
+      PrimitiveTypeName.INT64,
+      name,
+      OriginalType.TIMESTAMP_MILLIS
+    )
+  }
+
+  override def consume(orig: Any, recordConsumer: RecordConsumer): Unit = {
+    orig match {
+      case ts: Timestamp => recordConsumer.addLong(ts.toEpochMilli)
+      case _ =>
+        throw new DataException(
+          "given mismatched type value; expected type is timestamp"
+        )
     }
+  }
 }
 
-object TimestampMicrosLogicalTypeHandler
-    extends LogicalTypeHandler
-{
-    override def isConvertible(t: EType): Boolean =
-    {
-        t == Types.TIMESTAMP
-    }
-
-    override def newSchemaFieldType(name: String): PrimitiveType =
-    {
-        new PrimitiveType(PType.Repetition.OPTIONAL, PrimitiveTypeName.INT64, name, OriginalType.TIMESTAMP_MICROS)
-    }
-
-    override def consume(orig: Any,
-                         recordConsumer: RecordConsumer): Unit =
-    {
-        orig match {
-            case ts: Timestamp =>
-                val v = (ts.getEpochSecond * 1_000_000L) + (ts.getNano.asInstanceOf[Long] / 1_000L)
-                recordConsumer.addLong(v)
-            case _             => throw new DataException("given mismatched type value; expected type is timestamp")
-        }
+object TimestampMicrosLogicalTypeHandler extends LogicalTypeHandler {
+
+  override def isConvertible(t: EType): Boolean = {
+    t == Types.TIMESTAMP
+  }
+
+  override def newSchemaFieldType(name: String): PrimitiveType = {
+    new PrimitiveType(
+      PType.Repetition.OPTIONAL,
+      PrimitiveTypeName.INT64,
+      name,
+      OriginalType.TIMESTAMP_MICROS
+    )
+  }
+
+  override def consume(orig: Any, recordConsumer: RecordConsumer): Unit = {
+    orig match {
+      case ts: Timestamp =>
+        val v = (ts.getEpochSecond * 1_000_000L) + (ts.getNano
+          .asInstanceOf[Long] / 1_000L)
+        recordConsumer.addLong(v)
+      case _ =>
+        throw new DataException(
+          "given mismatched type value; expected type is timestamp"
+        )
     }
+  }
 }
 
-object Int8LogicalTypeHandler
-    extends IntLogicalTypeHandler(OriginalType.INT_8)
+object Int8LogicalTypeHandler extends IntLogicalTypeHandler(OriginalType.INT_8)
+
 object Int16LogicalTypeHandler
     extends IntLogicalTypeHandler(OriginalType.INT_16)
+
 object Int32LogicalTypeHandler
     extends IntLogicalTypeHandler(OriginalType.INT_32)
+
 object Int64LogicalTypeHandler
     extends IntLogicalTypeHandler(OriginalType.INT_64)
 
 object Uint8LogicalTypeHandler
     extends IntLogicalTypeHandler(OriginalType.UINT_8)
+
 object Uint16LogicalTypeHandler
     extends IntLogicalTypeHandler(OriginalType.UINT_16)
+
 object Uint32LogicalTypeHandler
     extends IntLogicalTypeHandler(OriginalType.UINT_32)
+
 object Uint64LogicalTypeHandler
     extends IntLogicalTypeHandler(OriginalType.UINT_64)
 
-object JsonLogicalTypeHandler
-    extends LogicalTypeHandler
-{
-    override def isConvertible(t: EType): Boolean =
-    {
-        t == Types.JSON
-    }
-
-    override def newSchemaFieldType(name: String): PrimitiveType =
-    {
-        new PrimitiveType(PType.Repetition.OPTIONAL, PrimitiveTypeName.BINARY, name, OriginalType.JSON)
-    }
-
-    override def consume(orig: Any,
-                         recordConsumer: RecordConsumer): Unit =
-    {
-        orig match {
-            case msgPack: Value =>
-                val bin = Binary.fromString(msgPack.toJson)
-                recordConsumer.addBinary(bin)
-            case _              => throw new DataException("given mismatched type value; expected type is json")
-        }
+object JsonLogicalTypeHandler extends LogicalTypeHandler {
+
+  override def isConvertible(t: EType): Boolean = {
+    t == Types.JSON
+  }
+
+  override def newSchemaFieldType(name: String): PrimitiveType = {
+    new PrimitiveType(
+      PType.Repetition.OPTIONAL,
+      PrimitiveTypeName.BINARY,
+      name,
+      OriginalType.JSON
+    )
+  }
+
+  override def consume(orig: Any, recordConsumer: RecordConsumer): Unit = {
+    orig match {
+      case msgPack: Value =>
+        val bin = Binary.fromString(msgPack.toJson)
+        recordConsumer.addBinary(bin)
+      case _ =>
+        throw new DataException(
+          "given mismatched type value; expected type is json"
+        )
     }
+  }
 }

data/src/main/scala/org/embulk/output/s3_parquet/parquet/LogicalTypeHandlerStore.scala

@@ -1,107 +1,114 @@
 package org.embulk.output.s3_parquet.parquet
 
-
 import org.embulk.spi.`type`.{Type, Types}
 import java.util.{Map => JMap}
 
 import org.embulk.config.ConfigException
-import org.embulk.output.s3_parquet.S3ParquetOutputPlugin.{ColumnOptionTask, TypeOptionTask}
+import org.embulk.output.s3_parquet.S3ParquetOutputPlugin.{
+  ColumnOptionTask,
+  TypeOptionTask
+}
 
 import scala.jdk.CollectionConverters._
 
-
 /**
- * A storage has mapping from logical type query (column name, type) to handler.
- *
- * @param fromEmbulkType
- * @param fromColumnName
- */
-case class LogicalTypeHandlerStore private(fromEmbulkType: Map[Type, LogicalTypeHandler],
-                                           fromColumnName: Map[String, LogicalTypeHandler])
-{
+  * A storage has mapping from logical type query (column name, type) to handler.
+  *
+  * @param fromEmbulkType
+  * @param fromColumnName
+  */
+case class LogicalTypeHandlerStore private (
+    fromEmbulkType: Map[Type, LogicalTypeHandler],
+    fromColumnName: Map[String, LogicalTypeHandler]
+) {
 
-    // Try column name lookup, then column type
-    def get(n: String,
-            t: Type): Option[LogicalTypeHandler] =
-    {
-        get(n) match {
-            case Some(h) => Some(h)
-            case _       =>
-                get(t) match {
-                    case Some(h) => Some(h)
-                    case _       => None
-                }
+  // Try column name lookup, then column type
+  def get(n: String, t: Type): Option[LogicalTypeHandler] = {
+    get(n) match {
+      case Some(h) => Some(h)
+      case _ =>
+        get(t) match {
+          case Some(h) => Some(h)
+          case _       => None
         }
     }
+  }
 
-    def get(t: Type): Option[LogicalTypeHandler] =
-    {
-        fromEmbulkType.get(t)
-    }
+  def get(t: Type): Option[LogicalTypeHandler] = {
+    fromEmbulkType.get(t)
+  }
 
-    def get(n: String): Option[LogicalTypeHandler] =
-    {
-        fromColumnName.get(n)
-    }
+  def get(n: String): Option[LogicalTypeHandler] = {
+    fromColumnName.get(n)
+  }
 }
 
-object LogicalTypeHandlerStore
-{
-    private val STRING_TO_EMBULK_TYPE = Map[String, Type](
-        "boolean" -> Types.BOOLEAN,
-        "long" -> Types.LONG,
-        "double" -> Types.DOUBLE,
-        "string" -> Types.STRING,
-        "timestamp" -> Types.TIMESTAMP,
-        "json" -> Types.JSON
-        )
+object LogicalTypeHandlerStore {
 
-    // Listed only older logical types that we can convert from embulk type
-    private val STRING_TO_LOGICAL_TYPE = Map[String, LogicalTypeHandler](
-        "timestamp-millis" -> TimestampMillisLogicalTypeHandler,
-        "timestamp-micros" -> TimestampMicrosLogicalTypeHandler,
-        "int8" -> Int8LogicalTypeHandler,
-        "int16" -> Int16LogicalTypeHandler,
-        "int32" -> Int32LogicalTypeHandler,
-        "int64" -> Int64LogicalTypeHandler,
-        "uint8" -> Uint8LogicalTypeHandler,
-        "uint16" -> Uint16LogicalTypeHandler,
-        "uint32" -> Uint32LogicalTypeHandler,
-        "uint64" -> Uint64LogicalTypeHandler,
-        "json" -> JsonLogicalTypeHandler
-        )
+  private val STRING_TO_EMBULK_TYPE = Map[String, Type](
+    "boolean" -> Types.BOOLEAN,
+    "long" -> Types.LONG,
+    "double" -> Types.DOUBLE,
+    "string" -> Types.STRING,
+    "timestamp" -> Types.TIMESTAMP,
+    "json" -> Types.JSON
+  )
 
-    def empty: LogicalTypeHandlerStore =
-    {
-        LogicalTypeHandlerStore(Map.empty[Type, LogicalTypeHandler], Map.empty[String, LogicalTypeHandler])
-    }
+  // Listed only older logical types that we can convert from embulk type
+  private val STRING_TO_LOGICAL_TYPE = Map[String, LogicalTypeHandler](
+    "timestamp-millis" -> TimestampMillisLogicalTypeHandler,
+    "timestamp-micros" -> TimestampMicrosLogicalTypeHandler,
+    "int8" -> Int8LogicalTypeHandler,
+    "int16" -> Int16LogicalTypeHandler,
+    "int32" -> Int32LogicalTypeHandler,
+    "int64" -> Int64LogicalTypeHandler,
+    "uint8" -> Uint8LogicalTypeHandler,
+    "uint16" -> Uint16LogicalTypeHandler,
+    "uint32" -> Uint32LogicalTypeHandler,
+    "uint64" -> Uint64LogicalTypeHandler,
+    "json" -> JsonLogicalTypeHandler
+  )
 
-    def fromEmbulkOptions(typeOpts: JMap[String, TypeOptionTask],
-                          columnOpts: JMap[String, ColumnOptionTask]): LogicalTypeHandlerStore =
-    {
-        val fromEmbulkType = typeOpts.asScala
-            .filter(_._2.getLogicalType.isPresent)
-            .map[Type, LogicalTypeHandler] { case (k, v) =>
-                val t = STRING_TO_EMBULK_TYPE.get(k)
-                val h = STRING_TO_LOGICAL_TYPE.get(v.getLogicalType.get)
-                (t, h) match {
-                    case (Some(tt), Some(hh)) => (tt, hh)
-                    case _                    => throw new ConfigException("invalid logical types in type_options")
-                }
-            }
-            .toMap
+  def empty: LogicalTypeHandlerStore = {
+    LogicalTypeHandlerStore(
+      Map.empty[Type, LogicalTypeHandler],
+      Map.empty[String, LogicalTypeHandler]
+    )
+  }
 
-        val fromColumnName = columnOpts.asScala
-            .filter(_._2.getLogicalType.isPresent)
-            .map[String, LogicalTypeHandler] { case (k, v) =>
-                val h = STRING_TO_LOGICAL_TYPE.get(v.getLogicalType.get)
-                h match {
-                    case Some(hh) => (k, hh)
-                    case _        => throw new ConfigException("invalid logical types in column_options")
-                }
-            }
-            .toMap
+  def fromEmbulkOptions(
+      typeOpts: JMap[String, TypeOptionTask],
+      columnOpts: JMap[String, ColumnOptionTask]
+  ): LogicalTypeHandlerStore = {
+    val fromEmbulkType = typeOpts.asScala
+      .filter(_._2.getLogicalType.isPresent)
+      .map[Type, LogicalTypeHandler] {
+        case (k, v) =>
+          val t = STRING_TO_EMBULK_TYPE.get(k)
+          val h = STRING_TO_LOGICAL_TYPE.get(v.getLogicalType.get)
+          (t, h) match {
+            case (Some(tt), Some(hh)) => (tt, hh)
+            case _ =>
+              throw new ConfigException("invalid logical types in type_options")
+          }
+      }
+      .toMap
 
-        LogicalTypeHandlerStore(fromEmbulkType, fromColumnName)
-    }
+    val fromColumnName = columnOpts.asScala
+      .filter(_._2.getLogicalType.isPresent)
+      .map[String, LogicalTypeHandler] {
+        case (k, v) =>
+          val h = STRING_TO_LOGICAL_TYPE.get(v.getLogicalType.get)
+          h match {
+            case Some(hh) => (k, hh)
+            case _ =>
+              throw new ConfigException(
+                "invalid logical types in column_options"
+              )
+          }
+      }
+      .toMap
+
+    LogicalTypeHandlerStore(fromEmbulkType, fromColumnName)
+  }
 }