embulk-output-s3_parquet 0.0.3 → 0.1.0

data/src/test/scala/org/embulk/output/s3_parquet/TestS3ParquetOutputPlugin.scala

@@ -4,9 +4,9 @@ package org.embulk.output.s3_parquet
 import java.io.File
 import java.nio.file.FileSystems
 
-import cloud.localstack.{DockerTestUtils, Localstack, TestUtils}
-import cloud.localstack.docker.LocalstackDocker
-import cloud.localstack.docker.annotation.LocalstackDockerConfiguration
+import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials}
+import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
+import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
 import com.amazonaws.services.s3.transfer.TransferManagerBuilder
 import com.google.common.io.Resources
 import org.apache.hadoop.fs.{Path => HadoopPath}
@@ -18,7 +18,7 @@ import org.embulk.test.{EmbulkTests, TestingEmbulk}
 import org.junit.Rule
 import org.junit.runner.RunWith
 import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, DiagrammedAssertions, FunSuite}
-import org.scalatest.junit.JUnitRunner
+import org.scalatestplus.junit.JUnitRunner
 
 import scala.annotation.meta.getter
 import scala.jdk.CollectionConverters._
@@ -33,22 +33,11 @@ class TestS3ParquetOutputPlugin
 {
 
     val RESOURCE_NAME_PREFIX: String = "org/embulk/output/s3_parquet/"
-    val BUCKET_NAME: String = "my-bucket"
-
-    val LOCALSTACK_DOCKER: LocalstackDocker = LocalstackDocker.INSTANCE
-
-    override protected def beforeAll(): Unit =
-    {
-        Localstack.teardownInfrastructure()
-        LOCALSTACK_DOCKER.startup(LocalstackDockerConfiguration.DEFAULT)
-        super.beforeAll()
-    }
-
-    override protected def afterAll(): Unit =
-    {
-        LOCALSTACK_DOCKER.stop()
-        super.afterAll()
-    }
+    val TEST_S3_ENDPOINT: String = "http://localhost:4572"
+    val TEST_S3_REGION: String = "us-east-1"
+    val TEST_S3_ACCESS_KEY_ID: String = "test"
+    val TEST_S3_SECRET_ACCESS_KEY: String = "test"
+    val TEST_BUCKET_NAME: String = "my-bucket"
 
     @(Rule@getter)
     val embulk: TestingEmbulk = TestingEmbulk.builder()
@@ -56,7 +45,11 @@ class TestS3ParquetOutputPlugin
         .build()
 
     before {
-        DockerTestUtils.getClientS3.createBucket(BUCKET_NAME)
+        withLocalStackS3Client(_.createBucket(TEST_BUCKET_NAME))
+    }
+
+    after {
+        withLocalStackS3Client(_.deleteBucket(TEST_BUCKET_NAME))
     }
 
     def defaultOutConfig(): ConfigSource =
@@ -64,11 +57,11 @@ class TestS3ParquetOutputPlugin
         embulk.newConfig()
             .set("type", "s3_parquet")
             .set("endpoint", "http://localhost:4572") // See https://github.com/localstack/localstack#overview
-            .set("bucket", BUCKET_NAME)
+            .set("bucket", TEST_BUCKET_NAME)
             .set("path_prefix", "path/to/p")
             .set("auth_method", "basic")
-            .set("access_key_id", TestUtils.TEST_ACCESS_KEY)
-            .set("secret_access_key", TestUtils.TEST_SECRET_KEY)
+            .set("access_key_id", TEST_S3_ACCESS_KEY_ID)
+            .set("secret_access_key", TEST_S3_SECRET_ACCESS_KEY)
             .set("path_style_access_enabled", true)
             .set("default_timezone", "Asia/Tokyo")
     }
@@ -111,12 +104,14 @@ class TestS3ParquetOutputPlugin
     def readParquetFile(bucket: String,
                         key: String): Seq[Map[String, String]] =
     {
-        val xfer = TransferManagerBuilder.standard()
-            .withS3Client(DockerTestUtils.getClientS3)
-            .build()
         val createdParquetFile = embulk.createTempFile("in")
-        try xfer.download(bucket, key, createdParquetFile.toFile).waitForCompletion()
-        finally xfer.shutdownNow()
+        withLocalStackS3Client {s3 =>
+            val xfer = TransferManagerBuilder.standard()
+                .withS3Client(s3)
+                .build()
+            try xfer.download(bucket, key, createdParquetFile.toFile).waitForCompletion()
+            finally xfer.shutdownNow()
+        }
 
         val reader: ParquetReader[SimpleRecord] = ParquetReader
             .builder(new SimpleReadSupport(), new HadoopPath(createdParquetFile.toString))
@@ -146,4 +141,14 @@ class TestS3ParquetOutputPlugin
         FileSystems.getDefault.getPath(new File(url.toURI).getAbsolutePath)
     }
 
+    private def withLocalStackS3Client[A](f: AmazonS3 => A): A = {
+        val client: AmazonS3 = AmazonS3ClientBuilder.standard
+            .withEndpointConfiguration(new EndpointConfiguration(TEST_S3_ENDPOINT, TEST_S3_REGION))
+            .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(TEST_S3_ACCESS_KEY_ID, TEST_S3_SECRET_ACCESS_KEY)))
+            .withPathStyleAccessEnabled(true)
+            .build()
+
+        try f(client)
+        finally client.shutdown()
+    }
 }

data/src/test/scala/org/embulk/output/s3_parquet/parquet/TestLogicalTypeHandler.scala

@@ -0,0 +1,78 @@
+package org.embulk.output.s3_parquet.parquet
+
+
+import org.embulk.spi.DataException
+import org.embulk.spi.`type`.Types
+import org.junit.runner.RunWith
+import org.scalatest.FunSuite
+import org.scalatestplus.junit.JUnitRunner
+
+import scala.util.Try
+
+
+@RunWith(classOf[JUnitRunner])
+class TestLogicalTypeHandler
+    extends FunSuite
+{
+
+    test("IntLogicalTypeHandler.isConvertible() returns true for long") {
+        val h = Int8LogicalTypeHandler
+
+        assert(h.isConvertible(Types.LONG))
+        assert(!h.isConvertible(Types.BOOLEAN))
+    }
+
+    test("IntLogicalTypeHandler.consume() raises DataException if given type is not long") {
+        val h = Int8LogicalTypeHandler
+        val actual = Try(h.consume("invalid", null))
+
+        assert(actual.isFailure)
+        assert(actual.failed.get.isInstanceOf[DataException])
+    }
+
+
+    test("TimestampMillisLogicalTypeHandler.isConvertible() returns true for timestamp") {
+        val h = TimestampMillisLogicalTypeHandler
+
+        assert(h.isConvertible(Types.TIMESTAMP))
+        assert(!h.isConvertible(Types.BOOLEAN))
+    }
+
+    test("TimestampMillisLogicalTypeHandler.consume() raises DataException if given type is not timestamp") {
+        val h = TimestampMillisLogicalTypeHandler
+        val actual = Try(h.consume("invalid", null))
+
+        assert(actual.isFailure)
+        assert(actual.failed.get.isInstanceOf[DataException])
+    }
+
+
+    test("TimestampMicrosLogicalTypeHandler.isConvertible() returns true for timestamp") {
+        val h = TimestampMicrosLogicalTypeHandler
+
+        assert(h.isConvertible(Types.TIMESTAMP))
+        assert(!h.isConvertible(Types.BOOLEAN))
+    }
+
+    test("TimestampMicrosLogicalTypeHandler.consume() raises DataException if given type is not timestamp") {
+        val h = TimestampMicrosLogicalTypeHandler
+        val actual = Try(h.consume("invalid", null))
+
+        assert(actual.isFailure)
+        assert(actual.failed.get.isInstanceOf[DataException])
+    }
+
+    test("JsonLogicalTypeHandler.isConvertible() returns true for json") {
+        val h = JsonLogicalTypeHandler
+
+        assert(h.isConvertible(Types.JSON))
+        assert(!h.isConvertible(Types.BOOLEAN))
+    }
+
+    test("JsonLogicalTypeHandler.consume() raises DataException if given type is not json") {
+        val h = JsonLogicalTypeHandler
+        val actual = Try(h.consume("invalid", null))
+        assert(actual.isFailure)
+        assert(actual.failed.get.isInstanceOf[DataException])
+    }
+}

data/src/test/scala/org/embulk/output/s3_parquet/parquet/TestLogicalTypeHandlerStore.scala

@@ -0,0 +1,162 @@
+package org.embulk.output.s3_parquet.parquet
+
+
+import java.util.Optional
+
+import com.google.common.base.{Optional => GOptional}
+import org.embulk.config.{ConfigException, TaskSource}
+import org.embulk.output.s3_parquet.S3ParquetOutputPlugin.{ColumnOptionTask, TypeOptionTask}
+import org.embulk.spi.`type`.{Types, Type => EType}
+import org.junit.runner.RunWith
+import org.scalatest.FunSuite
+import org.scalatestplus.junit.JUnitRunner
+
+import scala.jdk.CollectionConverters._
+import scala.util.Try
+
+
+@RunWith(classOf[JUnitRunner])
+class TestLogicalTypeHandlerStore
+    extends FunSuite
+{
+    test("empty() returns empty maps") {
+        val rv = LogicalTypeHandlerStore.empty
+
+        assert(rv.fromColumnName.isEmpty)
+        assert(rv.fromEmbulkType.isEmpty)
+    }
+
+    test("fromEmbulkOptions() returns handlers for valid option tasks") {
+        val typeOpts = Map[String, TypeOptionTask](
+            "timestamp" -> DummyTypeOptionTask(Optional.of[String]("timestamp-millis")),
+            ).asJava
+        val columnOpts = Map[String, ColumnOptionTask](
+            "col1" -> DummyColumnOptionTask(Optional.of[String]("timestamp-micros")),
+            ).asJava
+
+        val expected1 = Map[EType, LogicalTypeHandler](
+            Types.TIMESTAMP -> TimestampMillisLogicalTypeHandler,
+            )
+        val expected2 = Map[String, LogicalTypeHandler](
+            "col1" -> TimestampMicrosLogicalTypeHandler,
+            )
+
+        val rv = LogicalTypeHandlerStore.fromEmbulkOptions(typeOpts, columnOpts)
+
+        assert(rv.fromEmbulkType == expected1)
+        assert(rv.fromColumnName == expected2)
+    }
+
+    test("fromEmbulkOptions() raises ConfigException if invalid option tasks given") {
+        val emptyTypeOpts = Map.empty[String, TypeOptionTask].asJava
+        val emptyColumnOpts = Map.empty[String, ColumnOptionTask].asJava
+
+        val invalidTypeOpts = Map[String, TypeOptionTask](
+            "unknown-embulk-type-name" -> DummyTypeOptionTask(Optional.of[String]("timestamp-millis")),
+            "timestamp" -> DummyTypeOptionTask(Optional.of[String]("unknown-parquet-logical-type-name")),
+            ).asJava
+        val invalidColumnOpts = Map[String, ColumnOptionTask](
+            "col1" -> DummyColumnOptionTask(Optional.of[String]("unknown-parquet-logical-type-name")),
+            ).asJava
+
+        val try1 = Try(LogicalTypeHandlerStore.fromEmbulkOptions(invalidTypeOpts, emptyColumnOpts))
+        assert(try1.isFailure)
+        assert(try1.failed.get.isInstanceOf[ConfigException])
+
+        val try2 = Try(LogicalTypeHandlerStore.fromEmbulkOptions(emptyTypeOpts, invalidColumnOpts))
+        assert(try2.isFailure)
+        assert(try2.failed.get.isInstanceOf[ConfigException])
+
+        val try3 = Try(LogicalTypeHandlerStore.fromEmbulkOptions(invalidTypeOpts, invalidColumnOpts))
+        assert(try3.isFailure)
+        assert(try3.failed.get.isInstanceOf[ConfigException])
+    }
+
+    test("get() returns a handler matched with primary column name condition") {
+        val typeOpts = Map[String, TypeOptionTask](
+            "timestamp" -> DummyTypeOptionTask(Optional.of[String]("timestamp-millis")),
+            ).asJava
+        val columnOpts = Map[String, ColumnOptionTask](
+            "col1" -> DummyColumnOptionTask(Optional.of[String]("timestamp-micros")),
+            ).asJava
+
+        val handlers = LogicalTypeHandlerStore.fromEmbulkOptions(typeOpts, columnOpts)
+
+        // It matches both of column name and embulk type, and column name should be primary
+        val expected = Some(TimestampMicrosLogicalTypeHandler)
+        val actual = handlers.get("col1", Types.TIMESTAMP)
+
+        assert(actual == expected)
+    }
+
+    test("get() returns a handler matched with type name condition") {
+        val typeOpts = Map[String, TypeOptionTask](
+            "timestamp" -> DummyTypeOptionTask(Optional.of[String]("timestamp-millis")),
+            ).asJava
+        val columnOpts = Map.empty[String, ColumnOptionTask].asJava
+
+        val handlers = LogicalTypeHandlerStore.fromEmbulkOptions(typeOpts, columnOpts)
+
+        // It matches column name
+        val expected = Some(TimestampMillisLogicalTypeHandler)
+        val actual = handlers.get("col1", Types.TIMESTAMP)
+
+        assert(actual == expected)
+    }
+
+    test("get() returns None if not matched") {
+        val typeOpts = Map.empty[String, TypeOptionTask].asJava
+        val columnOpts = Map.empty[String, ColumnOptionTask].asJava
+
+        val handlers = LogicalTypeHandlerStore.fromEmbulkOptions(typeOpts, columnOpts)
+
+        // It matches embulk type
+        val actual = handlers.get("col1", Types.TIMESTAMP)
+
+        assert(actual.isEmpty)
+    }
+
+    private case class DummyTypeOptionTask(lt: Optional[String])
+        extends TypeOptionTask
+    {
+        override def getLogicalType: Optional[String] =
+        {
+            lt
+        }
+
+        override def validate(): Unit =
+        {}
+
+        override def dump(): TaskSource =
+        {
+            null
+        }
+    }
+
+    private case class DummyColumnOptionTask(lt: Optional[String])
+        extends ColumnOptionTask
+    {
+        override def getTimeZoneId: GOptional[String] =
+        {
+            GOptional.absent[String]
+        }
+
+        override def getFormat: GOptional[String] =
+        {
+            GOptional.absent[String]
+        }
+
+        override def getLogicalType: Optional[String] =
+        {
+            lt
+        }
+
+        override def validate(): Unit =
+        {}
+
+        override def dump(): TaskSource =
+        {
+            null
+        }
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-s3_parquet
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.1.0
 platform: ruby
 authors:
 - Civitaspo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-07-17 00:00:00.000000000 Z
+date: 2019-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -45,6 +45,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/FUNDING.yml"
+- ".github/workflows/release.yml"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - CHANGELOG.md
 - LICENSE.txt
@@ -59,14 +62,14 @@ files:
 - classpath/asm-3.1.jar
 - classpath/asm-5.0.4.jar
 - classpath/avro-1.7.7.jar
-- classpath/aws-java-sdk-core-1.11.592.jar
-- classpath/aws-java-sdk-glue-1.11.592.jar
-- classpath/aws-java-sdk-kms-1.11.592.jar
-- classpath/aws-java-sdk-s3-1.11.592.jar
-- classpath/aws-java-sdk-sts-1.11.592.jar
+- classpath/aws-java-sdk-core-1.11.676.jar
+- classpath/aws-java-sdk-glue-1.11.676.jar
+- classpath/aws-java-sdk-kms-1.11.676.jar
+- classpath/aws-java-sdk-s3-1.11.676.jar
+- classpath/aws-java-sdk-sts-1.11.676.jar
 - classpath/commons-beanutils-1.7.0.jar
 - classpath/commons-cli-1.2.jar
-- classpath/commons-codec-1.10.jar
+- classpath/commons-codec-1.11.jar
 - classpath/commons-collections-3.2.2.jar
 - classpath/commons-compress-1.4.1.jar
 - classpath/commons-configuration-1.6.jar
@@ -80,17 +83,17 @@ files:
 - classpath/curator-client-2.7.1.jar
 - classpath/curator-framework-2.7.1.jar
 - classpath/curator-recipes-2.7.1.jar
-- classpath/embulk-output-s3_parquet-0.0.3.jar
+- classpath/embulk-output-s3_parquet-0.1.0.jar
 - classpath/gson-2.2.4.jar
 - classpath/hadoop-annotations-2.9.2.jar
 - classpath/hadoop-auth-2.9.2.jar
 - classpath/hadoop-common-2.9.2.jar
 - classpath/htrace-core4-4.1.0-incubating.jar
-- classpath/httpclient-4.5.5.jar
-- classpath/httpcore-4.4.9.jar
+- classpath/httpclient-4.5.9.jar
+- classpath/httpcore-4.4.11.jar
 - classpath/ion-java-1.0.2.jar
 - classpath/jackson-core-asl-1.9.13.jar
-- classpath/jackson-databind-2.6.7.2.jar
+- classpath/jackson-databind-2.6.7.3.jar
 - classpath/jackson-dataformat-cbor-2.6.7.jar
 - classpath/jackson-jaxrs-1.8.3.jar
 - classpath/jackson-mapper-asl-1.9.13.jar
@@ -108,7 +111,7 @@ files:
 - classpath/jetty-sslengine-6.1.26.jar
 - classpath/jetty-util-6.1.26.jar
 - classpath/jline-0.9.94.jar
-- classpath/jmespath-java-1.11.592.jar
+- classpath/jmespath-java-1.11.676.jar
 - classpath/jsch-0.1.54.jar
 - classpath/json-smart-2.3.jar
 - classpath/jsp-api-2.1.jar
@@ -124,7 +127,7 @@ files:
 - classpath/parquet-hadoop-1.10.1.jar
 - classpath/parquet-jackson-1.10.1.jar
 - classpath/protobuf-java-2.5.0.jar
-- classpath/scala-library-2.13.0.jar
+- classpath/scala-library-2.13.1.jar
 - classpath/servlet-api-2.5-20081211.jar
 - classpath/servlet-api-2.5.jar
 - classpath/slf4j-api-1.7.25.jar
@@ -138,6 +141,8 @@ files:
 - classpath/zookeeper-3.4.6.jar
 - example/config.yml
 - example/data.tsv
+- example/with_catalog.yml
+- example/with_logicaltypes.yml
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
@@ -154,11 +159,15 @@ files:
 - src/main/scala/org/embulk/output/s3_parquet/aws/AwsS3Configuration.scala
 - src/main/scala/org/embulk/output/s3_parquet/aws/HttpProxy.scala
 - src/main/scala/org/embulk/output/s3_parquet/parquet/EmbulkMessageType.scala
+- src/main/scala/org/embulk/output/s3_parquet/parquet/LogicalTypeHandler.scala
+- src/main/scala/org/embulk/output/s3_parquet/parquet/LogicalTypeHandlerStore.scala
 - src/main/scala/org/embulk/output/s3_parquet/parquet/ParquetFileWriteSupport.scala
 - src/main/scala/org/embulk/output/s3_parquet/parquet/ParquetFileWriter.scala
 - src/test/resources/org/embulk/output/s3_parquet/in1.csv
 - src/test/resources/org/embulk/output/s3_parquet/out1.tsv
 - src/test/scala/org/embulk/output/s3_parquet/TestS3ParquetOutputPlugin.scala
+- src/test/scala/org/embulk/output/s3_parquet/parquet/TestLogicalTypeHandler.scala
+- src/test/scala/org/embulk/output/s3_parquet/parquet/TestLogicalTypeHandlerStore.scala
 homepage: https://github.com/civitaspo/embulk-output-s3_parquet
 licenses:
 - MIT