embulk-parser-firebase_avro 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9335540798899c6d98b90f79d9afeddf525cf50d
-  data.tar.gz: 700ea6f57bbc2699431aefe46627bb90b7edfb23
+  metadata.gz: b4ac8ed35dad7dbbde18928b68dca357b9fc5784
+  data.tar.gz: 898d4040eaf4d63ed5b2a5dbd5de8d361a33c962
 SHA512:
-  metadata.gz: a2fb05f739e76c481b62bd8b909fd1d4617d48642b1ae85516abe1a12aa9a5cca547ee3be44ce2087740fe5eebd5ade48058933fc533ec080230b56adc4e09d4
-  data.tar.gz: 8f5ffc78f5ebd333a787a14b77d6a3d7f833fa782b10397a7fe9913ee80a38b5f3b4341b41434ec5a88f78937ef705dad09bcf428664da093a4d5d9771d98f96
+  metadata.gz: a07439fba8c76371a24d7d981b637b5fb9aed5b45885f10280923c44bf968acb29d5f2ea8923a82848257a3621d68425514f17c81082a0b060e65479bba94be5
+  data.tar.gz: 2b2582964a00485b7067296a6bb12d72b90173e06c6cadc7a4b6bdf5e5ce08009500f0258e774f69c2ebab87c7a98f11e2f422faa4f96f376aa3f3c302120c80

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.1.2"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/build.sbt

@@ -1,29 +1,32 @@
-lazy val root = (project in file(".")).
-  settings(
-    inThisBuild(List(
+enablePlugins(ScalafmtPlugin)
+
+lazy val root = (project in file(".")).settings(
+  inThisBuild(
+    List(
       organization := "com.example",
       scalaVersion := "2.11.11",
-      version      := "0.1.0-SNAPSHOT"
+      version := "0.1.0-SNAPSHOT"
     )),
-    name := "embulk-parser-firebase_avro"
-  )
+  name := "embulk-parser-firebase_avro",
+  scalafmtOnCompile in ThisBuild := true,
+  scalafmtTestOnCompile in ThisBuild := true
+)
 
 enablePlugins(ScalafmtPlugin)
 
 resolvers += Resolver.jcenterRepo
 resolvers += Resolver.sonatypeRepo("releases")
 
-
 lazy val circeVersion = "0.8.0"
 libraryDependencies ++= Seq(
-  "com.sksamuel.avro4s" %% "avro4s-core" % "1.6.4",
-  "org.jruby" % "jruby-complete" % "1.6.5",
-  "org.embulk" % "embulk-core" % "0.8.22",
-  "com.chuusai" %% "shapeless" % "2.3.2",
-  "io.circe" %% "circe-core" % circeVersion,
-  "io.circe" %% "circe-generic" % circeVersion,
-  "org.scalacheck" %% "scalacheck" % "1.13.4"  % Test,
-  "org.scalatest" %% "scalatest" % "3.0.1" % Test,
-  "org.scalamock" %% "scalamock-scalatest-support" % "3.6.0" % Test,
-  "com.github.alexarchambault" %% "scalacheck-shapeless_1.13" % "1.1.5" % Test
-)
+  "com.sksamuel.avro4s"        %% "avro4s-core"                 % "1.6.4",
+  "org.jruby"                  % "jruby-complete"               % "1.6.5",
+  "org.embulk"                 % "embulk-core"                  % "0.8.22",
+  "com.chuusai"                %% "shapeless"                   % "2.3.2",
+  "io.circe"                   %% "circe-core"                  % circeVersion,
+  "io.circe"                   %% "circe-generic"               % circeVersion,
+  "org.scalacheck"             %% "scalacheck"                  % "1.13.4" % Test,
+  "org.scalatest"              %% "scalatest"                   % "3.0.1" % Test,
+  "org.scalamock"              %% "scalamock-scalatest-support" % "3.6.0" % Test,
+  "com.github.alexarchambault" %% "scalacheck-shapeless_1.13"   % "1.1.5" % Test
+)

data/project/plugins.sbt

@@ -1,3 +1,3 @@
 addSbtPlugin("com.julianpeeters" % "sbt-avrohugger" % "0.16.0")
-addSbtPlugin("com.lucidchart" % "sbt-scalafmt" % "0.4")
+addSbtPlugin("com.lucidchart" % "sbt-scalafmt" % "1.7")
 addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-RC6")

data/settings.gradle

@@ -0,0 +1 @@
+rootProject.name = 'embulk-parser-firebase_avro'

data/src/main/scala/org/embulk/parser/firebase_avro/FirebaseAvroParserPlugin.scala

@@ -1,11 +1,10 @@
 package org.embulk.parser.firebase_avro
 
-import java.io.InputStream
+import com.google.common.io.ByteStreams
 
 import scala.collection.JavaConverters._
 import com.sksamuel.avro4s.AvroInputStream
 import io.circe.Json
-import org.apache.commons.compress.utils.IOUtils
 import org.embulk.config.ConfigSource
 import org.embulk.config.TaskSource
 import org.embulk.parser.firebase_avro.column.Columns
@@ -22,45 +21,46 @@ object FirebaseAvroParserPlugin {
 }
 
 class FirebaseAvroParserPlugin extends ParserPlugin {
+  val logger = Exec.getLogger(classOf[FirebaseAvroParserPlugin])
   override def transaction(config: ConfigSource, control: ParserPlugin.Control): Unit = {
     val task = config.loadConfig(classOf[PluginTask])
     control.run(task.dump, FirebaseAvroParserPlugin.buildColumn())
   }
 
-  override def run(taskSource: TaskSource, schema: Schema, input: FileInput, output: PageOutput): Unit =
-    LoanPattern(new FileInputInputStream(input)) { efis =>
-      LoanPattern(new PageBuilder(Exec.getBufferAllocator, schema, output)) { pb =>
-        while (efis.nextFile()) {
-          addRecords(efis, pb)
+  override def run(taskSource: TaskSource, schema: Schema, input: FileInput, output: PageOutput): Unit = {
+    LoanPattern(new PageBuilder(Exec.getBufferAllocator, schema, output)) { pb =>
+      while (input.nextFile()) {
+        val bytes = ByteStreams.toByteArray(new FileInputInputStream(input))
+        AvroInputStream.data[Root](bytes).iterator().toList.foreach { record =>
+          addRecords(pb, record)
         }
-        pb.finish()
       }
+      pb.finish()
     }
+  }
 
-  def addRecords(is: InputStream, pb: PageBuilder): Unit =
-    AvroInputStream.data[Root](IOUtils.toByteArray(is)).iterator().foreach { record =>
-      Parser(record).foreach { rows =>
-        rows.foreach {
-          case ValueHolder(c, Some(x: Int)) =>
-            pb.setLong(c, x)
-          case ValueHolder(c, Some(x: Long)) =>
-            pb.setLong(c, x)
-          case ValueHolder(c, Some(x: Double)) =>
-            pb.setDouble(c, x)
-          case ValueHolder(c, Some(x: Float)) =>
-            pb.setDouble(c, x)
-          case ValueHolder(c, Some(x: Boolean)) =>
-            pb.setBoolean(c, x)
-          case ValueHolder(c, Some(x: String)) =>
-            pb.setString(c, x)
-          case ValueHolder(c, Some(x: Json)) =>
-            pb.setJson(c, new JsonParser().parse(x.noSpaces))
-          case ValueHolder(c, Some(x: Timestamp)) =>
-            pb.setTimestamp(c, x)
-          case ValueHolder(c, None) =>
-            pb.setNull(c)
-        }
-        pb.addRecord()
+  def addRecords(pb: PageBuilder, record: Root): Unit =
+    Parser(record).foreach { rows =>
+      rows.foreach {
+        case ValueHolder(c, Some(x: Int)) =>
+          pb.setLong(c, x)
+        case ValueHolder(c, Some(x: Long)) =>
+          pb.setLong(c, x)
+        case ValueHolder(c, Some(x: Double)) =>
+          pb.setDouble(c, x)
+        case ValueHolder(c, Some(x: Float)) =>
+          pb.setDouble(c, x)
+        case ValueHolder(c, Some(x: Boolean)) =>
+          pb.setBoolean(c, x)
+        case ValueHolder(c, Some(x: String)) =>
+          pb.setString(c, x)
+        case ValueHolder(c, Some(x: Json)) =>
+          pb.setJson(c, new JsonParser().parse(x.noSpaces))
+        case ValueHolder(c, Some(x: Timestamp)) =>
+          pb.setTimestamp(c, x)
+        case ValueHolder(c, None) =>
+          pb.setNull(c)
       }
+      pb.addRecord()
     }
 }

data/src/main/scala/org/embulk/parser/firebase_avro/Parser.scala

@@ -12,9 +12,14 @@ object Parser {
 
   def apply(record: Root): Seq[Seq[ValueHolder[_]]] = {
     val userFields = userDims(record.user_dim.getOrElse(sys.error("could not get user")))
-    if (record.event_dim.isEmpty) sys.error("empty event")
-    record.event_dim.map {
-      userFields ++ eventDims(_)
+    if (record.event_dim.isEmpty) {
+      record.copy(event_dim = List(Event_Dim.empty)).event_dim.map {
+        userFields ++ eventDims(_)
+      }
+    } else {
+      record.event_dim.map {
+        userFields ++ eventDims(_)
+      }
     }
   }
 

data/src/main/scala/org/embulk/parser/firebase_avro/define/root/Event_Dim.scala

@@ -6,3 +6,7 @@ case class Event_Dim(date: Option[String],
                      timestamp_micros: Option[Long],
                      previous_timestamp_micros: Option[Long],
                      value_in_usd: Option[Double])
+
+object Event_Dim {
+  val empty = Event_Dim(None, None, Nil, None, None, None)
+}

data/src/test/scala/org/embulk/parser/firebase_avro/column/ColumnsTest.scala

@@ -8,11 +8,11 @@ class ColumnsTest extends FlatSpec with MustMatchers {
   "columuns" should "be indexing" in {
     val columns = Columns()
     val summery = columns.indices.sum
-    columns.map(_.embulkColumn).map(_.getIndex).sum should be (summery)
+    columns.map(_.embulkColumn).map(_.getIndex).sum should be(summery)
   }
 
   "columns" should "be finding" in {
-    Columns.find("user_dim" , "first_open_timestamp_micros")
+    Columns.find("user_dim", "first_open_timestamp_micros")
   }
 
 }

data/src/test/scala/org/embulk/parser/firebase_avro/json/event_dim/EventParmsJsonSerializerTest.scala

@@ -8,12 +8,12 @@ import org.scalatest._
 class EventParmsJsonSerializerTest extends FlatSpec with MustMatchers {
 
   "parameter" should "be encoding" in {
-    val string = Params(Option("key_string") , Option(Value(string_value = Some("abc"))))
-    val int = Params(Option("key_int") , Option(Value(int_value = Some(1))))
-    val double = Params(Option("key_double") , Option(Value(double_value = Some(10D))))
-    val float = Params(Option("key_float") , Option(Value(float_value = Some(10F))))
-    val given = List(string , int , double , float)
-    val that = EventParmsJsonSerializer(given).map(_.noSpaces)
+    val string = Params(Option("key_string"), Option(Value(string_value = Some("abc"))))
+    val int    = Params(Option("key_int"), Option(Value(int_value = Some(1))))
+    val double = Params(Option("key_double"), Option(Value(double_value = Some(10D))))
+    val float  = Params(Option("key_float"), Option(Value(float_value = Some(10F))))
+    val given  = List(string, int, double, float)
+    val that   = EventParmsJsonSerializer(given).map(_.noSpaces)
     that mustBe Some("{\"key_float\":10.0,\"key_int\":1,\"key_string\":\"abc\",\"key_double\":10.0}")
   }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-firebase_avro
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - smdmts
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-07-12 00:00:00.000000000 Z
+date: 2017-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -59,6 +59,7 @@ files:
 - lib/embulk/parser/firebase_avro.rb
 - project/build.properties
 - project/plugins.sbt
+- settings.gradle
 - src/main/scala/org/embulk/parser/firebase_avro/FirebaseAvroParserPlugin.scala
 - src/main/scala/org/embulk/parser/firebase_avro/LoanPattern.scala
 - src/main/scala/org/embulk/parser/firebase_avro/Parser.scala
@@ -99,7 +100,7 @@ files:
 - classpath/circe-generic_2.11-0.8.0.jar
 - classpath/circe-numbers_2.11-0.8.0.jar
 - classpath/commons-compress-1.8.1.jar
-- classpath/embulk-parser-firebase_avro-0.1.1.jar
+- classpath/embulk-parser-firebase_avro-0.1.2.jar
 - classpath/jackson-core-asl-1.9.13.jar
 - classpath/jackson-mapper-asl-1.9.13.jar
 - classpath/machinist_2.11-0.6.1.jar