embulk-parser-firebase_avro 0.1.0

data/gradlew.bat

@@ -0,0 +1,84 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windows variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/lib/embulk/parser/firebase_avro.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_parser(
+  "firebase_avro", "org.embulk.parser.firebase_avro.FirebaseAvroParserPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/project/build.properties

@@ -0,0 +1 @@
+sbt.version=0.13.15

data/project/plugins.sbt

@@ -0,0 +1,3 @@
+addSbtPlugin("com.julianpeeters" % "sbt-avrohugger" % "0.16.0")
+addSbtPlugin("com.lucidchart" % "sbt-scalafmt" % "0.4")
+addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-RC6")

data/src/main/scala/org/embulk/parser/firebase_avro/FirebaseAvroParserPlugin.scala

@@ -0,0 +1,66 @@
+package org.embulk.parser.firebase_avro
+
+import java.io.InputStream
+
+import scala.collection.JavaConverters._
+import com.sksamuel.avro4s.AvroInputStream
+import io.circe.Json
+import org.apache.commons.compress.utils.IOUtils
+import org.embulk.config.ConfigSource
+import org.embulk.config.TaskSource
+import org.embulk.parser.firebase_avro.column.Columns
+import org.embulk.parser.firebase_avro.define.Root
+import org.embulk.spi._
+import org.embulk.spi.json.JsonParser
+import org.embulk.spi.time.Timestamp
+import org.embulk.spi.util.FileInputInputStream
+
+object FirebaseAvroParserPlugin {
+  def buildColumn(): Schema = {
+    new Schema(Columns.instance.map(_.embulkColumn).asJava)
+  }
+}
+
+class FirebaseAvroParserPlugin extends ParserPlugin {
+  override def transaction(config: ConfigSource, control: ParserPlugin.Control): Unit = {
+    val task = config.loadConfig(classOf[PluginTask])
+    control.run(task.dump, FirebaseAvroParserPlugin.buildColumn())
+  }
+
+  override def run(taskSource: TaskSource, schema: Schema, input: FileInput, output: PageOutput): Unit =
+    LoanPattern(new FileInputInputStream(input)) { efis =>
+      LoanPattern(new PageBuilder(Exec.getBufferAllocator, schema, output)) { pb =>
+        while (efis.nextFile()) {
+          addRecords(efis, pb)
+        }
+        pb.finish()
+      }
+    }
+
+  def addRecords(is: InputStream, pb: PageBuilder): Unit =
+    AvroInputStream.data[Root](IOUtils.toByteArray(is)).iterator().foreach { record =>
+      Parser(record).foreach { rows =>
+        rows.foreach {
+          case ValueHolder(c, Some(x: Int)) =>
+            pb.setLong(c, x)
+          case ValueHolder(c, Some(x: Long)) =>
+            pb.setLong(c, x)
+          case ValueHolder(c, Some(x: Double)) =>
+            pb.setDouble(c, x)
+          case ValueHolder(c, Some(x: Float)) =>
+            pb.setDouble(c, x)
+          case ValueHolder(c, Some(x: Boolean)) =>
+            pb.setBoolean(c, x)
+          case ValueHolder(c, Some(x: String)) =>
+            pb.setString(c, x)
+          case ValueHolder(c, Some(x: Json)) =>
+            pb.setJson(c, new JsonParser().parse(x.noSpaces))
+          case ValueHolder(c, Some(x: Timestamp)) =>
+            pb.setTimestamp(c, x)
+          case ValueHolder(c, None) =>
+            pb.setNull(c)
+        }
+        pb.addRecord()
+      }
+    }
+}

data/src/main/scala/org/embulk/parser/firebase_avro/LoanPattern.scala

@@ -0,0 +1,19 @@
+package org.embulk.parser.firebase_avro
+
+import scala.util.control.Exception.ignoring
+
+object LoanPattern {
+
+  type Closable = { def close(): Unit }
+
+  def apply[R <: Closable, A](resource: R)(f: R => A): A = {
+    try {
+      f(resource)
+    } finally {
+      ignoring(classOf[Throwable]) apply {
+        resource.close()
+      }
+    }
+  }
+
+}

data/src/main/scala/org/embulk/parser/firebase_avro/Parser.scala

@@ -0,0 +1,111 @@
+package org.embulk.parser.firebase_avro
+
+import org.embulk.parser.firebase_avro.column.Columns
+import org.embulk.parser.firebase_avro.define.Root
+import org.embulk.parser.firebase_avro.define.root.{Event_Dim, User_Dim}
+import org.embulk.parser.firebase_avro.json.event_dim.EventParmsJsonSerializer
+import org.embulk.parser.firebase_avro.json.user_dim.UserPropertiesJsonSerializer
+
+import scala.language.reflectiveCalls
+
+object Parser {
+
+  def apply(record: Root): Seq[Seq[ValueHolder[_]]] = {
+    val userFields = userDims(record.user_dim.getOrElse(sys.error("could not get user")))
+    if (record.event_dim.isEmpty) sys.error("empty event")
+    record.event_dim.map {
+      userFields ++ eventDims(_)
+    }
+  }
+
+  private def eventDims(eventDim: Event_Dim) = {
+    val c = Columns.find("event_dim", _: String)
+    Seq(
+      ValueHolder(c("name"), eventDim.name),
+      ValueHolder(c("date"), eventDim.date),
+      ValueHolder(c("timestamp_micros"), eventDim.timestamp_micros),
+      ValueHolder(c("previous_timestamp_micros"), eventDim.previous_timestamp_micros),
+      ValueHolder(c("value_in_usd"), eventDim.value_in_usd),
+      ValueHolder(c("params"), EventParmsJsonSerializer(eventDim.params))
+    )
+  }
+
+  private def userDims(userDim: User_Dim) = {
+    val userFields = {
+      val c = Columns.find("user_dim", _: String)
+      Seq(
+        ValueHolder(c("user_id"), userDim.user_id),
+        ValueHolder(c("first_open_timestamp_micros"), userDim.first_open_timestamp_micros),
+        ValueHolder(c("user_properties"), UserPropertiesJsonSerializer(userDim.user_properties))
+      )
+    }
+
+    val appInfo = {
+      val c    = Columns.find("app_info", _: String)
+      val that = userDim.app_info
+      Seq(
+        ValueHolder(c("app_id"), that.flatMap(_.app_id)),
+        ValueHolder(c("app_instance_id"), that.flatMap(_.app_instance_id)),
+        ValueHolder(c("app_platform"), that.flatMap(_.app_platform)),
+        ValueHolder(c("app_store"), that.flatMap(_.app_store)),
+        ValueHolder(c("app_version"), that.flatMap(_.app_version))
+      )
+    }
+
+    val bundleInfo = {
+      val c    = Columns.find("bundle_info", _: String)
+      val that = userDim.bundle_info
+      Seq(
+        ValueHolder(c("bundle_sequence_id"), that.flatMap(_.bundle_sequence_id)),
+        ValueHolder(c("server_timestamp_offset_micros"), that.flatMap(_.server_timestamp_offset_micros))
+      )
+    }
+
+    val geoInfo = {
+      val c    = Columns.find("geo_info", _: String)
+      val that = userDim.geo_info
+      Seq(
+        ValueHolder(c("city"), that.flatMap(_.city)),
+        ValueHolder(c("continent"), that.flatMap(_.continent)),
+        ValueHolder(c("country"), that.flatMap(_.country)),
+        ValueHolder(c("region"), that.flatMap(_.region))
+      )
+    }
+
+    val deviceInfo = {
+      val c    = Columns.find("device_info", _: String)
+      val that = userDim.device_info
+      Seq(
+        ValueHolder(c("device_category"), that.flatMap(_.device_category)),
+        ValueHolder(c("device_id"), that.flatMap(_.device_id)),
+        ValueHolder(c("device_model"), that.flatMap(_.device_model)),
+        ValueHolder(c("device_time_zone_offset_seconds"), that.flatMap(_.device_time_zone_offset_seconds)),
+        ValueHolder(c("limited_ad_tracking"), that.flatMap(_.limited_ad_tracking)),
+        ValueHolder(c("mobile_brand_name"), that.flatMap(_.mobile_brand_name)),
+        ValueHolder(c("mobile_marketing_name"), that.flatMap(_.mobile_marketing_name)),
+        ValueHolder(c("mobile_model_name"), that.flatMap(_.mobile_model_name)),
+        ValueHolder(c("platform_version"), that.flatMap(_.platform_version)),
+        ValueHolder(c("resettable_device_id"), that.flatMap(_.resettable_device_id)),
+        ValueHolder(c("user_default_language"), that.flatMap(_.user_default_language))
+      )
+    }
+
+    val trafficSource = {
+      val c    = Columns.find("traffic_source", _: String)
+      val that = userDim.traffic_source
+      Seq(
+        ValueHolder(c("user_acquired_campaign"), that.flatMap(_.user_acquired_campaign)),
+        ValueHolder(c("user_acquired_medium"), that.flatMap(_.user_acquired_medium)),
+        ValueHolder(c("user_acquired_source"), that.flatMap(_.user_acquired_source))
+      )
+    }
+
+    val ltvInfo = {
+      val c    = Columns.find("ltv_info", _: String)
+      val that = userDim.ltv_info
+      Seq(ValueHolder(c("currency"), that.flatMap(_.currency)), ValueHolder(c("revenue"), that.flatMap(_.revenue)))
+    }
+
+    userFields ++ appInfo ++ bundleInfo ++ geoInfo ++ deviceInfo ++ trafficSource ++ ltvInfo
+  }
+}

data/src/main/scala/org/embulk/parser/firebase_avro/PluginTask.scala

@@ -0,0 +1,5 @@
+package org.embulk.parser.firebase_avro
+
+import org.embulk.config.Task
+
+trait PluginTask extends Task

data/src/main/scala/org/embulk/parser/firebase_avro/ValueHolder.scala

@@ -0,0 +1,5 @@
+package org.embulk.parser.firebase_avro
+
+import org.embulk.spi.Column
+
+case class ValueHolder[+A](column: Column, value: Option[A] = None)

data/src/main/scala/org/embulk/parser/firebase_avro/column/Column.scala

@@ -0,0 +1,61 @@
+package org.embulk.parser.firebase_avro.column
+
+import org.embulk.spi.{Column => EmbulkColumn}
+import org.embulk.spi.`type`.Types
+
+import scala.language.implicitConversions
+
+object Column {
+
+  import scala.reflect.runtime.universe._
+
+  def getGeneral[A](index: Int)(implicit tpe: TypeTag[A]): Seq[Column] = {
+    var counter = index
+    typeOf[A].members.filter(!_.isMethod).flatMap { implicit member =>
+      val column = member.typeSignature match {
+        case t if t =:= typeOf[Option[String]] =>
+          Some(factory(counter, Types.STRING))
+        case t if t =:= typeOf[Option[Int]] =>
+          Some(factory(counter, Types.LONG))
+        case t if t =:= typeOf[Option[Long]] =>
+          Some(factory(counter, Types.LONG))
+        case t if t =:= typeOf[Option[Double]] =>
+          Some(factory(counter, Types.DOUBLE))
+        case t if t =:= typeOf[Option[Boolean]] =>
+          Some(factory(counter, Types.BOOLEAN))
+        case t if t =:= typeOf[String] =>
+          Some(factory(counter, Types.STRING))
+        case t if t =:= typeOf[Int] =>
+          Some(factory(counter, Types.LONG))
+        case t if t =:= typeOf[Long] =>
+          Some(factory(counter, Types.LONG))
+        case t if t =:= typeOf[Double] =>
+          Some(factory(counter, Types.DOUBLE))
+        case t if t =:= typeOf[Boolean] =>
+          Some(factory(counter, Types.BOOLEAN))
+        case _ =>
+          counter = counter - 1
+          None
+      }
+      counter = counter + 1
+      column
+    }
+  }.toList
+
+  private def factory[A](counter: Int, tpe: org.embulk.spi.`type`.Type)(implicit tt: TypeTag[A],
+                                                                        symbol: Symbol): Column = {
+    val fullpath =
+      (tt.tpe.etaExpand.toString.toLowerCase + "." + symbol.name.toString).trim
+    Column(fullpath, new EmbulkColumn(counter, name, tpe))
+  }
+
+  def name[A](implicit tpe: TypeTag[A], symbol: Symbol): String = {
+    val fullpath   = tpe.tpe.etaExpand.toString
+    val lessLength = "org.embulk.parser.firebase_avro.define.root.".length
+    (fullpath
+      .substring(lessLength)
+      .toLowerCase + "." + symbol.name.toString).trim
+  }
+}
+
+case class Column(fullPath: String, embulkColumn: EmbulkColumn)

data/src/main/scala/org/embulk/parser/firebase_avro/column/Columns.scala

@@ -0,0 +1,47 @@
+package org.embulk.parser.firebase_avro.column
+
+import org.embulk.spi.{Column => EmbulkColumn}
+
+object Columns {
+  val instance = Columns()
+  private val map: Map[String, EmbulkColumn] = instance.map { c =>
+    c.fullPath -> c.embulkColumn
+  }.toMap
+
+  private var cache: Map[String, EmbulkColumn] = Map.empty
+
+  def find(className: String, fieldName: String): EmbulkColumn = {
+    cache.get(s"$className.$fieldName") match {
+      case Some(v) => v
+      case None =>
+        val detect = map.flatMap {
+          case (name, column) =>
+            if (name.contains(s"$className.$fieldName")) Some(column)
+            else None
+        }
+        if (detect.size == 1) {
+          val result = detect.head
+          cache = cache ++ Map(s"$className.$fieldName" -> detect.head)
+          result
+        } else
+          sys.error(s"could not find column. ${className + "." + fieldName}")
+    }
+  }
+
+  def apply(): Seq[Column] = {
+    val userColumns  = UserDimension(0)
+    val eventColumns = EventDimension(userColumns.size)
+    userColumns ++ eventColumns
+  }
+
+  def generate(startIndex: Int)(s: Seq[((Int) => Seq[Column])]): Seq[Column] = {
+    var index = startIndex
+    s.foldLeft[List[Column]](Nil) {
+      case (a, b) =>
+        val additionalColumn = b(index)
+        index = index + additionalColumn.size
+        a ++ additionalColumn
+    }
+  }
+
+}

data/src/main/scala/org/embulk/parser/firebase_avro/column/EventDimension.scala

@@ -0,0 +1,19 @@
+package org.embulk.parser.firebase_avro.column
+
+import org.embulk.spi.{Column => EmbulkColumn}
+import org.embulk.spi.`type`.Types
+
+object EventDimension {
+  def apply(index: Int): Seq[Column] = {
+    Columns.generate(index) {
+      List(
+        Column
+          .getGeneral[org.embulk.parser.firebase_avro.define.root.Event_Dim],
+        (i: Int) =>
+          List(
+            Column("org.embulk.parser.firebase_avro.define.root.event_dim.params",
+                   new EmbulkColumn(i, "event_dim.params", Types.JSON)))
+      )
+    }
+  }
+}

data/src/main/scala/org/embulk/parser/firebase_avro/column/UserDimension.scala

@@ -0,0 +1,28 @@
+package org.embulk.parser.firebase_avro.column
+
+import org.embulk.parser.firebase_avro.define.root.user_dim._
+import org.embulk.spi.{Column => EmbulkColumn}
+import org.embulk.spi.`type`.Types
+
+object UserDimension {
+  def apply(index: Int): Seq[Column] = {
+    Columns.generate(index) {
+      List(
+        Column
+          .getGeneral[org.embulk.parser.firebase_avro.define.root.User_Dim],
+        (i: Int) =>
+          List(
+            Column("org.embulk.parser.firebase_avro.define.root.user_dim.user_properties",
+                   new EmbulkColumn(i, "user_dim.user_property", Types.JSON))),
+        Column.getGeneral[Device_Info],
+        Column.getGeneral[Geo_Info],
+        Column.getGeneral[App_Info],
+        Column.getGeneral[Traffic_Source],
+        Column.getGeneral[Bundle_Info],
+        Column.getGeneral[Ltv_Info]
+      )
+    }
+
+  }
+
+}