embulk-parser-firebase_avro 0.1.0

data/src/main/scala/org/embulk/parser/firebase_avro/define/Root.scala

@@ -0,0 +1,5 @@
+package org.embulk.parser.firebase_avro.define
+
+import org.embulk.parser.firebase_avro.define.root._
+
+case class Root(user_dim: Option[User_Dim], event_dim: List[Event_Dim])

data/src/main/scala/org/embulk/parser/firebase_avro/define/root/Event_Dim.scala

@@ -0,0 +1,8 @@
+package org.embulk.parser.firebase_avro.define.root
+
+case class Event_Dim(date: Option[String],
+                     name: Option[String],
+                     params: List[event_dim.Params],
+                     timestamp_micros: Option[Long],
+                     previous_timestamp_micros: Option[Long],
+                     value_in_usd: Option[Double])

data/src/main/scala/org/embulk/parser/firebase_avro/define/root/User_Dim.scala

@@ -0,0 +1,11 @@
+package org.embulk.parser.firebase_avro.define.root
+
+case class User_Dim(user_id: Option[String],
+                    first_open_timestamp_micros: Option[Long],
+                    user_properties: List[user_dim.User_Properties],
+                    device_info: Option[user_dim.Device_Info],
+                    geo_info: Option[user_dim.Geo_Info],
+                    app_info: Option[user_dim.App_Info],
+                    traffic_source: Option[user_dim.Traffic_Source],
+                    bundle_info: Option[user_dim.Bundle_Info],
+                    ltv_info: Option[user_dim.Ltv_Info])

data/src/main/scala/org/embulk/parser/firebase_avro/define/root/event_dim/Params.scala

@@ -0,0 +1,3 @@
+package org.embulk.parser.firebase_avro.define.root.event_dim
+
+case class Params(key: Option[String], value: Option[params.Value])

data/src/main/scala/org/embulk/parser/firebase_avro/define/root/event_dim/params/Value.scala

@@ -0,0 +1,6 @@
+package org.embulk.parser.firebase_avro.define.root.event_dim.params
+
+case class Value(string_value: Option[String] = None,
+                 int_value: Option[Long] = None,
+                 float_value: Option[Double] = None,
+                 double_value: Option[Double] = None)

data/src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/App_Info.scala

@@ -0,0 +1,7 @@
+package org.embulk.parser.firebase_avro.define.root.user_dim
+
+case class App_Info(app_version: Option[String],
+                    app_instance_id: Option[String],
+                    app_store: Option[String],
+                    app_platform: Option[String],
+                    app_id: Option[String])

data/src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/Bundle_Info.scala

@@ -0,0 +1,3 @@
+package org.embulk.parser.firebase_avro.define.root.user_dim
+
+case class Bundle_Info(bundle_sequence_id: Option[Long], server_timestamp_offset_micros: Option[Long])

data/src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/Device_Info.scala

@@ -0,0 +1,13 @@
+package org.embulk.parser.firebase_avro.define.root.user_dim
+
+case class Device_Info(device_category: Option[String],
+                       mobile_brand_name: Option[String],
+                       mobile_model_name: Option[String],
+                       mobile_marketing_name: Option[String],
+                       device_model: Option[String],
+                       platform_version: Option[String],
+                       device_id: Option[String],
+                       resettable_device_id: Option[String],
+                       user_default_language: Option[String],
+                       device_time_zone_offset_seconds: Option[Long],
+                       limited_ad_tracking: Option[Boolean])

data/src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/Geo_Info.scala

@@ -0,0 +1,3 @@
+package org.embulk.parser.firebase_avro.define.root.user_dim
+
+case class Geo_Info(continent: Option[String], country: Option[String], region: Option[String], city: Option[String])

data/src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/Ltv_Info.scala

@@ -0,0 +1,3 @@
+package org.embulk.parser.firebase_avro.define.root.user_dim
+
+case class Ltv_Info(revenue: Option[Double], currency: Option[String])

data/src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/Traffic_Source.scala

@@ -0,0 +1,5 @@
+package org.embulk.parser.firebase_avro.define.root.user_dim
+
+case class Traffic_Source(user_acquired_campaign: Option[String],
+                          user_acquired_source: Option[String],
+                          user_acquired_medium: Option[String])

data/src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/User_Properties.scala

@@ -0,0 +1,3 @@
+package org.embulk.parser.firebase_avro.define.root.user_dim
+
+case class User_Properties(key: Option[String], value: Option[user_properties.Value])

data/src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/user_properties/Value.scala

@@ -0,0 +1,5 @@
+package org.embulk.parser.firebase_avro.define.root.user_dim.user_properties
+
+case class Value(value: Option[org.embulk.parser.firebase_avro.define.root.user_dim.user_properties.value.Value],
+                 set_timestamp_usec: Option[Long],
+                 index: Option[Long])

data/src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/user_properties/value/Value.scala

@@ -0,0 +1,6 @@
+package org.embulk.parser.firebase_avro.define.root.user_dim.user_properties.value
+
+case class Value(string_value: Option[String] = None,
+                 int_value: Option[Long] = None,
+                 float_value: Option[Double] = None,
+                 double_value: Option[Double] = None)

data/src/main/scala/org/embulk/parser/firebase_avro/json/CustomEncoder.scala

@@ -0,0 +1,21 @@
+package org.embulk.parser.firebase_avro.json
+
+import io.circe.{Encoder, Json}
+
+object CustomEncoder {
+  implicit val mapEncoder: Encoder[Map[String, Any]] =
+    new Encoder[Map[String, Any]] {
+      final def apply(a: Map[String, Any]): Json = {
+        val jsonValues = a.keys.map { key =>
+          a(key) match {
+            case v: String => (key, Json.fromString(v))
+            case v: Double =>
+              (key, Json.fromDouble(v).getOrElse(Json.fromString(v.toString)))
+            case v: Long => (key, Json.fromLong(v))
+            case _       => (key, Json.Null)
+          }
+        }
+        Json.fromFields(jsonValues)
+      }
+    }
+}

data/src/main/scala/org/embulk/parser/firebase_avro/json/event_dim/EventParmsJsonSerializer.scala

@@ -0,0 +1,29 @@
+package org.embulk.parser.firebase_avro.json.event_dim
+
+import org.embulk.parser.firebase_avro.define.root.event_dim.Params
+
+import scala.collection.mutable
+import io.circe._
+import io.circe.syntax._
+import org.embulk.parser.firebase_avro.json.CustomEncoder.mapEncoder
+
+object EventParmsJsonSerializer {
+  def apply(params: List[Params]): Option[Json] = {
+    val map = mutable.Map[String, Any]()
+    params.foreach { p =>
+      val value: Option[Any] = p.value.flatMap { v =>
+        v.float_value
+          .orElse(v.double_value)
+          .orElse(v.int_value)
+          .orElse(v.string_value)
+      }
+      for {
+        key   <- p.key
+        value <- value
+      } yield map.put(key, value)
+    }
+    if (map.nonEmpty) {
+      Some(map.toMap.asJson)
+    } else None
+  }
+}

data/src/main/scala/org/embulk/parser/firebase_avro/json/user_dim/UserPropertiesJsonSerializer.scala

@@ -0,0 +1,34 @@
+package org.embulk.parser.firebase_avro.json.user_dim
+
+import io.circe.Json
+import io.circe.syntax._
+import org.embulk.parser.firebase_avro.define.root.user_dim.User_Properties
+import org.embulk.parser.firebase_avro.json.CustomEncoder.mapEncoder
+
+import scala.collection.mutable
+
+object UserPropertiesJsonSerializer {
+  def apply(properties: List[User_Properties]): Option[Json] = {
+    val map = mutable.Map[String, Any]()
+    properties.foreach { p =>
+      val value: Option[Any] = for {
+        v  <- p.value
+        iv <- v.value
+      } yield {
+        v.set_timestamp_usec.foreach(a => map.put("set_timestamp_usec", a))
+        v.index.foreach(a => map.put("index", a))
+        iv.float_value
+          .orElse(iv.double_value)
+          .orElse(iv.int_value)
+          .orElse(iv.string_value)
+      }
+      for {
+        key   <- p.key
+        value <- value
+      } yield map.put(key, value)
+    }
+    if (map.nonEmpty) {
+      Some(map.toMap.asJson)
+    } else None
+  }
+}

data/src/test/scala/org/embulk/parser/firebase_avro/Implicitly.scala

@@ -0,0 +1,9 @@
+package org.embulk.parser.firebase_avro
+
+import org.embulk.parser.firebase_avro.define.Root
+import org.scalacheck.Arbitrary
+import org.scalacheck.Shapeless._
+
+object Implicitly {
+  implicitly[Arbitrary[Root]]
+}

data/src/test/scala/org/embulk/parser/firebase_avro/ParserTest.scala

@@ -0,0 +1,22 @@
+package org.embulk.parser.firebase_avro
+
+import org.embulk.parser.firebase_avro.define.Root
+import org.scalacheck.Shapeless._
+import org.scalatest.{MustMatchers, PropSpec}
+import org.scalatest.prop.PropertyChecks
+
+class ParserTest extends PropSpec with PropertyChecks with MustMatchers {
+  property("could be parsing.") {
+    forAll(minSize(1000)) { (root: Root) =>
+      root.user_dim match {
+        case Some(v) =>
+          // could be parse.
+          val result = Parser(root)
+        case None =>
+          intercept[RuntimeException] {
+            Parser(root)
+          }
+      }
+    }
+  }
+}

data/src/test/scala/org/embulk/parser/firebase_avro/column/ColumnsTest.scala

@@ -0,0 +1,18 @@
+package org.embulk.parser.firebase_avro.column
+
+import org.scalatest._
+import Matchers._
+
+class ColumnsTest extends FlatSpec with MustMatchers {
+
+  "columuns" should "be indexing" in {
+    val columns = Columns()
+    val summery = columns.indices.sum
+    columns.map(_.embulkColumn).map(_.getIndex).sum should be (summery)
+  }
+
+  "columns" should "be finding" in {
+    Columns.find("user_dim" , "first_open_timestamp_micros")
+  }
+
+}

data/src/test/scala/org/embulk/parser/firebase_avro/json/event_dim/EventParmsJsonSerializerTest.scala

@@ -0,0 +1,19 @@
+package org.embulk.parser.firebase_avro.json.event_dim
+
+import org.embulk.parser.firebase_avro.define.root.event_dim.Params
+import org.embulk.parser.firebase_avro.define.root.event_dim.params.Value
+
+import org.scalatest._
+
+class EventParmsJsonSerializerTest extends FlatSpec with MustMatchers {
+
+  "parameter" should "be encoding" in {
+    val string = Params(Option("key_string") , Option(Value(string_value = Some("abc"))))
+    val int = Params(Option("key_int") , Option(Value(int_value = Some(1))))
+    val double = Params(Option("key_double") , Option(Value(double_value = Some(10D))))
+    val float = Params(Option("key_float") , Option(Value(float_value = Some(10F))))
+    val given = List(string , int , double , float)
+    val that = EventParmsJsonSerializer(given).map(_.noSpaces)
+    that mustBe Some("{\"key_float\":10.0,\"key_int\":1,\"key_string\":\"abc\",\"key_double\":10.0}")
+  }
+}

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification
+name: embulk-parser-firebase_avro
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- smdmts
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2017-06-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Parses Firebase Avro files read by other file input plugins.
+email:
+- smdmts@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .circleci/config.yml
+- .gitignore
+- .scalafmt.conf
+- LICENSE.txt
+- README.md
+- build.gradle
+- build.sbt
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/parser/firebase_avro.rb
+- project/build.properties
+- project/plugins.sbt
+- src/main/scala/org/embulk/parser/firebase_avro/FirebaseAvroParserPlugin.scala
+- src/main/scala/org/embulk/parser/firebase_avro/LoanPattern.scala
+- src/main/scala/org/embulk/parser/firebase_avro/Parser.scala
+- src/main/scala/org/embulk/parser/firebase_avro/PluginTask.scala
+- src/main/scala/org/embulk/parser/firebase_avro/ValueHolder.scala
+- src/main/scala/org/embulk/parser/firebase_avro/column/Column.scala
+- src/main/scala/org/embulk/parser/firebase_avro/column/Columns.scala
+- src/main/scala/org/embulk/parser/firebase_avro/column/EventDimension.scala
+- src/main/scala/org/embulk/parser/firebase_avro/column/UserDimension.scala
+- src/main/scala/org/embulk/parser/firebase_avro/define/Root.scala
+- src/main/scala/org/embulk/parser/firebase_avro/define/root/Event_Dim.scala
+- src/main/scala/org/embulk/parser/firebase_avro/define/root/User_Dim.scala
+- src/main/scala/org/embulk/parser/firebase_avro/define/root/event_dim/Params.scala
+- src/main/scala/org/embulk/parser/firebase_avro/define/root/event_dim/params/Value.scala
+- src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/App_Info.scala
+- src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/Bundle_Info.scala
+- src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/Device_Info.scala
+- src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/Geo_Info.scala
+- src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/Ltv_Info.scala
+- src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/Traffic_Source.scala
+- src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/User_Properties.scala
+- src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/user_properties/Value.scala
+- src/main/scala/org/embulk/parser/firebase_avro/define/root/user_dim/user_properties/value/Value.scala
+- src/main/scala/org/embulk/parser/firebase_avro/json/CustomEncoder.scala
+- src/main/scala/org/embulk/parser/firebase_avro/json/event_dim/EventParmsJsonSerializer.scala
+- src/main/scala/org/embulk/parser/firebase_avro/json/user_dim/UserPropertiesJsonSerializer.scala
+- src/test/scala/org/embulk/parser/firebase_avro/Implicitly.scala
+- src/test/scala/org/embulk/parser/firebase_avro/ParserTest.scala
+- src/test/scala/org/embulk/parser/firebase_avro/column/ColumnsTest.scala
+- src/test/scala/org/embulk/parser/firebase_avro/json/event_dim/EventParmsJsonSerializerTest.scala
+- classpath/avro-1.8.1.jar
+- classpath/avro4s-core_2.11-1.6.4.jar
+- classpath/avro4s-macros_2.11-1.6.4.jar
+- classpath/cats-core_2.11-0.9.0.jar
+- classpath/cats-kernel_2.11-0.9.0.jar
+- classpath/cats-macros_2.11-0.9.0.jar
+- classpath/circe-core_2.11-0.8.0.jar
+- classpath/circe-generic_2.11-0.8.0.jar
+- classpath/circe-numbers_2.11-0.8.0.jar
+- classpath/commons-compress-1.8.1.jar
+- classpath/embulk-parser-firebase_avro-0.1.0.jar
+- classpath/jackson-core-asl-1.9.13.jar
+- classpath/jackson-mapper-asl-1.9.13.jar
+- classpath/machinist_2.11-0.6.1.jar
+- classpath/macro-compat_2.11-1.1.1.jar
+- classpath/paranamer-2.7.jar
+- classpath/scala-library-2.11.11.jar
+- classpath/scala-reflect-2.11.8.jar
+- classpath/shapeless_2.11-2.3.2.jar
+- classpath/simulacrum_2.11-0.10.0.jar
+- classpath/snappy-java-1.1.1.3.jar
+- classpath/xz-1.5.jar
+homepage: https://github.com/smdmts/embulk-parser-firebase_avro
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Firebase Avro parser plugin for Embulk
+test_files: []