embulk-parser-twitter_ads_stats 0.1.0

data/lib/embulk/guess/twitter_ads_stats.rb

@@ -0,0 +1,61 @@
+module Embulk
+  module Guess
+
+    # TODO implement guess plugin to make this command work:
+    #      $ embulk guess -g "twitter_ads_stats" partial-config.yml
+    #
+    #      Depending on the file format the plugin uses, you can use choose
+    #      one of binary guess (GuessPlugin), text guess (TextGuessPlugin),
+    #      or line guess (LineGuessPlugin).
+
+    # class TwitterAdsStats < GuessPlugin
+    #   Plugin.register_guess("twitter_ads_stats", self)
+    #
+    #   def guess(config, sample_buffer)
+    #     if sample_buffer[0,2] == GZIP_HEADER
+    #       guessed = {}
+    #       guessed["type"] = "twitter_ads_stats"
+    #       guessed["property1"] = "guessed-value"
+    #       return {"parser" => guessed}
+    #     else
+    #       return {}
+    #     end
+    #   end
+    # end
+
+    # class TwitterAdsStats < TextGuessPlugin
+    #   Plugin.register_guess("twitter_ads_stats", self)
+    #
+    #   def guess_text(config, sample_text)
+    #     js = JSON.parse(sample_text) rescue nil
+    #     if js && js["mykeyword"] == "keyword"
+    #       guessed = {}
+    #       guessed["type"] = "twitter_ads_stats"
+    #       guessed["property1"] = "guessed-value"
+    #       return {"parser" => guessed}
+    #     else
+    #       return {}
+    #     end
+    #   end
+    # end
+
+    # class TwitterAdsStats < LineGuessPlugin
+    #   Plugin.register_guess("twitter_ads_stats", self)
+    #
+    #   def guess_lines(config, sample_lines)
+    #     all_line_matched = sample_lines.all? do |line|
+    #       line =~ /mypattern/
+    #     end
+    #     if all_line_matched
+    #       guessed = {}
+    #       guessed["type"] = "twitter_ads_stats"
+    #       guessed["property1"] = "guessed-value"
+    #       return {"parser" => guessed}
+    #     else
+    #       return {}
+    #     end
+    #   end
+    # end
+
+  end
+end

data/lib/embulk/parser/twitter_ads_stats.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_parser(
+  "twitter_ads_stats", "org.embulk.parser.twitter_ads_stats.TwitterAdsStatsParserPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/project/Dependencies.scala

@@ -0,0 +1,11 @@
+import sbt._
+
+object Dependencies {
+
+  val value = Seq(
+    "org.scalatest" %% "scalatest" % "3.0.1" % Test,
+    "org.embulk" % "embulk-core" % "0.8.35",
+    "org.embulk" % "embulk-core" % "0.8.35" classifier "tests",
+    "io.spray" %%  "spray-json" % "1.3.3"
+  )
+}

data/project/build.properties

@@ -0,0 +1 @@
+sbt.version=0.13.16

data/project/plugins.sbt

@@ -0,0 +1,2 @@
+addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.8.2")
+addSbtPlugin("com.lucidchart" % "sbt-scalafmt" % "1.12")

data/src/main/scala/org/embulk/parser/twitter_ads_stats/Column.scala

@@ -0,0 +1,50 @@
+package org.embulk.parser.twitter_ads_stats
+
+import java.time.LocalDate
+
+import org.embulk.spi.`type`.Types
+import org.embulk.spi.{Column => EmbulkColumn}
+
+object Column {
+  def createEmbulkColumns(metricElementNames: MetricElementNames): Seq[EmbulkColumn] = {
+    @scala.annotation.tailrec
+    def loop(
+              curIndex: Int,
+              curNames: List[String],
+              acc: Seq[EmbulkColumn]
+            ): Seq[EmbulkColumn] = {
+      curNames match {
+        case Nil => acc.reverse
+        case x :: xs =>
+          loop(
+            curIndex + 1,
+            xs,
+            new EmbulkColumn(curIndex, x, Types.JSON) +: acc
+          )
+      }
+    }
+
+
+    val baseColumns = Seq(
+      new EmbulkColumn(0, "id", Types.STRING),
+      new EmbulkColumn(1, "date", Types.STRING),
+      new EmbulkColumn(2, "segment", Types.STRING),
+      new EmbulkColumn(3, "placement", Types.STRING)
+    )
+
+    val metricsColumns = loop(
+      4,
+      metricElementNames.getSortedMetricsGroupNames,
+      Nil
+    )
+    baseColumns ++ metricsColumns
+  }
+}
+
+case class Column(
+                   id: String,
+                   date: LocalDate,
+                   segment: Option[String],
+                   placement: String,
+                   metricsGroup: Map[String, MetricsGroup]
+                 )

data/src/main/scala/org/embulk/parser/twitter_ads_stats/LoanPattern.scala

@@ -0,0 +1,19 @@
+package org.embulk.parser.twitter_ads_stats
+
+import scala.util.control.Exception.ignoring
+
+object LoanPattern {
+
+  type Closable = { def close(): Unit }
+
+  def apply[R <: Closable, A](resource: R)(f: R => A): A = {
+    try {
+      f(resource)
+    } finally {
+      ignoring(classOf[Throwable]) apply {
+        resource.close()
+      }
+    }
+  }
+
+}

data/src/main/scala/org/embulk/parser/twitter_ads_stats/MetricElementNames.scala

@@ -0,0 +1,47 @@
+package org.embulk.parser.twitter_ads_stats
+
+import org.embulk.parser.twitter_ads_stats.define.Metrics
+import spray.json.{JsObject, JsValue}
+
+/**
+  * Metricの要素名
+  *
+  *
+  * @param names key: MetricsGroup名 value: Metricの要素s
+  * (example)
+  * Map(
+  *  "engagement" -> Seq("engagements", "impressions"),
+  *  "web_conversion" -> Seq("conversion_purchases.assisted", "conversion_sign_ups.assisted")
+  * )
+  */
+case class MetricElementNames(names: Map[String, Seq[String]]) {
+
+  import MetricElementNames._
+
+  def getSortedMetricsGroupNames:List[String] = names.keys.toList.sorted
+
+  private[twitter_ads_stats] def resolveMetrics(resolveMetricTimeSeries: (List[String], Option[JsValue]) => MetricTimeSeries, json: JsObject): Metrics =
+    Metrics(
+      names.flatMap { v =>
+        v._2.map { value =>
+          val keys = splitSeparator(value)
+          (keys.mkString(separator), resolveMetricTimeSeries(keys, Some(json)))
+        }
+      }
+    )
+}
+
+object MetricElementNames {
+
+  private val separator = "_"
+
+  private def splitSeparator(name: String): List[String] = {
+    name.split("[.+]").toList
+  }
+
+  // @todo MetricElementNamesのスコープ内に閉じられるように
+  // @todo 例外処理
+  def replaceSeparator(name: String): String = {
+    name.replaceAll("[.]", separator)
+  }
+}

data/src/main/scala/org/embulk/parser/twitter_ads_stats/MetricsGroupJson.scala

@@ -0,0 +1,19 @@
+package org.embulk.parser.twitter_ads_stats
+
+import spray.json.DefaultJsonProtocol._
+import spray.json._
+
+object MetricsGroupJson {
+  implicit object MetricsGroupJsonWriter extends RootJsonWriter[MetricsGroup] {
+
+    private def toJValue(element: Option[Long]): JsValue = element match {
+      case Some(e) => e.toJson
+      case None => JsNull
+    }
+
+    override def write(obj: MetricsGroup): JsValue = {
+      val jsFields = obj.map(v => (v._1, toJValue(v._2)))
+      JsObject(jsFields)
+    }
+  }
+}

data/src/main/scala/org/embulk/parser/twitter_ads_stats/ParseException.scala

@@ -0,0 +1,27 @@
+package org.embulk.parser.twitter_ads_stats
+
+sealed abstract class ParseException(
+                                      message: String,
+                                      cause: Throwable
+                                    ) extends Throwable
+
+
+case class InvalidMetricTimeSeriesException(
+                                             message: String,
+                                             cause: Throwable
+                                           ) extends ParseException(message, cause) {
+  def this(cause: Throwable, index: Int) = {
+    this(s"Not Found index: $index", cause)
+  }
+}
+
+case class InvalidInputFileException(
+                                      message: String,
+                                      cause: Throwable
+                                    )
+  extends ParseException(message, cause) {
+
+  def this(cause: Throwable) = {
+    this(s"Input file can't parse", cause)
+  }
+}

data/src/main/scala/org/embulk/parser/twitter_ads_stats/PluginTask.scala

@@ -0,0 +1,9 @@
+package org.embulk.parser.twitter_ads_stats
+
+import org.embulk.config.{Config, ConfigDefault, Task}
+
+trait PluginTask extends Task {
+  @Config("stop_on_invalid_record")
+  @ConfigDefault("false")
+  def getStopOnInvalidRecord: Boolean
+}

data/src/main/scala/org/embulk/parser/twitter_ads_stats/TwitterAdsStatsParserPlugin.scala

@@ -0,0 +1,98 @@
+package org.embulk.parser.twitter_ads_stats
+
+import org.embulk.config._
+import org.embulk.spi._
+import org.embulk.spi.json.JsonParser
+import org.embulk.spi.util.FileInputInputStream
+import org.slf4j.Logger
+import spray.json.{pimpAny, pimpString}
+
+import scala.util.control.NonFatal
+import scala.collection.JavaConverters._
+import MetricsGroupJson._
+import org.embulk.parser.twitter_ads_stats.define.{Root, RootJson}
+
+class TwitterAdsStatsParserPlugin extends ParserPlugin {
+
+  import TwitterAdsStatsParserPlugin._
+
+  override def transaction(config: ConfigSource, control: ParserPlugin.Control): Unit = {
+    val task = config.loadConfig(classOf[PluginTask])
+    control.run(
+      task.dump(),
+      new Schema(Column.createEmbulkColumns(metricElementNames).asJava)
+    )
+  }
+
+  override def run(taskSource: TaskSource, schema: Schema, input: FileInput, output: PageOutput): Unit = {
+
+    val task = taskSource.loadTask(classOf[PluginTask])
+    val stopOnInvalidRecord = task.getStopOnInvalidRecord
+
+    LoanPattern(new PageBuilder(Exec.getBufferAllocator, schema, output)) { pb =>
+      while (input.nextFile()) {
+        (for {
+          root <- createRootFrom(input)
+          columns <- root.resolveColumns(metricElementNames)
+        } yield addRecord(pb, columns, root)) match {
+          case Right(_) =>
+          case Left(e) =>
+            if (stopOnInvalidRecord) {
+              throw new DataException(e.getMessage, e)
+            } else {
+              logger.warn(s"Skipped invalid record $e")
+            }
+        }
+      }
+      pb.finish()
+    }
+  }
+
+  private def addRecord(pb: PageBuilder, columns: Seq[Column], root: Root): Unit = {
+    columns.foreach { column =>
+      Column.createEmbulkColumns(metricElementNames).foreach { embulkColumn =>
+        (column, embulkColumn.getName) match {
+          case (Column(id, _, _, _, _), "id") =>
+            pb.setString(embulkColumn, id)
+          case (Column(_, date, _, _,_), "date") =>
+            pb.setString(embulkColumn, date.toString)
+          case (Column(_, _, Some(segment),_, _), "segment") =>
+            pb.setString(embulkColumn, segment)
+          case (Column(_, _, None,_, _), "segment") =>
+            pb.setNull(embulkColumn)
+          case (Column(_, _, _,placement, _), "placement") =>
+            pb.setString(embulkColumn, placement)
+          case (Column(_, _, _, _,metricsGroup), key) =>
+            metricsGroup.get(key) match {
+              case Some(m) =>
+                pb.setJson(
+                  embulkColumn,
+                  jsonParser.parse(m.toJson.compactPrint)
+                )
+              case None =>
+                pb.setNull(embulkColumn)
+
+            }
+          case _ => throw new RuntimeException
+        }
+      }
+      pb.addRecord()
+    }
+  }
+
+  private def createRootFrom(input: FileInput): Either[ParseException, Root] = {
+    val stream = new FileInputInputStream(input)
+    try {
+      val jsValue = scala.io.Source.fromInputStream(stream).mkString.parseJson
+      val root = new RootJson(metricElementNames).RootReader.read(jsValue)
+      Right(root)
+    } catch {
+      case NonFatal(e) => Left(new InvalidInputFileException(e))
+    }
+  }
+}
+
+object TwitterAdsStatsParserPlugin {
+  val logger: Logger = Exec.getLogger(classOf[TwitterAdsStatsParserPlugin])
+  val jsonParser = new JsonParser
+}

data/src/main/scala/org/embulk/parser/twitter_ads_stats/define/Data.scala

@@ -0,0 +1,19 @@
+package org.embulk.parser.twitter_ads_stats.define
+
+import org.embulk.parser.twitter_ads_stats.{Column, MetricElementNames, ParseException}
+
+case class Data(id: String, id_data: Seq[IDData]) {
+  private[define] def resolveColumns(metricElementNames: MetricElementNames, request: Request): Either[ParseException, Seq[Column]] = {
+    id_data.map { idData =>
+      idData.resolveColumns(id, metricElementNames, request)
+    }.foldRight[Either[ParseException, Seq[Column]]](Right(Nil)) {
+      case (Left(e), _) => Left(e)
+      case (Right(_), Left(e)) => Left(e)
+      case (Right(seq1), Right(seq2)) => Right(seq1 ++: seq2)
+    }
+  }
+}
+
+object Data {
+  val fieldNames:Array[String] = FieldNameUtil.fieldList[Data]
+}

data/src/main/scala/org/embulk/parser/twitter_ads_stats/define/FieldNameUtil.scala

@@ -0,0 +1,9 @@
+package org.embulk.parser.twitter_ads_stats.define
+
+import scala.reflect.ClassTag
+
+object FieldNameUtil {
+  //Innerクラスは利用しないこと
+  def fieldList[T](implicit tag: ClassTag[T]): Array[String] =
+    tag.runtimeClass.getDeclaredFields.map(_.getName)
+}

data/src/main/scala/org/embulk/parser/twitter_ads_stats/define/IDData.scala

@@ -0,0 +1,36 @@
+package org.embulk.parser.twitter_ads_stats.define
+
+import java.time.LocalDate
+
+import org.embulk.parser.twitter_ads_stats._
+
+case class IDData(metrics: Metrics, segment: Option[String]) {
+  private def resolveColumn(
+                             id: String,
+                             metricElementNames: MetricElementNames,
+                             date: (LocalDate, Int),
+                             placement: String
+                           ): Either[ParseException, Column] = {
+    metricElementNames.names.map { name =>
+      (name._1, metrics.findMetricsGroup(date._2, name._2))
+    }.foldRight[Either[ParseException, Map[String, MetricsGroup]]](Right(Map.empty)) {
+      case ((_, Left(e)), _) => Left(e)
+      case ((_, Right(_)), Left(e)) => Left(e)
+      case ((s, Right(r)), Right(acc)) => Right(acc + (s -> r))
+    }.map(Column(id, date._1, segment, placement, _))
+  }
+
+  private[define] def resolveColumns(id: String, metricElementNames: MetricElementNames, request: Request): Either[ParseException, Seq[Column]] = {
+    request.params.targetDates.zipWithIndex.map { date =>
+      resolveColumn(id, metricElementNames, date, request.params.placement)
+    }.foldRight[Either[ParseException, Seq[Column]]](Right(Nil)) {
+      case (Left(e), _) => Left(e)
+      case (Right(_), Left(e)) => Left(e)
+      case (Right(a), Right(seq)) => Right(a +: seq)
+    }
+  }
+}
+
+object IDData {
+  val fieldNames: Array[String] = FieldNameUtil.fieldList[IDData]
+}

data/src/main/scala/org/embulk/parser/twitter_ads_stats/define/Metrics.scala

@@ -0,0 +1,23 @@
+package org.embulk.parser.twitter_ads_stats.define
+
+import org.embulk.parser.twitter_ads_stats._
+
+import scala.util.control.NonFatal
+
+case class Metrics(map: Map[String, MetricTimeSeries]) {
+
+  private[define] def findMetricsGroup(index: Int, metricNames: Seq[String]): Either[ParseException, MetricsGroup] = {
+    try {
+      val metricsGroup = metricNames.map { name =>
+        val n = MetricElementNames.replaceSeparator(name)
+        (
+          n,
+          map.get(n).flatten.map(v => v(index))
+        )
+      }.toMap
+      Right(metricsGroup)
+    } catch {
+      case NonFatal(e) => Left(new InvalidMetricTimeSeriesException(e, index))
+    }
+  }
+}