embulk-output-fluentd 0.1.0

data/src/main/scala/org/embulk/output/fluentd/sender/SenderBuilder.scala

@@ -0,0 +1,40 @@
+package org.embulk.output.fluentd.sender
+
+import java.time.Instant
+
+import akka.actor.ActorSystem
+import org.embulk.output.fluentd.PluginTask
+import org.embulk.spi.Exec
+import wvlet.airframe.{Design, newDesign}
+import com.typesafe.config.ConfigFactory
+
+object SenderBuilder {
+
+  def apply(task: PluginTask): Design = {
+    implicit val logger = Exec.getLogger(classOf[Sender])
+    implicit val system = ActorSystem("fluentd-sender",
+                                      ConfigFactory.load(this.getClass.getClassLoader, "application.conf"),
+                                      this.getClass.getClassLoader)
+    val timeKeyOpt = if (task.getTimeKey.isPresent) {
+      Some(task.getTimeKey.get())
+    } else None
+
+    newDesign
+      .bind[SenderFlow]
+      .toInstance(SenderFlowImpl(task.getTag, Instant.now().getEpochSecond, timeKeyOpt))
+      .bind[ActorManager]
+      .toInstance(ActorManagerImpl())
+      .bind[Sender]
+      .toProvider { (senderFlow: SenderFlow, actorManager: ActorManager) =>
+        SenderImpl(task.getHost,
+                   task.getPort,
+                   task.getRequestGroupingSize,
+                   task.getAsyncSize,
+                   senderFlow,
+                   actorManager,
+                   task.getRequestPerSeconds,
+                   retryCount = 10)
+      }
+  }
+
+}

data/src/main/scala/org/embulk/output/fluentd/sender/SenderFlow.scala

@@ -0,0 +1,36 @@
+package org.embulk.output.fluentd.sender
+
+import akka.NotUsed
+import akka.actor.ActorSystem
+import akka.stream.scaladsl.{Flow, Tcp}
+import akka.stream.scaladsl.Tcp.OutgoingConnection
+import akka.util.ByteString
+import org.velvia.MsgPack
+
+import scala.concurrent.Future
+
+trait SenderFlow {
+  val msgPackFlow: Flow[Seq[Seq[Map[String, AnyRef]]], (Int, ByteString), NotUsed]
+  def tcpConnectionFlow(host: String, port: Int)(
+      implicit s: ActorSystem): Flow[ByteString, ByteString, Future[OutgoingConnection]]
+}
+
+case class SenderFlowImpl private[sender] (tag: String, unixtime: Long, timeKeyOpt: Option[String])
+    extends SenderFlow {
+  override val msgPackFlow: Flow[Seq[Seq[Map[String, AnyRef]]], (Int, ByteString), NotUsed] =
+    Flow[Seq[Seq[Map[String, AnyRef]]]].map { value =>
+      val packing = value.flatten.map { v =>
+        val eventTime = for {
+          timeKey   <- timeKeyOpt
+          timeValue <- v.get(timeKey)
+        } yield timeValue.toString.toLong
+        val logTime = eventTime.getOrElse(unixtime)
+        Seq(logTime, v)
+      }
+      (packing.size, ByteString(MsgPack.pack(Seq(tag, packing))))
+    }
+  override def tcpConnectionFlow(host: String, port: Int)(
+      implicit s: ActorSystem): Flow[ByteString, ByteString, Future[OutgoingConnection]] =
+    Tcp().outgoingConnection(host, port)
+
+}

data/src/main/scala/org/embulk/output/fluentd/sender/SuperVisor.scala

@@ -0,0 +1,46 @@
+package org.embulk.output.fluentd.sender
+
+import akka.actor._
+import org.slf4j.Logger
+
+class SuperVisor extends Actor {
+  var complete = 0
+  var failed   = 0
+  var retried  = 0
+  var counter  = 0
+  var closed   = false
+  override def receive: Receive = {
+    case Record(v) =>
+      counter = counter + v
+    case Complete(v) =>
+      complete = complete + v
+    case Failed(v)  => failed = failed + v
+    case Retried(v) => retried = retried + v
+    case GetStatus =>
+      if (failed == 0) {
+        sender() ! Result(counter, complete, failed, retried)
+      } else {
+        sender() ! Stop(counter, complete, failed, retried)
+      }
+    case Close =>
+      val result = ClosedStatus(closed)
+      if (!closed) {
+        closed = true
+      }
+      sender() ! result
+    case LogStatus(logger) =>
+      logger.info(
+        s"$counter was queued and $complete records was completed. $failed records was failed and retried $retried records.")
+  }
+}
+
+case class Result(record: Int, complete: Int, failed: Int, retried: Int)
+case object GetStatus
+case class Stop(record: Int, complete: Int, failed: Int, retried: Int)
+case object Close
+case class ClosedStatus(alreadyClosed: Boolean)
+case class LogStatus(logger: Logger)
+case class Record(count: Int)   extends AnyVal
+case class Complete(count: Int) extends AnyVal
+case class Failed(count: Int)   extends AnyVal
+case class Retried(count: Int)  extends AnyVal

data/src/test/scala/org/embulk/output/fluentd/TestActorManager.scala

@@ -0,0 +1,42 @@
+package org.embulk.output.fluentd
+
+import java.io.IOException
+import java.net.ServerSocket
+import java.util.concurrent.ThreadLocalRandom
+
+import akka.actor.{ActorRef, ActorSystem}
+import akka.stream.ActorMaterializer
+import akka.testkit.TestActorRef
+import org.embulk.output.fluentd.sender._
+
+import scala.concurrent.ExecutionContext
+
+case class TestActorManager(s: ActorSystem) extends ActorManager {
+  implicit val system = s
+  val port: Int       = freePort(8888, 8999)
+  val host: String    = "127.0.0.1"
+
+  def freePort(from: Int, to: Int): Int = {
+    var port = from
+    while (true) {
+      if (isLocalPortFree(port)) return port
+      else port = ThreadLocalRandom.current.nextInt(from, to)
+    }
+    port
+  }
+
+  private def isLocalPortFree(port: Int) =
+    try {
+      new ServerSocket(port).close()
+      true
+    } catch {
+      case _: IOException =>
+        false
+    }
+
+  val testActorRef = TestActorRef(new SuperVisor)
+
+  override val supervisor: ActorRef                     = testActorRef
+  override implicit val materializer: ActorMaterializer = ActorMaterializer()
+  override implicit val dispatcher: ExecutionContext    = ExecutionContext.global
+}

data/src/test/scala/org/embulk/output/fluentd/sender/SenderFlowImplTest.scala

@@ -0,0 +1,87 @@
+package org.embulk.output.fluentd.sender
+
+import akka.actor.ActorSystem
+import akka.stream.{ActorMaterializer, ActorMaterializerSettings}
+import akka.stream.scaladsl.Keep
+import akka.stream.testkit.scaladsl.{TestSink, TestSource}
+import akka.testkit.{ImplicitSender, TestKit}
+import org.velvia.MsgPackUtils._
+import org.embulk.output.fluentd.TestActorManager
+import org.scalatest._
+
+class SenderFlowImplTest
+    extends TestKit(ActorSystem("MySpec"))
+    with ImplicitSender
+    with FlatSpecLike
+    with Matchers
+    with BeforeAndAfterAll {
+
+  override def afterAll {
+    TestKit.shutdownActorSystem(system)
+  }
+
+  "msgPackFlow" should "converting Success" in {
+    val target                = SenderFlowImpl("dummy", 123, None)
+    val actorManager          = TestActorManager(system)
+    implicit val materializer = ActorMaterializer(ActorMaterializerSettings(system))
+    val map                   = Map[String, AnyRef]("a" -> Int.box(1), "b" -> "e")
+    val request               = Seq(Seq(map), Seq(map))
+
+    val (pub, sub) = TestSource
+      .probe[Seq[Seq[Map[String, AnyRef]]]]
+      .via(target.msgPackFlow)
+      .toMat(TestSink.probe[(Int, akka.util.ByteString)])(Keep.both)
+      .run()
+
+    sub.request(n = 1)
+    pub.sendNext(request)
+
+    val (recordSize, result) = sub.expectNext()
+
+    val that    = unpackSeq(result.toByteBuffer.array())
+    val records = that(1).asInstanceOf[Vector[_]]
+    recordSize should be(2) // record size
+    that.head should be("dummy")
+    records.size should be(2) // internal record size
+    val one = records(0).asInstanceOf[Vector[_]]
+    one(0) should be(123) // time
+    one(1).asInstanceOf[Map[_, _]] should be(map)
+
+    val two = records(1).asInstanceOf[Vector[_]]
+    two(0) should be(123) // time
+    two(1).asInstanceOf[Map[_, _]] should be(map)
+  }
+
+  "msgPackFlow" should "converting Success with time key" in {
+    val target                = SenderFlowImpl("dummy", 123, Option("time"))
+    val actorManager          = TestActorManager(system)
+    implicit val materializer = ActorMaterializer(ActorMaterializerSettings(system))
+    val map                   = Map[String, AnyRef]("a" -> Int.box(1), "b" -> "e", "time" -> Long.box(12345))
+    val request               = Seq(Seq(map), Seq(map))
+
+    val (pub, sub) = TestSource
+      .probe[Seq[Seq[Map[String, AnyRef]]]]
+      .via(target.msgPackFlow)
+      .toMat(TestSink.probe[(Int, akka.util.ByteString)])(Keep.both)
+      .run()
+
+    sub.request(n = 1)
+    pub.sendNext(request)
+
+    val (recordSize, result) = sub.expectNext()
+
+    val that    = unpackSeq(result.toByteBuffer.array())
+    val records = that(1).asInstanceOf[Vector[_]]
+    recordSize should be(2) // record size
+    that.head should be("dummy")
+    records.size should be(2) // internal record size
+    val one = records(0).asInstanceOf[Vector[_]]
+    one(0) should be(12345) // time
+    one(1).asInstanceOf[Map[_, _]] should be(map)
+
+    val two = records(1).asInstanceOf[Vector[_]]
+    two(0) should be(12345) // time
+    two(1).asInstanceOf[Map[_, _]] should be(map)
+  }
+
+}

data/src/test/scala/org/embulk/output/fluentd/sender/SenderImplTest.scala

@@ -0,0 +1,119 @@
+package org.embulk.output.fluentd.sender
+
+import java.util.concurrent.TimeUnit
+
+import akka.actor.ActorSystem
+import akka.stream._
+import akka.stream.scaladsl._
+import akka.util.{ByteString, Timeout}
+import org.scalatest.{BeforeAndAfterAll, FlatSpecLike, Matchers}
+import org.embulk.output.fluentd.TestActorManager
+import org.slf4j.helpers.NOPLogger
+
+import scala.concurrent.duration._
+
+class SenderImplTest extends FlatSpecLike with Matchers with BeforeAndAfterAll {
+
+  implicit val logger  = NOPLogger.NOP_LOGGER
+  implicit val timeout = Timeout(5.seconds)
+
+  "Sending Success fast server" should "receive dummy server" in {
+
+    val system       = ActorSystem("MySpec")
+    val actorManager = TestActorManager(system)
+    bootDummyServer(system, "127.0.0.1", actorManager.port, Duration.create(0, TimeUnit.SECONDS))
+    Thread.sleep(100) // wait for server boot.
+    val sender = SenderImpl(
+      "localhost",
+      port = actorManager.port,
+      groupedSize = 1,
+      asyncSize = 1,
+      SenderFlowImpl("tag", 0, None),
+      actorManager
+    )
+
+    (1 to 100).foreach { _ =>
+      sender(recode)
+    }
+
+    sender.waitForComplete()
+
+    actorManager.testActorRef.underlyingActor.counter should be(100)
+    actorManager.testActorRef.underlyingActor.complete should be(100)
+    actorManager.testActorRef.underlyingActor.retried should be(0)
+
+    actorManager.system.terminate()
+
+  }
+
+  "Sending Success slow server" should "receive dummy server" in {
+
+    val system       = ActorSystem("MySpec")
+    val actorManager = TestActorManager(system)
+    bootDummyServer(system, "127.0.0.1", actorManager.port, Duration.create(3, TimeUnit.SECONDS))
+    Thread.sleep(100) // wait for server boot.
+    val sender = SenderImpl(
+      "localhost",
+      port = actorManager.port,
+      groupedSize = 1,
+      asyncSize = 1,
+      SenderFlowImpl("tag", 0, None),
+      actorManager
+    )
+
+    (1 to 2).foreach { _ =>
+      sender(recode)
+    }
+
+    sender.waitForComplete()
+
+    actorManager.testActorRef.underlyingActor.counter should be(2)
+    actorManager.testActorRef.underlyingActor.complete should be(2)
+    actorManager.testActorRef.underlyingActor.retried should be(0)
+
+    actorManager.system.terminate()
+  }
+
+  def recode: Seq[Map[String, AnyRef]] = Seq(Map[String, AnyRef]("a" -> Int.box(1), "b" -> "c"))
+
+  def bootDummyServer(system: ActorSystem, address: String, port: Int, duration: Duration): Unit = {
+    implicit val sys          = system
+    implicit val materializer = ActorMaterializer()
+    val connections           = Tcp().bind(address, port)
+    connections runForeach { connection =>
+      val echo = Flow[ByteString]
+        .via(Framing.delimiter(ByteString("\n"), maximumFrameLength = 256, allowTruncation = true))
+        .map { v =>
+          TimeUnit.SECONDS.sleep(duration.toSeconds)
+          ByteString(v.utf8String)
+        }
+      connection.handleWith(echo)
+    }
+  }
+
+  "All Failure" should "retry count is correct" in {
+    val system       = ActorSystem("MySpec")
+    val actorManager = TestActorManager(system)
+    val sender = SenderImpl(
+      "localhost",
+      port = 9999,
+      groupedSize = 1,
+      asyncSize = 1,
+      SenderFlowImpl("tag", 0, None),
+      actorManager,
+      retryCount = 2, // 2 times
+      retryDelayIntervalSecond = 1 // 1 seconds
+    )
+
+    val recode = Seq(Map[String, AnyRef]("a" -> Int.box(1), "b" -> "c"))
+    sender(recode)
+    sender.waitForComplete()
+
+    actorManager.testActorRef.underlyingActor.counter should be(1)
+    actorManager.testActorRef.underlyingActor.retried should be(2)
+    actorManager.testActorRef.underlyingActor.complete should be(0)
+
+    actorManager.system.terminate()
+  }
+
+}

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: embulk-output-fluentd
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- smdmts
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2017-07-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Dumps records to Fluentd.
+email:
+- smdmts@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .circleci/config.yml
+- .gitignore
+- .scalafmt.conf
+- LICENSE.txt
+- README.md
+- build.gradle
+- build.sbt
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/output/fluentd.rb
+- project/build.properties
+- project/plugins.sbt
+- settings.gradle
+- src/main/resources/application.conf
+- src/main/scala/org/embulk/output/fluentd/ColumnVisitor.scala
+- src/main/scala/org/embulk/output/fluentd/FluentdOutputPlugin.scala
+- src/main/scala/org/embulk/output/fluentd/FluentdTransactionalPageOutput.scala
+- src/main/scala/org/embulk/output/fluentd/PluginTask.scala
+- src/main/scala/org/embulk/output/fluentd/sender/ActorManager.scala
+- src/main/scala/org/embulk/output/fluentd/sender/Sender.scala
+- src/main/scala/org/embulk/output/fluentd/sender/SenderBuilder.scala
+- src/main/scala/org/embulk/output/fluentd/sender/SenderFlow.scala
+- src/main/scala/org/embulk/output/fluentd/sender/SuperVisor.scala
+- src/test/scala/org/embulk/output/fluentd/TestActorManager.scala
+- src/test/scala/org/embulk/output/fluentd/sender/SenderFlowImplTest.scala
+- src/test/scala/org/embulk/output/fluentd/sender/SenderImplTest.scala
+- classpath/airframe_2.11-0.15.jar
+- classpath/akka-actor_2.11-2.5.3.jar
+- classpath/akka-slf4j_2.11-2.5.3.jar
+- classpath/akka-stream_2.11-2.5.3.jar
+- classpath/config-1.3.1.jar
+- classpath/embulk-core-0.8.25.jar
+- classpath/embulk-output-fluentd-0.1.0.jar
+- classpath/logback-core-1.1.7.jar
+- classpath/msgpack4s_2.11-0.6.0.jar
+- classpath/reactive-streams-1.0.0.jar
+- classpath/scala-java8-compat_2.11-0.7.0.jar
+- classpath/scala-library-2.11.11.jar
+- classpath/scala-parser-combinators_2.11-1.0.4.jar
+- classpath/scala-reflect-2.11.11.jar
+- classpath/slf4j-api-1.7.23.jar
+- classpath/ssl-config-core_2.11-0.2.1.jar
+- classpath/surface_2.11-0.15.jar
+- classpath/wvlet-log_2.11-1.2.3.jar
+homepage: https://github.com/smdmts/embulk-output-fluentd
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Fluentd output plugin for Embulk
+test_files: []