embulk-filter-key_in_redis 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8c07a54c906730f2de5c0a2e32a1792cc0170877
-  data.tar.gz: 3cf0b84f743814a7ee32a35cb9a084eca9db287d
+  metadata.gz: 37e4700ee70663be6a24b53e20ea86dbeea87336
+  data.tar.gz: 845fc7bc8b2d19f964483a43f9f860a9674eb752
 SHA512:
-  metadata.gz: 5b51785e0dfe0a5007cea67fd60c96aaa9c805ec605b074db141310532418847cd07f4d38d5d6745ff9a5067f7f5500f0fda004212e9f88a22d9d98018d00b00
-  data.tar.gz: d73e095906c90aef7216fcba69a5f514a7871acde3d1b94ce07918f16cdfecb5ffa05b04bfe8485baf0f3ef62c408a6a11004864fd9a24e6facb87ddc7ef5508
+  metadata.gz: 04400de1c5bc2041f28e29b4a3e2a27c407c4171e49408d70f3e02d75e2109efaf14fda42ba49cb9f2a9e94ece3184248b018fe98f43989f35cca82f234ee73c
+  data.tar.gz: 8bef1f124c5fdd3f93f7fac9e9b7a9ad52f071050e1013d07f0be320a015f89b05e05565855f7baa5ead073b60495afe27f9946c7b1f7d53af5722adc5ef65de

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.1.3"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/resources/application.conf

@@ -0,0 +1,26 @@
+akka.actor.deployment {
+  /filtering_actor {
+    dispatcher = embulk-filter_key_in_redis-dispatcher
+  }
+}
+
+embulk-filter_key_in_redis-dispatcher {
+  # Dispatcher is the name of the event-based dispatcher
+  type = Dispatcher
+  # What kind of ExecutionService to use
+  executor = "fork-join-executor"
+  # Configuration for the fork join pool
+  fork-join-executor {
+    # Min number of threads to cap factor-based parallelism number to
+    parallelism-min = 2
+    # Parallelism (threads) ... ceil(available processors * factor)
+    parallelism-factor = 2.0
+    # Max number of threads to cap factor-based parallelism number to
+    parallelism-max = 10
+  }
+
+  # Throughput defines the maximum number of messages to be
+  # processed per actor before the thread jumps to the next actor.
+  # Set to 1 for as fair as possible.
+  throughput = 100
+}

data/src/main/scala/org/embulk/filter/key_in_redis/PageOutput.scala

@@ -1,9 +1,14 @@
 package org.embulk.filter.key_in_redis
 
+import java.util.concurrent.TimeUnit
+
 import com.google.common.base.Optional
-import org.embulk.filter.key_in_redis.column._
+import org.embulk.filter.key_in_redis.actor._
+import org.embulk.filter.key_in_redis.row._
+import org.embulk.filter.key_in_redis.ToFutureExtensionOps._
+import akka.pattern.ask
+import akka.util.Timeout
 
-import scala.collection.mutable.ListBuffer
 import scala.collection.JavaConverters._
 import org.embulk.spi.time.TimestampFormatter
 import org.embulk.spi.{
@@ -20,13 +25,13 @@ case class PageOutput(task: PluginTask,
                       output: EmbulkPageOutput)
     extends EmbulkPageOutput {
   val pageBuilder = new PageBuilder(Exec.getBufferAllocator, schema, output)
+  var finished = false
   def timestampFormatter(): TimestampFormatter =
     new TimestampFormatter(task, Optional.absent())
 
   override def add(page: Page): Unit = {
     val baseReader: PageReader = new PageReader(schema)
     baseReader.setPage(page)
-    val rows = new ListBuffer[SetValueColumnVisitor]()
     while (baseReader.nextRecord()) {
       val setValueVisitor = SetValueColumnVisitor(
         baseReader,
@@ -36,19 +41,49 @@ case class PageOutput(task: PluginTask,
         task.getAppender,
         task.getMatchAsMD5)
       schema.visitColumns(setValueVisitor)
-      rows.append(setValueVisitor)
+      Actors.register ! setValueVisitor.getRow(pageBuilder)
     }
-    KeyInRedisFilterPlugin.redis.foreach { redis =>
-      val result = redis.exists(rows.map(_.getMatchKey))
-      rows.foreach { row =>
-        if (!result(row.getMatchKey)) {
-          row.addRecord(pageBuilder)
-        }
+    baseReader.close()
+  }
+
+  def counter(): Int = {
+    import scala.concurrent.ExecutionContext.Implicits.global
+    implicit val timeout = Timeout(24, TimeUnit.HOURS)
+    (Actors.register ? Counter(pageBuilder))
+      .mapTo[Int]
+      .toTask
+      .unsafePerformSync
+  }
+
+  def forceWrite(): Unit = {
+    Actors.register ! ForceWrite(pageBuilder)
+  }
+
+  override def finish(): Unit = {
+    var lastRecord = false
+    while (counter() != 0) {
+      if (!lastRecord) {
+        forceWrite()
+        lastRecord = true
       }
+      Thread.sleep(1000)
+    }
+    if (!finished) {
+      pageBuilder.finish()
+      finished = true
     }
   }
 
-  override def finish(): Unit = pageBuilder.finish()
-  override def close(): Unit = pageBuilder.close()
+  override def close(): Unit = {
+    var lastRecord = false
+    while (counter() != 0 & finished) {
+      if (!lastRecord) {
+        forceWrite()
+        lastRecord = true
+      }
+      Thread.sleep(1000)
+    }
+    pageBuilder.close()
+  }
 
-}
+}

data/src/main/scala/org/embulk/filter/key_in_redis/{redis/TaskExtensionOps.scala → TaskExtensionOps.scala}

@@ -1,8 +1,8 @@
-package org.embulk.filter.key_in_redis.redis
+package org.embulk.filter.key_in_redis
 
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Failure, Success}
-import scalaz._, Scalaz._
+import scalaz.Scalaz._
 import scalaz.concurrent._
 
 final class FutureExtensionOps[A](self: Future[A]) {

data/src/main/scala/org/embulk/filter/key_in_redis/actor/Actors.scala

@@ -0,0 +1,12 @@
+package org.embulk.filter.key_in_redis.actor
+
+import akka.actor.{ActorRef, ActorSystem, Props}
+
+object Actors {
+  implicit val actorSystem: ActorSystem = akka.actor.ActorSystem(
+    "redis-register",
+    classLoader = Some(this.getClass.getClassLoader))
+  val register: ActorRef =
+    actorSystem.actorOf(Props(classOf[Register]), "filtering_actor")
+
+}

data/src/main/scala/org/embulk/filter/key_in_redis/actor/Register.scala

@@ -0,0 +1,65 @@
+package org.embulk.filter.key_in_redis.actor
+
+import akka.actor._
+import org.embulk.filter.key_in_redis.KeyInRedisFilterPlugin
+import org.embulk.filter.key_in_redis.redis.Redis
+import org.embulk.filter.key_in_redis.row.Row
+import org.embulk.spi.PageBuilder
+
+import scala.collection.mutable
+import scala.concurrent._
+
+class Register extends Actor {
+
+  implicit val ec: ExecutionContextExecutor = context.system.dispatcher
+  var rowList: List[Row] = List.empty[Row]
+  lazy val redis: Redis =
+    KeyInRedisFilterPlugin.redis.getOrElse(sys.error("redis is undefined."))
+  val counter: mutable.Map[PageBuilder, Int] = mutable.Map[PageBuilder, Int]()
+
+  override def receive: PartialFunction[Any, Unit] = {
+    case row: Row =>
+      rowList = row :: rowList
+      counter.put(row.pageBuilder, counter.getOrElse(row.pageBuilder, 0) + 1)
+      if (rowList.size == 500) {
+        addRecords(rowList)
+        rowList = List.empty[Row]
+      }
+    case Counter(pb) =>
+      sender() ! counter.getOrElse(pb, 0)
+    case ForceWrite(pb) =>
+      val (owned, other) =
+        rowList.partition(_.pageBuilder == pb)
+      addRecords(owned)
+      rowList = other
+    case Add(row) =>
+      counter.put(row.pageBuilder, counter(row.pageBuilder) - 1)
+      row.addRecord()
+    case Ignore(row) =>
+      counter.put(row.pageBuilder, counter(row.pageBuilder) - 1)
+    case TotalCount =>
+      sender() ! counter.foldLeft[Int](0) {
+        case (total, (_, counter: Int)) =>
+          total + counter
+      }
+  }
+
+  private def addRecords(rows: List[Row]): Unit = {
+    redis.exists(rows.map(_.matchKey)).foreach { resultMap =>
+      rows.foreach { row =>
+        val result = resultMap(row.matchKey)
+        if (!result) {
+          self ! Add(row)
+        } else {
+          self ! Ignore(row)
+        }
+      }
+    }
+  }
+}
+
+case object TotalCount
+case class Add(row: Row)
+case class Ignore(row: Row)
+case class Counter(pageBuilder: PageBuilder)
+case class ForceWrite(pageBuilder: PageBuilder)

data/src/main/scala/org/embulk/filter/key_in_redis/redis/Redis.scala

@@ -1,14 +1,20 @@
 package org.embulk.filter.key_in_redis.redis
 
+import java.util.concurrent.TimeUnit
+
+import akka.util.Timeout
+import akka.pattern.ask
 import org.slf4j.Logger
 import redis._
+import org.embulk.filter.key_in_redis.actor.Actors._
 
 import scala.annotation.tailrec
 import scala.concurrent.duration._
 import scala.concurrent._
 import scala.util._
-
 import scala.collection.mutable
+import org.embulk.filter.key_in_redis.ToFutureExtensionOps._
+import org.embulk.filter.key_in_redis.actor._
 
 class Redis(setKey: String,
             host: String,
@@ -16,10 +22,8 @@ class Redis(setKey: String,
             replicaHosts: Map[String, Int],
             db: Option[Int],
             loadOnMemory: Boolean)(implicit logger: Logger) {
-  implicit val actorSystem = akka.actor.ActorSystem(
-    "redis-client",
-    classLoader = Some(this.getClass.getClassLoader))
 
+  implicit val ec: ExecutionContextExecutor = actorSystem.dispatcher
   lazy val cacheInstance: Option[Cache] = if (loadOnMemory) {
     Some(Cache(() => loadAll()))
   } else None
@@ -37,8 +41,7 @@ class Redis(setKey: String,
 
   def loadAll(): mutable.Set[String] = {
     logger.info(s"Loading from Redis start.")
-    import scala.concurrent.ExecutionContext.Implicits.global
-    import ToFutureExtensionOps._
+    import org.embulk.filter.key_in_redis.ToFutureExtensionOps._
     val buffer = mutable.Set.empty[String]
     @tailrec
     def _scan(cursor: Int): Unit = {
@@ -57,10 +60,10 @@ class Redis(setKey: String,
   }
 
   def ping(): String = {
-    import scala.concurrent.ExecutionContext.Implicits.global
     val s: Future[String] = redis.ping()
     s.onComplete {
-      case Success(result) => result
+      case Success(result) =>
+        result
       case Failure(t) =>
         actorSystem.shutdown()
         throw t
@@ -68,38 +71,63 @@ class Redis(setKey: String,
     Await.result(s, 10.minute)
   }
 
-  def exists(values: Seq[String]): Map[String, Boolean] = cacheInstance match {
-    case Some(cached) =>
-      values.map { v =>
-        v -> cached.contains(v)
-      }.toMap
-    case None =>
-      import scala.concurrent.ExecutionContext.Implicits.global
-      import ToFutureExtensionOps._
-      val input = values.zipWithIndex.map(_.swap).toMap
-      val transaction = redis.transaction()
-      val f = values.map { v =>
-        transaction.sismember(setKey, v)
-      }
-      transaction.exec()
-      val results = Future
-        .sequence(f)
-        .toTask
-        .unsafePerformSync
-        .zipWithIndex
-        .map(_.swap)
-        .toMap
-      results.map {
-        case (index, result) =>
-          input(index) -> result
+  def keyExists(): Unit = {
+    val s: Future[Boolean] = redis.exists(setKey)
+    s.onComplete {
+      case Success(_) =>
+      case Failure(t) =>
+        actorSystem.shutdown()
+        throw t
+    }
+    val result = Await.result(s, 10.minute)
+    if (!result) {
+      actorSystem.shutdown()
+      throw sys.error(s"key not found in redis. $setKey")
+    }
+  }
+
+  def exists(values: Seq[String]): Future[mutable.Map[String, Boolean]] = {
+    val futureResult = cacheInstance match {
+      case Some(cached) =>
+        values.map { v =>
+          Future.successful(v -> cached.contains(v))
+        }
+      case None =>
+        val transaction = redis.transaction()
+        val futures = values.map { v =>
+          transaction.sismember(setKey, v).map { result =>
+            (v ,result)
+          }
+        }
+        transaction.exec()
+        futures
+    }
+    Future.sequence(futureResult).map { sequence =>
+      val result = mutable.ListMap[String,Boolean]()
+      sequence.foreach {
+        case (key, value) =>
+          result.put(key, value)
       }
+      result
+    }
   }
 
   def close(): Unit = {
+    while (counter() != 0) {
+      Thread.sleep(1000)
+    }
     redis.stop()
     // wait for stopping.
     Thread.sleep(1000)
     actorSystem.shutdown()
   }
 
+  def counter(): Int = {
+    implicit val timeout: Timeout = Timeout(24, TimeUnit.HOURS)
+    (Actors.register ? TotalCount)
+      .mapTo[Int]
+      .toTask
+      .unsafePerformSync
+  }
+
 }

data/src/main/scala/org/embulk/filter/key_in_redis/row/Row.scala

@@ -0,0 +1,38 @@
+package org.embulk.filter.key_in_redis.row
+
+import org.embulk.spi.PageBuilder
+import org.embulk.spi.`type`._
+import org.embulk.spi.time.Timestamp
+import org.msgpack.value.Value
+
+import scala.collection.mutable
+
+case class Row(matchKey: String,
+               seq: mutable.Set[ValueHolder[_]],
+               pageBuilder: PageBuilder) {
+  def addRecord(): Unit = {
+    seq.foreach { vh =>
+      vh.value match {
+        case Some(v: Boolean) if vh.column.getType.isInstanceOf[BooleanType] =>
+          pageBuilder.setBoolean(vh.column, v)
+        case Some(v: Long) if vh.column.getType.isInstanceOf[LongType] =>
+          pageBuilder.setLong(vh.column, v)
+        case Some(v: Double) if vh.column.getType.isInstanceOf[DoubleType] =>
+          pageBuilder.setDouble(vh.column, v)
+        case Some(v: String) if vh.column.getType.isInstanceOf[StringType] =>
+          pageBuilder.setString(vh.column, v)
+        case Some(v: Timestamp)
+            if vh.column.getType.isInstanceOf[TimestampType] =>
+          pageBuilder.setTimestamp(vh.column, v)
+        case Some(v: Value) if vh.column.getType.isInstanceOf[JsonType] =>
+          pageBuilder.setJson(vh.column, v)
+        case None =>
+          pageBuilder.setNull(vh.column)
+        case _ =>
+          sys.error("unmatched types.")
+      }
+    }
+    pageBuilder.addRecord()
+  }
+
+}

data/src/main/scala/org/embulk/filter/key_in_redis/{column → row}/SetValueColumnVisitor.scala

@@ -1,18 +1,16 @@
-package org.embulk.filter.key_in_redis.column
+package org.embulk.filter.key_in_redis.row
 
 import java.security.MessageDigest
 
 import org.bouncycastle.util.encoders.Hex
 import org.embulk.filter.key_in_redis.json.JsonParser
-import org.embulk.spi.`type`._
-import org.embulk.spi.time.{Timestamp, TimestampFormatter}
+import org.embulk.spi.time.TimestampFormatter
 import org.embulk.spi.{
   Column,
   PageBuilder,
   PageReader,
   ColumnVisitor => EmbulkColumnVisitor
 }
-import org.msgpack.value.Value
 
 case class SetValueColumnVisitor(reader: PageReader,
                                  timestampFormatter: TimestampFormatter,
@@ -84,8 +82,6 @@ case class SetValueColumnVisitor(reader: PageReader,
     result
   }
 
-  case class ValueHolder[A](column: Column, value: Option[A])
-
   def put(column: Column, value: String): Unit = {
     if (parameterKeys.contains(column.getName)) {
       recordMap.put(column.getName, value)
@@ -93,41 +89,18 @@ case class SetValueColumnVisitor(reader: PageReader,
     ()
   }
 
-  def addRecord(pageBuilder: PageBuilder): Unit = {
-    valueHolderSet.foreach { vh =>
-      vh.value match {
-        case Some(v: Boolean) if vh.column.getType.isInstanceOf[BooleanType] =>
-          pageBuilder.setBoolean(vh.column, v)
-        case Some(v: Long) if vh.column.getType.isInstanceOf[LongType] =>
-          pageBuilder.setLong(vh.column, v)
-        case Some(v: Double) if vh.column.getType.isInstanceOf[DoubleType] =>
-          pageBuilder.setDouble(vh.column, v)
-        case Some(v: String) if vh.column.getType.isInstanceOf[StringType] =>
-          pageBuilder.setString(vh.column, v)
-        case Some(v: Timestamp)
-            if vh.column.getType.isInstanceOf[TimestampType] =>
-          pageBuilder.setTimestamp(vh.column, v)
-        case Some(v: Value) if vh.column.getType.isInstanceOf[JsonType] =>
-          pageBuilder.setJson(vh.column, v)
-        case None =>
-          pageBuilder.setNull(vh.column)
-        case _ =>
-          sys.error("unmatched types.")
-      }
-    }
-    pageBuilder.addRecord()
-  }
-
-  def getMatchKey: String = {
+  lazy val matchKey: String = {
     val keys = sortedKeys
       .flatMap { key =>
         recordMap.get(key)
       }
       .mkString(appender)
-
     if (matchAsMd5) {
       Hex.toHexString(digestMd5.digest(keys.getBytes()))
     } else keys
   }
 
+  def getRow(pageBuilder: PageBuilder): Row =
+    Row(matchKey, valueHolderSet, pageBuilder)
+
 }

data/src/main/scala/org/embulk/filter/key_in_redis/row/ValueHolder.scala

@@ -0,0 +1,5 @@
+package org.embulk.filter.key_in_redis.row
+
+import org.embulk.spi.Column
+
+case class ValueHolder[A](column: Column, value: Option[A])

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-key_in_redis
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - smdmts
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-23 00:00:00.000000000 Z
+date: 2017-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -59,14 +59,19 @@ files:
 - project/build.properties
 - project/plugins.sbt
 - settings.gradle
+- src/main/resources/application.conf
 - src/main/scala/org/embulk/filter/key_in_redis/KeyInRedisFilterPlugin.scala
 - src/main/scala/org/embulk/filter/key_in_redis/PageOutput.scala
 - src/main/scala/org/embulk/filter/key_in_redis/PluginTask.scala
-- src/main/scala/org/embulk/filter/key_in_redis/column/SetValueColumnVisitor.scala
+- src/main/scala/org/embulk/filter/key_in_redis/TaskExtensionOps.scala
+- src/main/scala/org/embulk/filter/key_in_redis/actor/Actors.scala
+- src/main/scala/org/embulk/filter/key_in_redis/actor/Register.scala
 - src/main/scala/org/embulk/filter/key_in_redis/json/JsonParser.scala
 - src/main/scala/org/embulk/filter/key_in_redis/redis/Cache.scala
 - src/main/scala/org/embulk/filter/key_in_redis/redis/Redis.scala
-- src/main/scala/org/embulk/filter/key_in_redis/redis/TaskExtensionOps.scala
+- src/main/scala/org/embulk/filter/key_in_redis/row/Row.scala
+- src/main/scala/org/embulk/filter/key_in_redis/row/SetValueColumnVisitor.scala
+- src/main/scala/org/embulk/filter/key_in_redis/row/ValueHolder.scala
 - src/test/scala/org/embulk/filter/key_in_redis/json/JsonParserSpec.scala
 - classpath/akka-actor_2.11-2.3.6.jar
 - classpath/bcpkix-jdk15on-1.57.jar
@@ -81,7 +86,7 @@ files:
 - classpath/circe-numbers_2.11-0.8.0.jar
 - classpath/circe-parser_2.11-0.8.0.jar
 - classpath/config-1.2.1.jar
-- classpath/embulk-filter-key_in_redis-0.1.2.jar
+- classpath/embulk-filter-key_in_redis-0.1.3.jar
 - classpath/jawn-parser_2.11-0.10.4.jar
 - classpath/machinist_2.11-0.6.1.jar
 - classpath/macro-compat_2.11-1.1.1.jar