embulk-output-key_to_redis 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cacc8741d9141b1fc1de001245082f1542849df9
-  data.tar.gz: f91d317b22fc2a87d30044d14f0498a1bf39e368
+  metadata.gz: 05c65ae37d4a10e9ac98304434c2f9ab8421e3f2
+  data.tar.gz: e92b05472e3366ee478ddd04c80558053a32027d
 SHA512:
-  metadata.gz: cfeb481de84ff42d12b0c361b989736851bdf007ae73b25ea1de18837b2a676e3a35465c642cc82c887b7943404c5ed3a6c00a3064582761f6b1591a5bb05514
-  data.tar.gz: 3fcfac83f63fbbfc9cd7e2b5142d3ddd7bffe0158d0d8c5935fd2fab2ea40d37190061e2bfb30b81ddf6ceaff3bc77e0a3a5780e2d88c21a3ae9f64770c8d804
+  metadata.gz: e92c0012e6e0d9b4e6853fdb5c538908bc4d5e53c86a82cf9c552fdbe43d770805efe658e615bbb5af62e93083adf058472656e146ac7090a01f303f29560922
+  data.tar.gz: 6dba520261ff5f67242dca2d391fe1b5b00c4c01544f3d69eda24280ccbf99e6185f7dc028030c961a5b943f0c1c111bf11b28a1d24489e0aa5430df1bf008cd

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.1.2"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/scala/org/embulk/output/key_to_redis/KeyToRedisOutputPlugin.scala

@@ -14,6 +14,7 @@ class KeyToRedisOutputPlugin extends OutputPlugin {
                            control: OutputPlugin.Control): ConfigDiff = {
     val task = config.loadConfig(classOf[PluginTask])
     KeyToRedisOutputPlugin.createRedisInstance(task)
+    KeyToRedisOutputPlugin.taskCountOpt = Some(taskCount)
     if (task.getFlushOnStart) {
       KeyToRedisOutputPlugin.redis.foreach(_.flush())
     }
@@ -44,13 +45,17 @@ class KeyToRedisOutputPlugin extends OutputPlugin {
       case None => // for map reduce executor.
         KeyToRedisOutputPlugin.createRedisInstance(task)
     }
-    PageOutput(taskSource, schema, task.getPutAsMD5)
+    PageOutput(taskSource,
+               schema,
+               KeyToRedisOutputPlugin.taskCountOpt,
+               task.getPutAsMD5)
   }
 
 }
 
 object KeyToRedisOutputPlugin {
   var redis: Option[Redis] = None
+  var taskCountOpt: Option[Int] = None
 
   def createRedisInstance(task: PluginTask): Unit = {
     KeyToRedisOutputPlugin.redis = Some(

data/src/main/scala/org/embulk/output/key_to_redis/PageOutput.scala

@@ -11,12 +11,10 @@ import org.bouncycastle.util.encoders.Hex
 import org.embulk.output.key_to_redis.redis.Redis
 
 import scala.collection.JavaConverters._
-import scala.concurrent.duration._
-import scala.collection.mutable.ListBuffer
-import scala.concurrent.{Await, Future}
 
 case class PageOutput(taskSource: TaskSource,
                       schema: Schema,
+                      taskCountOpt: Option[Int],
                       putAsMD5: Boolean)
     extends TransactionalPageOutput {
   val task: PluginTask = taskSource.loadTask(classOf[PluginTask])
@@ -25,7 +23,6 @@ case class PageOutput(taskSource: TaskSource,
   def timestampFormatter(): TimestampFormatter =
     new TimestampFormatter(task, Optional.absent())
 
-  val buffer = new ListBuffer[Future[Long]]
   val redis: Redis =
     KeyToRedisOutputPlugin.redis.getOrElse(sys.error("could not find redis."))
 
@@ -45,21 +42,21 @@ case class PageOutput(taskSource: TaskSource,
         if (putAsMD5) {
           val hash = Hex.toHexString(
             digestMd5.digest(setValueVisitor.getValue.getBytes()))
-          buffer.append(redis.sadd(hash))
+          redis.sadd(hash)
         } else {
-          buffer.append(redis.sadd(setValueVisitor.getValue))
+          redis.sadd(setValueVisitor.getValue)
         }
       }
     }
     reader.close()
   }
-
   override def finish(): Unit = {
-    import scala.concurrent.ExecutionContext.Implicits.global
-    val sequence = Future.sequence(buffer)
-    Await.result(sequence, Duration.Inf)
+    // for map/reduce executor.
+    if (taskCountOpt.isEmpty) {
+      // close immediately.
+      redis.close()
+    }
   }
-
   override def close(): Unit = ()
   override def commit(): TaskReport = Exec.newTaskReport
   override def abort(): Unit = ()

data/src/main/scala/org/embulk/output/key_to_redis/redis/Redis.scala

@@ -1,5 +1,8 @@
 package org.embulk.output.key_to_redis.redis
 
+import akka.actor._
+import akka.pattern.ask
+import akka.util.Timeout
 import redis.RedisClient
 
 import scala.concurrent._
@@ -10,7 +13,9 @@ case class Redis(setKey: String, host: String, port: Int, db: Option[Int]) {
   implicit val actorSystem = akka.actor.ActorSystem(
     "redis-client",
     classLoader = Some(this.getClass.getClassLoader))
+
   val redis = RedisClient(host, port, db = db)
+  val sender: ActorRef = actorSystem.actorOf(Props(Sender(setKey, redis)))
 
   def ping(): String = {
     import scala.concurrent.ExecutionContext.Implicits.global
@@ -23,8 +28,9 @@ case class Redis(setKey: String, host: String, port: Int, db: Option[Int]) {
     }
     Await.result(s, 10.minute)
   }
-  def sadd(value: String): Future[Long] = {
-    redis.sadd(setKey, value)
+
+  def sadd(value: String): Unit = {
+    sender ! Message(value)
   }
 
   def flush(): Boolean = {
@@ -32,8 +38,20 @@ case class Redis(setKey: String, host: String, port: Int, db: Option[Int]) {
   }
 
   def close(): Unit = {
+    sender ! Close
+    implicit val timeout: Timeout = Timeout(1000.seconds)
+    var finished = false
+    while (!finished) {
+      val f = sender ? GetStatus
+      val result = Await.result(f.mapTo[Result], Duration.Inf)
+      if (!result.finished) {
+        Thread.sleep(1000)
+      } else {
+        finished = result.finished
+      }
+    }
     redis.stop()
-    // wait for stopping.
+    // wait for redis stop.
     Thread.sleep(1000)
     actorSystem.shutdown()
   }

data/src/main/scala/org/embulk/output/key_to_redis/redis/Sender.scala

@@ -0,0 +1,34 @@
+package org.embulk.output.key_to_redis.redis
+
+import akka.actor.{Actor, ActorSystem}
+import redis.RedisClient
+
+import scala.collection.mutable.ListBuffer
+import scala.concurrent.Future
+
+case class Sender(setKey: String, redis: RedisClient) extends Actor {
+
+  implicit val actorSystem: ActorSystem = context.system
+  val buffer = new ListBuffer[String]
+  val command = new ListBuffer[Future[Long]]
+
+  override def receive: Receive = {
+    case Message(v) =>
+      buffer.append(v)
+      // bulk insert
+      if (buffer.size == 10000) {
+        command.append(redis.sadd(setKey, buffer: _*))
+        buffer.clear()
+      }
+    case Close =>
+      command.append(redis.sadd(setKey, buffer: _*))
+      buffer.clear()
+    case GetStatus =>
+      sender() ! Result(command.forall(_.isCompleted))
+  }
+}
+
+case object GetStatus
+case class Result(finished: Boolean) extends AnyVal
+case object Close
+case class Message(v: String) extends AnyVal

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-key_to_redis
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - smdmts
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-16 00:00:00.000000000 Z
+date: 2017-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -65,6 +65,7 @@ files:
 - src/main/scala/org/embulk/output/key_to_redis/column/SetValueColumnVisitor.scala
 - src/main/scala/org/embulk/output/key_to_redis/json/JsonParser.scala
 - src/main/scala/org/embulk/output/key_to_redis/redis/Redis.scala
+- src/main/scala/org/embulk/output/key_to_redis/redis/Sender.scala
 - src/test/scala/org/embulk/output/key_to_redis/.gitkeep
 - classpath/akka-actor_2.11-2.3.6.jar
 - classpath/bcpkix-jdk15on-1.57.jar
@@ -78,7 +79,7 @@ files:
 - classpath/circe-numbers_2.11-0.8.0.jar
 - classpath/circe-parser_2.11-0.8.0.jar
 - classpath/config-1.2.1.jar
-- classpath/embulk-output-key_to_redis-0.1.1.jar
+- classpath/embulk-output-key_to_redis-0.1.2.jar
 - classpath/jawn-parser_2.11-0.10.4.jar
 - classpath/machinist_2.11-0.6.1.jar
 - classpath/macro-compat_2.11-1.1.1.jar