embulk-filter-key_in_redis 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: be35f78cca388bf1225008706a2b8f5264d384fc
-  data.tar.gz: 24ab40380a5326311caabfb2660d4fad8eb93863
+  metadata.gz: 6e093ef9f81cc900cf6eb92fd2a3b277f5e7340a
+  data.tar.gz: 6310042c14d5179a8093d2d289b495adb6a43109
 SHA512:
-  metadata.gz: 5e6465ebe30c04cf6885500356cb0d9e89c9838a02adf2146728c215ac33b2c61285c46e75d73694f51c29be58e8a73a905f17cf38439f4ec46373c6d221e72a
-  data.tar.gz: d574680ca584af8bd6ef0000141ff8cd3f30ac4fa865d5c32a326aafbc8595a9d573a39961b59b61d305ca943d0c6db676390ec16fa27125a3346371795001fc
+  metadata.gz: 58ffdafa04ed57b99cd88638e72c03798ec91390f3a0ee26aa8956600753fced81f4f4b62f88b557399ed87496946b14f90b041bd7aa8032424830c440401f7a
+  data.tar.gz: e6ad1f2a9993e9ea2b4003d04273fa1212ab4825ea91f24e665e42744d77d7440445aaf7930dfc9e66aff4ec96dc450bce2639bf3bde3af0f87a4ced54feebf9

data/.circleci/config.yml

@@ -0,0 +1,27 @@
+version: 2
+jobs:
+  build:
+    executorType: docker
+    docker:
+      - image: hseeberger/scala-sbt
+    working_directory: /root/embulk-filter-key_in_redis/
+    steps:
+      - checkout
+      - restore_cache:
+          name: Restoring Cache
+          keys:
+            - sbt
+      - setup_remote_docker
+      - run:
+          name: prepare
+          command: sbt update exit
+      - save_cache:
+          name: Saving Cache sbt
+          key: sbt
+          paths:
+            - "/root/.sbt"
+            - "/root/.ivy2"
+      - run:
+          name: compile
+          command: |
+            sbt compile test scalafmt::test exit

data/README.md

@@ -1,6 +1,6 @@
 # Key In Redis filter plugin for Embulk
 
-Filter the aggregated key into set of the Redis plugin.  
+Filtering by aggregated the keys in included Redis's SET.
 
 This plugin is designed to extract data set diff files used with the combination in below use cases.
 
@@ -25,8 +25,8 @@ This plugin is designed to extract data set diff files used with the combination
 |  host                                | string      | optional   | "127.0.0.1"              | redis servers host     |
 |  port                                | integer     | optional   | "6379"                   | redis servers port     |
 |  db                                  | integer     | optional   | "null"                   | redis servers db       |
-|  flush_on_start                      | boolean     | optional   | "false"                  | flush on start specified redis servers db |
 |  redis_set_key                       | string      | required   |                          | redis of key of set name |
+|  load_on_memory                      | boolean     | optional   | "false"                  | load all data from redis *1 |
 |  appender                            | string      | optional   | "-"                      | multi key of appender  |
 |  match_as_md5                        | boolean    | optional   | "false"                   | smembers the value to converted md5 |
 |  key_with_index                      | hash: Map<Int,String> | required with key_with_index or json_key_with_index or only one || index with key name |
@@ -34,6 +34,10 @@ This plugin is designed to extract data set diff files used with the combination
 |  default_timezone                    | string      | optional   | UTC                      | |
 |  default_timestamp_format            | string      | optional   | %Y-%m-%d %H:%M:%S.%6N    | |
 
+
+*1: load_on_memory mode requires JVM memory as all records stored on Redis.
+
+
 ## Example
 
 - inside redis

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.1"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -21,11 +21,14 @@ targetCompatibility = 1.7
 dependencies {
     compile  "org.embulk:embulk-core:0.8.26"
     compile "org.scala-lang:scala-library:2.11.11"
+    compile group: 'com.github.pathikrit', name: 'better-files_2.11', version: '2.17.1'
     compile group: 'io.circe', name: 'circe-core_2.11', version: '0.8.0'
     compile group: 'io.circe', name: 'circe-generic_2.11', version: '0.8.0'
     compile group: 'io.circe', name: 'circe-parser_2.11', version: '0.8.0'
     compile group: 'com.github.etaty', name: 'rediscala_2.11', version: '1.7.0'
     compile group: 'org.bouncycastle', name: 'bcpkix-jdk15on', version: '1.57'
+    compile group: 'org.scalaz', name: 'scalaz-core_2.11', version: '7.2.14'
+    compile group: 'org.scalaz', name: 'scalaz-concurrent_2.11', version: '7.2.14'
     testCompile group: 'org.scalatest', name: 'scalatest_2.11', version: '3.0.1'
     provided "org.embulk:embulk-core:0.8.26"
     testCompile "junit:junit:4.+"

data/build.sbt

@@ -22,6 +22,9 @@ libraryDependencies ++= Seq(
   "org.embulk" % "embulk-core" % "0.8.25",
   "com.github.etaty" %% "rediscala" % "1.7.0",
   "org.bouncycastle" % "bcpkix-jdk15on" % "1.57",
+  "com.github.pathikrit" %% "better-files" % "2.17.1",
+  "org.scalaz" %% "scalaz-core" % "7.2.14",
+  "org.scalaz" %% "scalaz-concurrent" % "7.2.14",
   "io.circe" %% "circe-core" % circeVersion,
   "io.circe" %% "circe-generic" % circeVersion,
   "io.circe" %% "circe-parser" % circeVersion,

data/src/main/scala/org/embulk/filter/key_in_redis/KeyInRedisFilterPlugin.scala

@@ -4,16 +4,22 @@ import org.embulk.config.{ConfigSource, TaskSource}
 import org.embulk.filter.key_in_redis.redis.Redis
 import org.embulk.spi
 import org.embulk.spi._
+import org.slf4j.Logger
+
+import scala.collection.JavaConverters._
 
 class KeyInRedisFilterPlugin extends FilterPlugin {
 
   override def transaction(config: ConfigSource,
                            inputSchema: Schema,
                            control: FilterPlugin.Control): Unit = {
+
     val task = config.loadConfig(classOf[PluginTask])
+    val taskSource = task.dump()
+
     KeyInRedisFilterPlugin.createRedisInstance(task)
     KeyInRedisFilterPlugin.redis.foreach(_.ping())
-    control.run(task.dump(), inputSchema)
+    control.run(taskSource, inputSchema)
     KeyInRedisFilterPlugin.redis.foreach(_.close())
   }
 
@@ -32,12 +38,20 @@ class KeyInRedisFilterPlugin extends FilterPlugin {
 }
 
 object KeyInRedisFilterPlugin {
+  lazy val cacheName = s"${this.getClass.getCanonicalName}-cache"
+  implicit val logger: Logger = Exec.getLogger(classOf[KeyInRedisFilterPlugin])
   var redis: Option[Redis] = None
   def createRedisInstance(task: PluginTask): Unit = {
     KeyInRedisFilterPlugin.redis = Some(
-      Redis(task.getRedisSetKey, task.getHost, task.getPort, {
-        if (task.getDb.isPresent) Some(task.getDb.get())
-        else None
-      }))
+      new Redis(
+        task.getRedisSetKey,
+        task.getHost,
+        task.getPort,
+        task.getReplicaHosts.asScala.toMap.mapValues(_.toInt), {
+          if (task.getDb.isPresent) Some(task.getDb.get())
+          else None
+        },
+        task.getLoadOnMemory
+      ))
   }
 }

data/src/main/scala/org/embulk/filter/key_in_redis/PageOutput.scala

@@ -6,6 +6,8 @@ import com.google.common.base.Optional
 import org.bouncycastle.util.encoders.Hex
 import org.embulk.filter.key_in_redis.column._
 
+import scala.collection.mutable.ListBuffer
+
 import scala.collection.JavaConverters._
 import org.embulk.spi.time.TimestampFormatter
 import org.embulk.spi.{
@@ -30,6 +32,7 @@ case class PageOutput(task: PluginTask,
   override def add(page: Page): Unit = {
     val reader: PageReader = new PageReader(schema)
     reader.setPage(page)
+    val handlerBuffer = new ListBuffer[PageHandler]()
     while (reader.nextRecord()) {
       val setValueVisitor = SetValueColumnVisitor(
         reader,
@@ -41,12 +44,15 @@ case class PageOutput(task: PluginTask,
       val matchValue = if (task.getMatchAsMD5) {
         Hex.toHexString(digestMd5.digest(setValueVisitor.getValue.getBytes()))
       } else setValueVisitor.getValue
-      KeyInRedisFilterPlugin.redis.foreach { redis =>
-        val passthroughColumnVisitor =
-          PassthroughColumnVisitor(reader, pageBuilder)
-        if (redis.nonExists(matchValue)) {
-          schema.visitColumns(passthroughColumnVisitor)
-          passthroughColumnVisitor.addRecord()
+      handlerBuffer.append(
+        PageHandler(matchValue, PassthroughColumnVisitor(reader, pageBuilder)))
+    }
+    KeyInRedisFilterPlugin.redis.foreach { redis =>
+      val result = redis.exists(handlerBuffer.map(_.matchValue))
+      handlerBuffer.foreach { value =>
+        if (!result(value.matchValue)) {
+          schema.visitColumns(value.visitor)
+          value.visitor.addRecord()
         }
       }
     }
@@ -57,3 +63,5 @@ case class PageOutput(task: PluginTask,
   override def close(): Unit = pageBuilder.close()
 
 }
+
+case class PageHandler(matchValue: String, visitor: PassthroughColumnVisitor)

data/src/main/scala/org/embulk/filter/key_in_redis/PluginTask.scala

@@ -13,6 +13,10 @@ trait PluginTask extends Task with TimestampFormatter.Task {
   @ConfigDefault("false")
   def getMatchAsMD5: Boolean
 
+  @Config("load_on_memory")
+  @ConfigDefault("false")
+  def getLoadOnMemory: Boolean
+
   @Config("key_with_index")
   @ConfigDefault("{}")
   def getKeyWithIndex: java.util.Map[String, String]
@@ -33,6 +37,10 @@ trait PluginTask extends Task with TimestampFormatter.Task {
   @ConfigDefault("6379")
   def getPort: Int
 
+  @Config("replica_hosts")
+  @ConfigDefault("{}")
+  def getReplicaHosts: java.util.Map[String, String]
+
   @Config("db")
   @ConfigDefault("null")
   def getDb: Optional[Int]

data/src/main/scala/org/embulk/filter/key_in_redis/redis/Cache.scala

@@ -0,0 +1,11 @@
+package org.embulk.filter.key_in_redis.redis
+
+import org.slf4j.Logger
+
+import scala.collection.mutable
+
+case class Cache(loadFromStorage: () => mutable.Set[String])(
+    implicit logger: Logger) {
+  private val cache = loadFromStorage()
+  def contains(value: String): Boolean = cache.contains(value)
+}

data/src/main/scala/org/embulk/filter/key_in_redis/redis/Redis.scala

@@ -1,16 +1,60 @@
 package org.embulk.filter.key_in_redis.redis
 
-import redis.RedisClient
+import org.slf4j.Logger
+import redis._
 
+import scala.annotation.tailrec
 import scala.concurrent.duration._
 import scala.concurrent._
 import scala.util._
 
-case class Redis(setKey: String, host: String, port: Int, db: Option[Int]) {
+import scala.collection.mutable
+
+class Redis(setKey: String,
+            host: String,
+            port: Int,
+            replicaHosts: Map[String, Int],
+            db: Option[Int],
+            loadOnMemory: Boolean)(implicit logger: Logger) {
   implicit val actorSystem = akka.actor.ActorSystem(
     "redis-client",
     classLoader = Some(this.getClass.getClassLoader))
-  val redis = RedisClient(host, port, db = db)
+
+  lazy val cacheInstance: Option[Cache] = if (loadOnMemory) {
+    Some(Cache(() => loadAll()))
+  } else None
+
+  val redisServers: Seq[RedisClient] = {
+    val primary = RedisClient(host, port, db = db)
+    val replica = replicaHosts.map {
+      case (host: String, port: Int) =>
+        RedisClient(host, port, db = db)
+    }
+    Seq(primary) ++ replica.toSeq
+  }
+
+  def redis: RedisClient = Random.shuffle(redisServers).head
+
+  def loadAll(): mutable.Set[String] = {
+    logger.info(s"Loading from Redis start.")
+    import scala.concurrent.ExecutionContext.Implicits.global
+    import ToFutureExtensionOps._
+    val buffer = mutable.Set.empty[String]
+    @tailrec
+    def _scan(cursor: Int): Unit = {
+      val task = redis.sscan[String](setKey, cursor, Option(500)).toTask
+      val result = task.unsafePerformSync
+      result.data.foreach { v =>
+        buffer.add(v)
+      }
+      if (result.index != 0) {
+        _scan(result.index)
+      }
+    }
+    _scan(0)
+    logger.info(s"Loading from Redis finished. record size is ${buffer.size}")
+    buffer
+  }
 
   def ping(): String = {
     import scala.concurrent.ExecutionContext.Implicits.global
@@ -24,13 +68,33 @@ case class Redis(setKey: String, host: String, port: Int, db: Option[Int]) {
     Await.result(s, 10.minute)
   }
 
-  def exists(value: String): Boolean = {
-    val s = redis.sismember(setKey, value)
-    Await.result(s, 10.minute)
+  def exists(values: Seq[String]): Map[String, Boolean] = cacheInstance match {
+    case Some(cached) =>
+      values.map { v =>
+        v -> cached.contains(v)
+      }.toMap
+    case None =>
+      import scala.concurrent.ExecutionContext.Implicits.global
+      import ToFutureExtensionOps._
+      val input = values.zipWithIndex.map(_.swap).toMap
+      val transaction = redis.transaction()
+      val f = values.map { v =>
+        transaction.sismember(setKey, v)
+      }
+      transaction.exec()
+      val results = Future
+        .sequence(f)
+        .toTask
+        .unsafePerformSync
+        .zipWithIndex
+        .map(_.swap)
+        .toMap
+      results.map {
+        case (index, result) =>
+          input(index) -> result
+      }
   }
 
-  def nonExists(value: String): Boolean = !exists(value)
-
   def close(): Unit = {
     redis.stop()
     // wait for stopping.

data/src/main/scala/org/embulk/filter/key_in_redis/redis/TaskExtensionOps.scala

@@ -0,0 +1,23 @@
+package org.embulk.filter.key_in_redis.redis
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{Failure, Success}
+import scalaz._, Scalaz._
+import scalaz.concurrent._
+
+final class FutureExtensionOps[A](self: Future[A]) {
+  def toTask(implicit ec: ExecutionContext): Task[A] = Task.async { register =>
+    self.onComplete {
+      case Success(v) => register(v.right)
+      case Failure(ex) => register(ex.left)
+    }
+  }
+}
+
+trait ToFutureExtensionOps {
+  implicit def toFutureExtensionOps[A](
+      future: Future[A]): FutureExtensionOps[A] =
+    new FutureExtensionOps(future)
+}
+
+object ToFutureExtensionOps extends ToFutureExtensionOps

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-key_in_redis
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - smdmts
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-14 00:00:00.000000000 Z
+date: 2017-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -45,6 +45,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- .circleci/config.yml
 - .gitignore
 - LICENSE.txt
 - README.md
@@ -64,12 +65,14 @@ files:
 - src/main/scala/org/embulk/filter/key_in_redis/column/PassthroughColumnVisitor.scala
 - src/main/scala/org/embulk/filter/key_in_redis/column/SetValueColumnVisitor.scala
 - src/main/scala/org/embulk/filter/key_in_redis/json/JsonParser.scala
+- src/main/scala/org/embulk/filter/key_in_redis/redis/Cache.scala
 - src/main/scala/org/embulk/filter/key_in_redis/redis/Redis.scala
-- src/test/scala/org/embulk/filter/key_in_redis/.gitkeep
+- src/main/scala/org/embulk/filter/key_in_redis/redis/TaskExtensionOps.scala
 - src/test/scala/org/embulk/filter/key_in_redis/json/JsonParserSpec.scala
 - classpath/akka-actor_2.11-2.3.6.jar
 - classpath/bcpkix-jdk15on-1.57.jar
 - classpath/bcprov-jdk15on-1.57.jar
+- classpath/better-files_2.11-2.17.1.jar
 - classpath/cats-core_2.11-0.9.0.jar
 - classpath/cats-kernel_2.11-0.9.0.jar
 - classpath/cats-macros_2.11-0.9.0.jar
@@ -79,7 +82,7 @@ files:
 - classpath/circe-numbers_2.11-0.8.0.jar
 - classpath/circe-parser_2.11-0.8.0.jar
 - classpath/config-1.2.1.jar
-- classpath/embulk-filter-key_in_redis-0.1.0.jar
+- classpath/embulk-filter-key_in_redis-0.1.1.jar
 - classpath/jawn-parser_2.11-0.10.4.jar
 - classpath/machinist_2.11-0.6.1.jar
 - classpath/macro-compat_2.11-1.1.1.jar
@@ -87,6 +90,9 @@ files:
 - classpath/scala-library-2.11.11.jar
 - classpath/scala-reflect-2.11.8.jar
 - classpath/scala-stm_2.11-0.7.jar
+- classpath/scalaz-concurrent_2.11-7.2.14.jar
+- classpath/scalaz-core_2.11-7.2.14.jar
+- classpath/scalaz-effect_2.11-7.2.14.jar
 - classpath/shapeless_2.11-2.3.2.jar
 - classpath/simulacrum_2.11-0.10.0.jar
 homepage: