embulk-filter-key_in_redis 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6e093ef9f81cc900cf6eb92fd2a3b277f5e7340a
-  data.tar.gz: 6310042c14d5179a8093d2d289b495adb6a43109
+  metadata.gz: 8c07a54c906730f2de5c0a2e32a1792cc0170877
+  data.tar.gz: 3cf0b84f743814a7ee32a35cb9a084eca9db287d
 SHA512:
-  metadata.gz: 58ffdafa04ed57b99cd88638e72c03798ec91390f3a0ee26aa8956600753fced81f4f4b62f88b557399ed87496946b14f90b041bd7aa8032424830c440401f7a
-  data.tar.gz: e6ad1f2a9993e9ea2b4003d04273fa1212ab4825ea91f24e665e42744d77d7440445aaf7930dfc9e66aff4ec96dc450bce2639bf3bde3af0f87a4ced54feebf9
+  metadata.gz: 5b51785e0dfe0a5007cea67fd60c96aaa9c805ec605b074db141310532418847cd07f4d38d5d6745ff9a5067f7f5500f0fda004212e9f88a22d9d98018d00b00
+  data.tar.gz: d73e095906c90aef7216fcba69a5f514a7871acde3d1b94ce07918f16cdfecb5ffa05b04bfe8485baf0f3ef62c408a6a11004864fd9a24e6facb87ddc7ef5508

data/README.md

@@ -28,7 +28,8 @@ This plugin is designed to extract data set diff files used with the combination
 |  redis_set_key                       | string      | required   |                          | redis of key of set name |
 |  load_on_memory                      | boolean     | optional   | "false"                  | load all data from redis *1 |
 |  appender                            | string      | optional   | "-"                      | multi key of appender  |
-|  match_as_md5                        | boolean    | optional   | "false"                   | smembers the value to converted md5 |
+|  match_as_md5                        | boolean     | optional   | "false"                  | smembers the value to converted md5 |
+|  replica_hosts                       | hash: Map<String,Int>    | optional   |             | list of replica redis servers host: port |
 |  key_with_index                      | hash: Map<Int,String> | required with key_with_index or json_key_with_index or only one || index with key name |
 |  json_key_with_index                 | hash: Map<Int,String> | required with key_with_index or json_key_with_index or only one || json columns's expanded key name |
 |  default_timezone                    | string      | optional   | UTC                      | |

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.1.2"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -73,7 +73,7 @@ Gem::Specification.new do |spec|
   spec.description   = %[Key In Redis]
   spec.email         = ["smdmts@gmail.com"]
   spec.licenses      = ["MIT"]
-  # TODO set this: spec.homepage      = "https://github.com/smdmts/embulk-filter-key_in_redis"
+  spec.homepage      = "https://github.com/smdmts/embulk-filter-key_in_redis"
 
   spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
   spec.test_files    = spec.files.grep(%r"^(test|spec)/")

data/src/main/scala/org/embulk/filter/key_in_redis/PageOutput.scala

@@ -1,13 +1,9 @@
 package org.embulk.filter.key_in_redis
 
-import java.security.MessageDigest
-
 import com.google.common.base.Optional
-import org.bouncycastle.util.encoders.Hex
 import org.embulk.filter.key_in_redis.column._
 
 import scala.collection.mutable.ListBuffer
-
 import scala.collection.JavaConverters._
 import org.embulk.spi.time.TimestampFormatter
 import org.embulk.spi.{
@@ -27,41 +23,32 @@ case class PageOutput(task: PluginTask,
   def timestampFormatter(): TimestampFormatter =
     new TimestampFormatter(task, Optional.absent())
 
-  val digestMd5: MessageDigest = MessageDigest.getInstance("MD5")
-
   override def add(page: Page): Unit = {
-    val reader: PageReader = new PageReader(schema)
-    reader.setPage(page)
-    val handlerBuffer = new ListBuffer[PageHandler]()
-    while (reader.nextRecord()) {
+    val baseReader: PageReader = new PageReader(schema)
+    baseReader.setPage(page)
+    val rows = new ListBuffer[SetValueColumnVisitor]()
+    while (baseReader.nextRecord()) {
       val setValueVisitor = SetValueColumnVisitor(
-        reader,
+        baseReader,
         timestampFormatter(),
         task.getKeyWithIndex.asScala.toMap,
         task.getJsonKeyWithIndex.asScala.toMap,
-        task.getAppender)
+        task.getAppender,
+        task.getMatchAsMD5)
       schema.visitColumns(setValueVisitor)
-      val matchValue = if (task.getMatchAsMD5) {
-        Hex.toHexString(digestMd5.digest(setValueVisitor.getValue.getBytes()))
-      } else setValueVisitor.getValue
-      handlerBuffer.append(
-        PageHandler(matchValue, PassthroughColumnVisitor(reader, pageBuilder)))
+      rows.append(setValueVisitor)
     }
     KeyInRedisFilterPlugin.redis.foreach { redis =>
-      val result = redis.exists(handlerBuffer.map(_.matchValue))
-      handlerBuffer.foreach { value =>
-        if (!result(value.matchValue)) {
-          schema.visitColumns(value.visitor)
-          value.visitor.addRecord()
+      val result = redis.exists(rows.map(_.getMatchKey))
+      rows.foreach { row =>
+        if (!result(row.getMatchKey)) {
+          row.addRecord(pageBuilder)
         }
       }
     }
-    reader.close()
   }
 
   override def finish(): Unit = pageBuilder.finish()
   override def close(): Unit = pageBuilder.close()
 
-}
-
-case class PageHandler(matchValue: String, visitor: PassthroughColumnVisitor)
+}

data/src/main/scala/org/embulk/filter/key_in_redis/column/SetValueColumnVisitor.scala

@@ -1,22 +1,31 @@
 package org.embulk.filter.key_in_redis.column
 
+import java.security.MessageDigest
+
+import org.bouncycastle.util.encoders.Hex
 import org.embulk.filter.key_in_redis.json.JsonParser
-import org.embulk.spi.time.TimestampFormatter
+import org.embulk.spi.`type`._
+import org.embulk.spi.time.{Timestamp, TimestampFormatter}
 import org.embulk.spi.{
   Column,
+  PageBuilder,
   PageReader,
   ColumnVisitor => EmbulkColumnVisitor
 }
+import org.msgpack.value.Value
 
 case class SetValueColumnVisitor(reader: PageReader,
                                  timestampFormatter: TimestampFormatter,
                                  keyMap: Map[String, String],
                                  jsonKeyMap: Map[String, String],
-                                 appender: String)
+                                 appender: String,
+                                 matchAsMd5: Boolean)
     extends EmbulkColumnVisitor {
   import scala.collection.mutable
   private val recordMap = mutable.Map[String, String]()
+  private val valueHolderSet = mutable.Set[ValueHolder[_]]()
 
+  val digestMd5: MessageDigest = MessageDigest.getInstance("MD5")
   val parameterKeys: Seq[String] = keyMap.values.toSeq
   val jsonKeys: Seq[String] = jsonKeyMap.values.toSeq
   val sortedKeys: List[String] = {
@@ -65,12 +74,17 @@ case class SetValueColumnVisitor(reader: PageReader,
       put(column, v.toJson)
     }
 
-  def value[A](column: Column, method: => (Column => A)): Option[A] =
-    if (reader.isNull(column)) {
+  def value[A](column: Column, method: => (Column => A)): Option[A] = {
+    val result = if (reader.isNull(column)) {
       None
     } else {
       Some(method(column))
     }
+    valueHolderSet.add(ValueHolder(column, result))
+    result
+  }
+
+  case class ValueHolder[A](column: Column, value: Option[A])
 
   def put(column: Column, value: String): Unit = {
     if (parameterKeys.contains(column.getName)) {
@@ -79,12 +93,41 @@ case class SetValueColumnVisitor(reader: PageReader,
     ()
   }
 
-  def getValue: String = {
-    sortedKeys
+  def addRecord(pageBuilder: PageBuilder): Unit = {
+    valueHolderSet.foreach { vh =>
+      vh.value match {
+        case Some(v: Boolean) if vh.column.getType.isInstanceOf[BooleanType] =>
+          pageBuilder.setBoolean(vh.column, v)
+        case Some(v: Long) if vh.column.getType.isInstanceOf[LongType] =>
+          pageBuilder.setLong(vh.column, v)
+        case Some(v: Double) if vh.column.getType.isInstanceOf[DoubleType] =>
+          pageBuilder.setDouble(vh.column, v)
+        case Some(v: String) if vh.column.getType.isInstanceOf[StringType] =>
+          pageBuilder.setString(vh.column, v)
+        case Some(v: Timestamp)
+            if vh.column.getType.isInstanceOf[TimestampType] =>
+          pageBuilder.setTimestamp(vh.column, v)
+        case Some(v: Value) if vh.column.getType.isInstanceOf[JsonType] =>
+          pageBuilder.setJson(vh.column, v)
+        case None =>
+          pageBuilder.setNull(vh.column)
+        case _ =>
+          sys.error("unmatched types.")
+      }
+    }
+    pageBuilder.addRecord()
+  }
+
+  def getMatchKey: String = {
+    val keys = sortedKeys
       .flatMap { key =>
         recordMap.get(key)
       }
       .mkString(appender)
+
+    if (matchAsMd5) {
+      Hex.toHexString(digestMd5.digest(keys.getBytes()))
+    } else keys
   }
 
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-key_in_redis
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - smdmts
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-21 00:00:00.000000000 Z
+date: 2017-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -62,7 +62,6 @@ files:
 - src/main/scala/org/embulk/filter/key_in_redis/KeyInRedisFilterPlugin.scala
 - src/main/scala/org/embulk/filter/key_in_redis/PageOutput.scala
 - src/main/scala/org/embulk/filter/key_in_redis/PluginTask.scala
-- src/main/scala/org/embulk/filter/key_in_redis/column/PassthroughColumnVisitor.scala
 - src/main/scala/org/embulk/filter/key_in_redis/column/SetValueColumnVisitor.scala
 - src/main/scala/org/embulk/filter/key_in_redis/json/JsonParser.scala
 - src/main/scala/org/embulk/filter/key_in_redis/redis/Cache.scala
@@ -82,7 +81,7 @@ files:
 - classpath/circe-numbers_2.11-0.8.0.jar
 - classpath/circe-parser_2.11-0.8.0.jar
 - classpath/config-1.2.1.jar
-- classpath/embulk-filter-key_in_redis-0.1.1.jar
+- classpath/embulk-filter-key_in_redis-0.1.2.jar
 - classpath/jawn-parser_2.11-0.10.4.jar
 - classpath/machinist_2.11-0.6.1.jar
 - classpath/macro-compat_2.11-1.1.1.jar
@@ -95,7 +94,7 @@ files:
 - classpath/scalaz-effect_2.11-7.2.14.jar
 - classpath/shapeless_2.11-2.3.2.jar
 - classpath/simulacrum_2.11-0.10.0.jar
-homepage:
+homepage: https://github.com/smdmts/embulk-filter-key_in_redis
 licenses:
 - MIT
 metadata: {}

data/src/main/scala/org/embulk/filter/key_in_redis/column/PassthroughColumnVisitor.scala

@@ -1,57 +0,0 @@
-package org.embulk.filter.key_in_redis.column
-
-import org.embulk.spi.{
-  Column,
-  PageBuilder,
-  PageReader,
-  ColumnVisitor => EmbulkColumnVisitor
-}
-
-case class PassthroughColumnVisitor(pageReader: PageReader,
-                                    pageBuilder: PageBuilder)
-    extends EmbulkColumnVisitor {
-
-  override def timestampColumn(column: Column): Unit =
-    if (pageReader.isNull(column)) {
-      pageBuilder.setNull(column)
-    } else {
-      pageBuilder.setTimestamp(column, pageReader.getTimestamp(column))
-    }
-
-  override def stringColumn(column: Column): Unit =
-    if (pageReader.isNull(column)) {
-      pageBuilder.setNull(column)
-    } else {
-      pageBuilder.setString(column, pageReader.getString(column))
-    }
-
-  override def longColumn(column: Column): Unit =
-    if (pageReader.isNull(column)) {
-      pageBuilder.setNull(column)
-    } else {
-      pageBuilder.setLong(column, pageReader.getLong(column))
-    }
-
-  override def doubleColumn(column: Column): Unit =
-    if (pageReader.isNull(column)) {
-      pageBuilder.setNull(column)
-    } else {
-      pageBuilder.setDouble(column, pageReader.getDouble(column))
-    }
-
-  override def booleanColumn(column: Column): Unit =
-    if (pageReader.isNull(column)) {
-      pageBuilder.setNull(column)
-    } else {
-      pageBuilder.setBoolean(column, pageReader.getBoolean(column))
-    }
-
-  override def jsonColumn(column: Column): Unit =
-    if (pageReader.isNull(column)) {
-      pageBuilder.setNull(column)
-    } else {
-      pageBuilder.setJson(column, pageReader.getJson(column))
-    }
-
-  def addRecord(): Unit = pageBuilder.addRecord()
-}