embulk-output-key_to_redis 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 57b3d726f99858c48efbd8330847c5559aefeeb0
-  data.tar.gz: 47c6ffb23ff8fe4855b3ae454a934e99f1e82b8e
+  metadata.gz: cacc8741d9141b1fc1de001245082f1542849df9
+  data.tar.gz: f91d317b22fc2a87d30044d14f0498a1bf39e368
 SHA512:
-  metadata.gz: a1414c7c8d0a857993158ef9676928eb0669fe127b89806867c8e6b961869a5b2e1e44be43841332522a30298b417c223e3786aa567de9daf975d1c96d4f647c
-  data.tar.gz: d681bb485688f4c644765fd0c3725b1b7c099b5da1d71e8b11c1c89fbc01c758cce26b322d0b878c4e9815f158f7b69b978095b6a992ea6dc4a2a5d839835e1d
+  metadata.gz: cfeb481de84ff42d12b0c361b989736851bdf007ae73b25ea1de18837b2a676e3a35465c642cc82c887b7943404c5ed3a6c00a3064582761f6b1591a5bb05514
+  data.tar.gz: 3fcfac83f63fbbfc9cd7e2b5142d3ddd7bffe0158d0d8c5935fd2fab2ea40d37190061e2bfb30b81ddf6ceaff3bc77e0a3a5780e2d88c21a3ae9f64770c8d804

data/.circleci/config.yml

@@ -0,0 +1,27 @@
+version: 2
+jobs:
+  build:
+    executorType: docker
+    docker:
+      - image: hseeberger/scala-sbt
+    working_directory: /root/embulk-output-key_to_redis/
+    steps:
+      - checkout
+      - restore_cache:
+          name: Restoring Cache
+          keys:
+            - sbt
+      - setup_remote_docker
+      - run:
+          name: prepare
+          command: sbt update exit
+      - save_cache:
+          name: Saving Cache sbt
+          key: sbt
+          paths:
+            - "/root/.sbt"
+            - "/root/.ivy2"
+      - run:
+          name: compile
+          command: |
+            sbt compile test scalafmt::test exit

data/README.md

@@ -1,6 +1,6 @@
 # Key To Redis output plugin for Embulk
 
-Output the aggregated key into set of the Redis plugin.  
+Generate the aggregated key from input values and output to Redis's SET value.
 
 This plugin is designed to extract data set diff files used with the combination in below use cases.
 

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.1"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/scala/org/embulk/output/key_to_redis/PageOutput.scala

@@ -8,8 +8,12 @@ import org.embulk.output.key_to_redis.column._
 import org.embulk.spi.time.TimestampFormatter
 import org.embulk.spi._
 import org.bouncycastle.util.encoders.Hex
+import org.embulk.output.key_to_redis.redis.Redis
 
 import scala.collection.JavaConverters._
+import scala.concurrent.duration._
+import scala.collection.mutable.ListBuffer
+import scala.concurrent.{Await, Future}
 
 case class PageOutput(taskSource: TaskSource,
                       schema: Schema,
@@ -21,6 +25,10 @@ case class PageOutput(taskSource: TaskSource,
   def timestampFormatter(): TimestampFormatter =
     new TimestampFormatter(task, Optional.absent())
 
+  val buffer = new ListBuffer[Future[Long]]
+  val redis: Redis =
+    KeyToRedisOutputPlugin.redis.getOrElse(sys.error("could not find redis."))
+
   override def add(page: Page): Unit = {
     val reader: PageReader = new PageReader(schema)
     reader.setPage(page)
@@ -37,17 +45,21 @@ case class PageOutput(taskSource: TaskSource,
         if (putAsMD5) {
           val hash = Hex.toHexString(
             digestMd5.digest(setValueVisitor.getValue.getBytes()))
-          KeyToRedisOutputPlugin.redis.foreach(_.sadd(hash))
+          buffer.append(redis.sadd(hash))
         } else {
-          KeyToRedisOutputPlugin.redis.foreach(
-            _.sadd(setValueVisitor.getValue))
+          buffer.append(redis.sadd(setValueVisitor.getValue))
         }
       }
     }
     reader.close()
   }
 
-  override def finish(): Unit = ()
+  override def finish(): Unit = {
+    import scala.concurrent.ExecutionContext.Implicits.global
+    val sequence = Future.sequence(buffer)
+    Await.result(sequence, Duration.Inf)
+  }
+
   override def close(): Unit = ()
   override def commit(): TaskReport = Exec.newTaskReport
   override def abort(): Unit = ()

data/src/main/scala/org/embulk/output/key_to_redis/redis/Redis.scala

@@ -23,9 +23,8 @@ case class Redis(setKey: String, host: String, port: Int, db: Option[Int]) {
     }
     Await.result(s, 10.minute)
   }
-  def sadd(value: String): Long = {
-    val s = redis.sadd(setKey, value)
-    Await.result(s, 10.minute)
+  def sadd(value: String): Future[Long] = {
+    redis.sadd(setKey, value)
   }
 
   def flush(): Boolean = {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-key_to_redis
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - smdmts
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-14 00:00:00.000000000 Z
+date: 2017-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -45,6 +45,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- .circleci/config.yml
 - .gitignore
 - LICENSE.txt
 - README.md
@@ -77,7 +78,7 @@ files:
 - classpath/circe-numbers_2.11-0.8.0.jar
 - classpath/circe-parser_2.11-0.8.0.jar
 - classpath/config-1.2.1.jar
-- classpath/embulk-output-key_to_redis-0.1.0.jar
+- classpath/embulk-output-key_to_redis-0.1.1.jar
 - classpath/jawn-parser_2.11-0.10.4.jar
 - classpath/machinist_2.11-0.6.1.jar
 - classpath/macro-compat_2.11-1.1.1.jar