embulk-output-aerospike 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 13abe937c76a27f33bc023971f909464cbcb616b
-  data.tar.gz: 3ad97e3420a42ca3de5af08b1d2c2586d3ad67b8
+  metadata.gz: c4a1acaffea6df8cbf6035f159eed41ca0309754
+  data.tar.gz: e663c8a6e0d909ecb991eeffc75e8565491b4ea0
 SHA512:
-  metadata.gz: 00f36b928015f69e3af871a52fdecfdbbd0d688ebb3d1143c2e8d8f1e7dca9d27467991c16349daebb24c4abc8c0805ef6727a8840ad0b91c300fed39a031c24
-  data.tar.gz: 94f0b593e7c7e926ba85a613723e1bf95fba2c6d0812bd4e4efee2890848b881dd92ac7df5a39f1a0b4815ac124ff4173019852be89a8f01308ded40a2881d20
+  metadata.gz: fe02878851b13d8fc585d93298f90436aedef1135c1b87cc5c90ea0fe01854ba019761189acd8fb20458f369058bc1702727a7e47bdc55d8eb73bc72c639ccae
+  data.tar.gz: 2482bb08fa0d7956ea94027f68311b17d4fcde9a260015ac49339ceb92d8c01792ede4c1bcec10c3931e4a15f675ff35b815301849762b71b2c598e610420cbd

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
     provided
 }
 
-version = "0.3.1"
+version = "0.3.2"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.5"

data/src/main/scala/org/embulk/output/aerospike/AerospikePageOutput.scala

@@ -1,5 +1,6 @@
 package org.embulk.output.aerospike
 
+import java.util.concurrent.{ CountDownLatch, ConcurrentLinkedQueue }
 import java.util.concurrent.atomic.AtomicLong
 
 import aerospiker._
@@ -16,6 +17,7 @@ import org.embulk.spi.time.Timestamp
 import scala.collection.concurrent.TrieMap
 import scala.collection.mutable.{ Map => MMap, ListBuffer }
 import scala.collection.JavaConversions._
+import scalaz.{ \/-, -\/ }
 import scalaz.concurrent.Task
 import scalaz.stream._
 
@@ -134,53 +136,63 @@ class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int
   }
 
   val updater: Sink[Task, Seq[Map[String, Any]]] = sink.lift[Task, Seq[Map[String, Any]]] { records =>
-    val t = Task.gatherUnordered {
-      records map { record =>
-        val keyObj = record.getOrElse(tsk.getKeyName.get, "")
-        val deRec = record - tsk.getKeyName.get
-        if (tsk.getSingleBinName.isPresent)
-          aerospike.put(keyObj.toString, Map(tsk.getSingleBinName.get() -> deRec))
-        else
-          aerospike.put(keyObj.toString, deRec)
-      }
-    } run
+    val latch = new CountDownLatch(records.size)
+    val queue = new ConcurrentLinkedQueue[Throwable Xor String]()
+    records foreach { record =>
+      val keyObj = record.getOrElse(tsk.getKeyName.get, "")
+      val deRec = record - tsk.getKeyName.get
+      if (tsk.getSingleBinName.isPresent)
+        aerospike.put(keyObj.toString, Map(tsk.getSingleBinName.get() -> deRec)) runAsync {
+          case -\/(e) => queue.add(Xor.left(e)); latch.countDown()
+          case \/-(r) => queue.add(r); latch.countDown()
+        }
+      else
+        aerospike.put(keyObj.toString, deRec) runAsync {
+          case -\/(e) => queue.add(Xor.left(e)); latch.countDown()
+          case \/-(r) => queue.add(r); latch.countDown()
+        }
+    }
+
+    latch.await()
 
     Task.delay {
-      for ( r <- t ) {
-        r match {
-          case Left(e @ PutError(key, cause)) =>
-            log.error(e.toString, e)
-            failures += key -> cause.getMessage
-            failCount.addAndGet(1L)
-          case Left(e) =>
-            log.error(e.toString, e)
-            failures += e.getMessage -> e.getMessage
-            failCount.addAndGet(1L)
-          case Right(_) =>
-            successCount.addAndGet(1L)
-        }
+      queue foreach {
+        case Left(e @ PutError(key, cause)) =>
+          log.error(e.toString, e)
+          failures += key -> cause.getMessage
+          failCount.addAndGet(1L)
+        case Left(e) =>
+          log.error(e.toString, e)
+          failures += e.getMessage -> e.getMessage
+          failCount.addAndGet(1L)
+        case Right(_) =>
+          successCount.addAndGet(1L)
       }
     }
   }
 
   val deleter: Sink[Task, Seq[Map[String, Any]]] = sink.lift[Task, Seq[Map[String, Any]]] { records =>
-    val t = Task.gatherUnordered {
-      records map { record =>
-        val keyObj = record.getOrElse(tsk.getKeyName.get, "")
-        aerospike.delete(keyObj.toString)
+    val latch = new CountDownLatch(records.size)
+    val queue = new ConcurrentLinkedQueue[DeleteError Xor Boolean]()
+    records foreach { record =>
+      val keyObj = record.getOrElse(tsk.getKeyName.get, "")
+      val key = keyObj.toString
+      aerospike.delete(key) runAsync {
+        case -\/(e) => queue.add(Xor.left(DeleteError(key, e))); latch.countDown()
+        case \/-(r) => queue.add(r); latch.countDown()
       }
-    } run
+    }
+
+    latch.await()
 
     Task.delay {
-      for ( r <- t ) {
-        r match {
-          case Left(DeleteError(key, cause)) =>
-            log.error(key, cause)
-            failures += key -> cause.getMessage
-            failCount.addAndGet(1L)
-          case Right(_) =>
-            successCount.addAndGet(1L)
-        }
+      queue foreach {
+        case Left(DeleteError(key, cause)) =>
+          log.error(key, cause)
+          failures += key -> cause.getMessage
+          failCount.addAndGet(1L)
+        case Right(_) =>
+          successCount.addAndGet(1L)
       }
     }
   }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-aerospike
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Takeru Sato
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-19 00:00:00.000000000 Z
+date: 2015-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -77,7 +77,7 @@ files:
 - classpath/circe-jawn_2.11-0.1.1.jar
 - classpath/commons-math3-3.2.jar
 - classpath/discipline_2.11-0.3.jar
-- classpath/embulk-output-aerospike-0.3.1.jar
+- classpath/embulk-output-aerospike-0.3.2.jar
 - classpath/gnu-crypto-2.0.1.jar
 - classpath/jawn-parser_2.11-0.8.0.jar
 - classpath/jbcrypt-0.3m.jar