embulk-output-aerospike 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bb2c68c087cb8d343a0a775e7d8207b2543c3538
-  data.tar.gz: 7c21ed64bcb6d35afad9a86641725d90360bf17c
+  metadata.gz: 13abe937c76a27f33bc023971f909464cbcb616b
+  data.tar.gz: 3ad97e3420a42ca3de5af08b1d2c2586d3ad67b8
 SHA512:
-  metadata.gz: 7efe78f207491420be5db28ab336b7595b2e20a076bef531a9553bace433874d6bde16080acd41534ff709f7cdc4c837574e935f2f6a03cdef513e71520d718e
-  data.tar.gz: 7ffbdb387bba2251233f212f29d1b113054fe7475a15a9f5b1e41ab80f552519911c081d884e6aa54da6532faf109b077621fc4d9f6a1e358c61a7354a5b1aff
+  metadata.gz: 00f36b928015f69e3af871a52fdecfdbbd0d688ebb3d1143c2e8d8f1e7dca9d27467991c16349daebb24c4abc8c0805ef6727a8840ad0b91c300fed39a031c24
+  data.tar.gz: 94f0b593e7c7e926ba85a613723e1bf95fba2c6d0812bd4e4efee2890848b881dd92ac7df5a39f1a0b4815ac124ff4173019852be89a8f01308ded40a2881d20

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
     provided
 }
 
-version = "0.3.0"
+version = "0.3.1"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.5"

data/src/main/scala/org/embulk/output/aerospike/AerospikeOutputPlugin.scala

@@ -98,8 +98,7 @@ class AerospikeOutputPlugin extends OutputPlugin {
 
   def transaction(config: ConfigSource, schema: Schema, taskCount: Int, control: Control): ConfigDiff = {
     val task = config.loadConfig(classOf[PluginTask])
-    control.run(task.dump)
-    Exec.newConfigDiff
+    control.run(task.dump).foldRight(Exec.newConfigDiff) { (l, r) => r.merge(l) }
   }
 
   def resume(taskSource: TaskSource, schema: Schema, taskCount: Int, control: Control): ConfigDiff =

data/src/main/scala/org/embulk/output/aerospike/AerospikePageOutput.scala

@@ -4,7 +4,7 @@ import java.util.concurrent.atomic.AtomicLong
 
 import aerospiker._
 import aerospiker.policy.{ ClientPolicy, WritePolicy }
-import aerospiker.task.Aerospike
+import aerospiker.task.{ DeleteError, PutError, Aerospike }
 import cats.data.Xor, Xor._
 import io.circe._, io.circe.syntax._
 import org.embulk.config.TaskReport
@@ -13,6 +13,7 @@ import org.embulk.spi._
 import org.embulk.spi.`type`.Type
 import org.embulk.spi.time.Timestamp
 
+import scala.collection.concurrent.TrieMap
 import scala.collection.mutable.{ Map => MMap, ListBuffer }
 import scala.collection.JavaConversions._
 import scalaz.concurrent.Task
@@ -28,6 +29,7 @@ class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int
   private[this] val tsk = taskSource.loadTask(classOf[AerospikeOutputPlugin.PluginTask])
   private[this] val successCount = new AtomicLong
   private[this] val failCount = new AtomicLong
+  private[this] val failures = TrieMap.empty[String, String]
 
   private[this] val wp: WritePolicy = {
     if (tsk.getWritePolicy.isPresent) {
@@ -146,8 +148,13 @@ class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int
     Task.delay {
       for ( r <- t ) {
         r match {
+          case Left(e @ PutError(key, cause)) =>
+            log.error(e.toString, e)
+            failures += key -> cause.getMessage
+            failCount.addAndGet(1L)
           case Left(e) =>
             log.error(e.toString, e)
+            failures += e.getMessage -> e.getMessage
             failCount.addAndGet(1L)
           case Right(_) =>
             successCount.addAndGet(1L)
@@ -167,8 +174,9 @@ class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int
     Task.delay {
       for ( r <- t ) {
         r match {
-          case Left(e) =>
-            log.error(e.key, e)
+          case Left(DeleteError(key, cause)) =>
+            log.error(key, cause)
+            failures += key -> cause.getMessage
             failCount.addAndGet(1L)
           case Right(_) =>
             successCount.addAndGet(1L)
@@ -195,7 +203,12 @@ class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int
 
   def abort(): Unit = log.error(s"abort ${tsk.getCommand} ok[${successCount.longValue}] ng[${failCount.longValue()}]")
 
-  def commit: TaskReport = Exec.newTaskReport
+  def commit: TaskReport = {
+    var r = Exec.newTaskReport
+    r.set("rans", successCount.longValue() + failCount.longValue())
+    r.set("failures", failures.toMap.asJson.pretty(Printer.noSpaces))
+    r
+  }
 }
 
 object ops {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-aerospike
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Takeru Sato
@@ -77,7 +77,7 @@ files:
 - classpath/circe-jawn_2.11-0.1.1.jar
 - classpath/commons-math3-3.2.jar
 - classpath/discipline_2.11-0.3.jar
-- classpath/embulk-output-aerospike-0.3.0.jar
+- classpath/embulk-output-aerospike-0.3.1.jar
 - classpath/gnu-crypto-2.0.1.jar
 - classpath/jawn-parser_2.11-0.8.0.jar
 - classpath/jbcrypt-0.3m.jar