embulk-output-aerospike 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dfe48bb0f3d7b1b99ea715a70fe95aaa59e4c11c
-  data.tar.gz: 6f7e98a99ef16ba823ffac4116a52d54334eb28e
+  metadata.gz: bb2c68c087cb8d343a0a775e7d8207b2543c3538
+  data.tar.gz: 7c21ed64bcb6d35afad9a86641725d90360bf17c
 SHA512:
-  metadata.gz: 0b85f656d793aed7e1f6fc68e877b261c11276ab224a71b065392aabbea7f75f64043f5e97dbba20ca00247e0193827c65fae7c1c4622061f60f00e3b7818efe
-  data.tar.gz: b33ca3cfad0112dd8d358dd653f0c19d0f8e7ddbb3975819b2e1c7584cc0731930f4a15b7f5957cbc170b76548674641f7b106a9ef25a0ee1b2a7f65df5e60e6
+  metadata.gz: 7efe78f207491420be5db28ab336b7595b2e20a076bef531a9553bace433874d6bde16080acd41534ff709f7cdc4c837574e935f2f6a03cdef513e71520d718e
+  data.tar.gz: 7ffbdb387bba2251233f212f29d1b113054fe7475a15a9f5b1e41ab80f552519911c081d884e6aa54da6532faf109b077621fc4d9f6a1e358c61a7354a5b1aff

data/build.gradle

@@ -17,12 +17,13 @@ configurations {
     provided
 }
 
-version = "0.2.1"
+version = "0.3.0"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.5"
     provided "org.embulk:embulk-core:0.7.5"
     compile 'org.scala-lang:scala-library:2.11.7'
+    compile 'org.scalaz.stream:scalaz-stream_2.11:0.8'
     compile 'com.github.tkrs:aerospiker-core_2.11:0.4.0-SNAPSHOT'
     compile 'com.github.tkrs:aerospiker-msgpack_2.11:0.4.0-SNAPSHOT'
     compile 'com.github.tkrs:aerospiker-task_2.11:0.4.0-SNAPSHOT'

data/src/main/scala/org/embulk/output/aerospike/AerospikeOutputPlugin.scala

@@ -4,10 +4,7 @@ import java.util.{ List => JList, Map => JMap }
 
 import com.google.common.base.Optional
 import org.embulk.config._
-import org.embulk.spi.Exec
-import org.embulk.spi.OutputPlugin
-import org.embulk.spi.Schema
-import org.embulk.spi.TransactionalPageOutput
+import org.embulk.spi._
 
 object AerospikeOutputPlugin {
 
@@ -97,6 +94,7 @@ object AerospikeOutputPlugin {
 class AerospikeOutputPlugin extends OutputPlugin {
   import OutputPlugin._
   import AerospikeOutputPlugin._
+  import scala.collection.JavaConversions._
 
   def transaction(config: ConfigSource, schema: Schema, taskCount: Int, control: Control): ConfigDiff = {
     val task = config.loadConfig(classOf[PluginTask])
@@ -104,7 +102,7 @@ class AerospikeOutputPlugin extends OutputPlugin {
     Exec.newConfigDiff
   }
 
-  def resume(taskSource: TaskSource, schema: Schema, taskCount: Int, control: OutputPlugin.Control): ConfigDiff =
+  def resume(taskSource: TaskSource, schema: Schema, taskCount: Int, control: Control): ConfigDiff =
     throw new UnsupportedOperationException("aerospike output plugin does not support resuming")
 
   def open(taskSource: TaskSource, schema: Schema, taskIndex: Int): TransactionalPageOutput =

data/src/main/scala/org/embulk/output/aerospike/AerospikePageOutput.scala

@@ -1,5 +1,7 @@
 package org.embulk.output.aerospike
 
+import java.util.concurrent.atomic.AtomicLong
+
 import aerospiker._
 import aerospiker.policy.{ ClientPolicy, WritePolicy }
 import aerospiker.task.Aerospike
@@ -7,29 +9,25 @@ import cats.data.Xor, Xor._
 import io.circe._, io.circe.syntax._
 import org.embulk.config.TaskReport
 import org.embulk.config.TaskSource
-import org.embulk.spi.Exec
-import org.embulk.spi.PageReader
-import org.embulk.spi.Schema
-import org.embulk.spi.TransactionalPageOutput
 import org.embulk.spi._
-import java.util.concurrent.atomic.AtomicLong
+import org.embulk.spi.`type`.Type
+import org.embulk.spi.time.Timestamp
 
-import scala.collection.mutable.{ Map => MMap }
+import scala.collection.mutable.{ Map => MMap, ListBuffer }
 import scala.collection.JavaConversions._
-
-import org.slf4j.Logger
-
-import scalaz.{\/-, -\/}
+import scalaz.concurrent.Task
+import scalaz.stream._
 
 class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int) extends TransactionalPageOutput {
 
-  import AerospikeOutputPlugin._
+  import org.embulk.output.aerospike.ops._
 
-  private[this] val log: Logger = Exec.getLogger(classOf[AerospikePageOutput])
-  private[this] val tsk: AerospikeOutputPlugin.PluginTask = taskSource.loadTask(classOf[AerospikeOutputPlugin.PluginTask])
-  private[this] val successCount: AtomicLong = new AtomicLong
-  private[this] val failCount: AtomicLong = new AtomicLong
+  import AerospikeOutputPlugin._
 
+  private[this] val log = Exec.getLogger(classOf[AerospikePageOutput])
+  private[this] val tsk = taskSource.loadTask(classOf[AerospikeOutputPlugin.PluginTask])
+  private[this] val successCount = new AtomicLong
+  private[this] val failCount = new AtomicLong
 
   private[this] val wp: WritePolicy = {
     if (tsk.getWritePolicy.isPresent) {
@@ -70,7 +68,6 @@ class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int
     override protected def namespace: String = tsk.getNamespace
     override protected def setName: String = tsk.getSetName
   }
-  private[this] val reader: PageReader = new PageReader(schema)
 
   private[this] def toJson(a: Any): Json = a match {
     case v: Boolean => v.asJson
@@ -85,87 +82,110 @@ class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int
 
   implicit val encoder = Encoder.instance[Any](toJson)
 
-  def add(page: Page) {
+  implicit private[this] val reader: PageReader = new PageReader(schema)
+
+  val createRecords: Page => Process[Task, Seq[Seq[Col]]] = { (page) =>
     reader.setPage(page)
-    while (reader.nextRecord()) {
-      val record: MMap[String, Any] = MMap.empty
-      val schema = reader.getSchema
-      schema.getColumns.foreach { col =>
-        val name = col.getName
-        col.getType.getName match {
-          case "long" => if (!(reader isNull col))
-            record += name -> reader.getLong(col)
-          case "double" => if (!(reader isNull col))
-            record += name -> reader.getDouble(col)
-          case "timestamp" => if (!(reader isNull col))
-            record += name -> reader.getTimestamp(col).toEpochMilli
-          case "boolean" => if (!(reader isNull col))
-            record += name -> reader.getBoolean(col)
-          case "string" => if (!(reader isNull col)) {
-            val cv = reader.getString(col)
-            if (tsk.getSplitters.isPresent) {
-              val sps = tsk.getSplitters.get.toMap
-              sps.get(name) match {
-                case None => //
-                  record += name -> cv
-                case Some(v) =>
-                  val sep = v.getSeparator
-                  v.getElementType match {
-                    case "long" =>
-                      val x = cv.split(sep).map(s => if (s.isEmpty) "0" else s).map(_.toLong)
-                      record += name -> x
-                    case "double" =>
-                      val x = cv.split(sep).map(s => if (s.isEmpty) "0" else s).map(_.toDouble)
-                      record += name -> x
-                    case "string" =>
-                      val x = cv.split(sep)
-                      record += name -> x
-                  }
+    Process.eval {
+      Task.delay {
+        val records: ListBuffer[Seq[Col]] = ListBuffer.empty
+        while (reader.nextRecord())
+          records += (for (col <- schema.getColumns.toStream) yield Col of col).toSeq
+        records.toSeq
+      }
+    }
+  }
+
+  val toRecords: Seq[Seq[Col]] => Seq[Map[String, Any]] = _ map { row =>
+    val rec: MMap[String, Any] = MMap.empty
+    row foreach {
+      case DoubleColumn(i, n, v) => rec += n -> v
+      case LongColumn(i, n, v) => rec += n -> v
+      case StringColumn(i, n, v) =>
+        if (tsk.getSplitters.isPresent) {
+          val sps = tsk.getSplitters.get.toMap
+          sps.get(n) match {
+            case None => //
+              rec += n -> v
+            case Some(sp) =>
+              val sep = sp.getSeparator
+              sp.getElementType match {
+                case "long" =>
+                  val x = v.split(sep).map(s => if (s.isEmpty) "0" else s).map(_.toLong)
+                  rec += n -> x
+                case "double" =>
+                  val x = v.split(sep).map(s => if (s.isEmpty) "0" else s).map(_.toDouble)
+                  rec += n -> x
+                case "string" =>
+                  val x = v.split(sep)
+                  rec += n -> x
               }
-            } else {
-              record += name -> cv
-            }
           }
-          case typ => log.error(typ + "[?]")
+        } else {
+          rec += n -> v
         }
+      case BooleanColumn(i, n, v) => rec += n -> v
+      case TimestampColumn(i, n, v) => rec += n -> v
+      case NullColumn(i, n, t) => // nop
+    }
+    rec.toMap
+  }
+
+  val updater: Sink[Task, Seq[Map[String, Any]]] = sink.lift[Task, Seq[Map[String, Any]]] { records =>
+    val t = Task.gatherUnordered {
+      records map { record =>
+        val keyObj = record.getOrElse(tsk.getKeyName.get, "")
+        val deRec = record - tsk.getKeyName.get
+        if (tsk.getSingleBinName.isPresent)
+          aerospike.put(keyObj.toString, Map(tsk.getSingleBinName.get() -> deRec))
+        else
+          aerospike.put(keyObj.toString, deRec)
       }
-      val keyObj = record.getOrElse(tsk.getKeyName.get, "")
-      record -= tsk.getKeyName.get
-      tsk.getCommand match {
-        case "put" =>
-          val t =
-            if (tsk.getSingleBinName.isPresent)
-              aerospike.put(keyObj.toString, Map(tsk.getSingleBinName.get() -> record.toMap))
-            else
-              aerospike.put(keyObj.toString, record.toMap)
-          t runAsync {
-            case -\/(e) => log.error(e.toString, e.getCause)
-            case \/-(x) => x match {
-              case Left(ex) =>
-                failCount.addAndGet(1L)
-                log.error(ex.toString, ex.getCause)
-              case Right(v) =>
-                successCount.addAndGet(1L)
-                log.debug(v.toString)
-            }
-          }
-        case "delete" =>
-          val t = aerospike.delete(keyObj.toString)
-          t runAsync {
-            case -\/(e) => log.error(e.toString, e.getCause)
-            case \/-(x) => x match {
-              case Left(ex) =>
-                failCount.addAndGet(1L)
-                log.error(ex.toString, ex.getCause)
-              case Right(v) =>
-                successCount.addAndGet(1L)
-                log.debug(v.toString)
-            }
-          }
+    } run
+
+    Task.delay {
+      for ( r <- t ) {
+        r match {
+          case Left(e) =>
+            log.error(e.toString, e)
+            failCount.addAndGet(1L)
+          case Right(_) =>
+            successCount.addAndGet(1L)
+        }
       }
     }
   }
 
+  val deleter: Sink[Task, Seq[Map[String, Any]]] = sink.lift[Task, Seq[Map[String, Any]]] { records =>
+    val t = Task.gatherUnordered {
+      records map { record =>
+        val keyObj = record.getOrElse(tsk.getKeyName.get, "")
+        aerospike.delete(keyObj.toString)
+      }
+    } run
+
+    Task.delay {
+      for ( r <- t ) {
+        r match {
+          case Left(e) =>
+            log.error(e.key, e)
+            failCount.addAndGet(1L)
+          case Right(_) =>
+            successCount.addAndGet(1L)
+        }
+      }
+    }
+  }
+
+  def add(page: Page) {
+    tsk.getCommand match {
+      case "put" =>
+        createRecords(page).takeWhile(_.nonEmpty).map(toRecords).to(updater).runLog.run
+      case "delete" =>
+        createRecords(page).takeWhile(_.nonEmpty).map(toRecords).to(deleter).runLog.run
+    }
+  }
+
   def finish(): Unit = log.info(s"finish ${tsk.getCommand} ok[${successCount.longValue}] ng[${failCount.longValue()}]")
 
   def close(): Unit = {
@@ -177,3 +197,33 @@ class AerospikePageOutput(taskSource: TaskSource, schema: Schema, taskIndex: Int
 
   def commit: TaskReport = Exec.newTaskReport
 }
+
+object ops {
+
+  sealed trait Col
+
+  object Col {
+    def of(c: Column)(implicit r: PageReader) =
+      if (r isNull c) NullColumn(c.getIndex, c.getName, c.getType)
+      else c.getType.getName match {
+        case "string" =>
+          StringColumn(c.getIndex, c.getName, r.getString(c))
+        case "double" =>
+          DoubleColumn(c.getIndex, c.getName, r.getDouble(c))
+        case "long" =>
+          LongColumn(c.getIndex, c.getName, r.getLong(c))
+        case "boolean" =>
+          BooleanColumn(c.getIndex, c.getName, r.getBoolean(c))
+        case "timestamp" =>
+          TimestampColumn(c.getIndex, c.getName, r.getTimestamp(c))
+      }
+  }
+
+  case class DoubleColumn(index: Int, name: String, value: Double) extends Col
+  case class StringColumn(index: Int, name: String, value: String) extends Col
+  case class BooleanColumn(index: Int, name: String, value: Boolean) extends Col
+  case class TimestampColumn(index: Int, name: String, value: Timestamp) extends Col
+  case class LongColumn(index: Int, name: String, value: Long) extends Col
+  case class NullColumn(index: Int, name: String, typ: Type) extends Col
+
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-aerospike
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Takeru Sato
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-15 00:00:00.000000000 Z
+date: 2015-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -77,7 +77,7 @@ files:
 - classpath/circe-jawn_2.11-0.1.1.jar
 - classpath/commons-math3-3.2.jar
 - classpath/discipline_2.11-0.3.jar
-- classpath/embulk-output-aerospike-0.2.1.jar
+- classpath/embulk-output-aerospike-0.3.0.jar
 - classpath/gnu-crypto-2.0.1.jar
 - classpath/jawn-parser_2.11-0.8.0.jar
 - classpath/jbcrypt-0.3m.jar
@@ -94,9 +94,11 @@ files:
 - classpath/scala-reflect-2.11.2.jar
 - classpath/scala-xml_2.11-1.0.4.jar
 - classpath/scalacheck_2.11-1.12.4.jar
-- classpath/scalaz-concurrent_2.11-7.1.3.jar
-- classpath/scalaz-core_2.11-7.1.3.jar
-- classpath/scalaz-effect_2.11-7.1.3.jar
+- classpath/scalaz-concurrent_2.11-7.1.4.jar
+- classpath/scalaz-core_2.11-7.1.4.jar
+- classpath/scalaz-effect_2.11-7.1.4.jar
+- classpath/scalaz-stream_2.11-0.8.jar
+- classpath/scodec-bits_2.11-1.0.9.jar
 - classpath/shapeless_2.11-2.2.5.jar
 - classpath/simulacrum_2.11-0.3.0.jar
 - classpath/test-interface-1.0.jar