embulk-parser-xpath2 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e137d05fc96ea14274af6132b979b748f179f13f
-  data.tar.gz: b0c73e40ec2c784d6d3c74e4ad034e87c145163c
+  metadata.gz: d53fe77af6f9aa2bd412603c52395679efcd3ea5
+  data.tar.gz: d65abd7ce1c01795be0360dae44a53e7598e49da
 SHA512:
-  metadata.gz: 646bc1169d84714eee9908bf20963a625d387b1145f9fcb57c32a1257a92b977373a848b2e3657d206714458a8d17c0bb74489e6b07d103329f4a3b6eefb1810
-  data.tar.gz: 4e41dca4a70dd30489e689d678656911292dcf40d90cdf2a1332149ba964df11c9cedd06e3b49eb28428b5f31021f5284e28b6b64ee3e97894ee970c73d69891
+  metadata.gz: cc2a9082b36e2270c4dc5269be02dae1ffebb08d848d338deaf4d4d1b2b630ef6285d28d7f6d4e11cf4fe915e1c79f2c03fea2154e3714aec40e2c514ac37f11
+  data.tar.gz: 5a6c07331cdc7ccbc96a4d5fbbb10457be84ddd0d22f7367d09c353f2de9114442699d477cc2bc4f0ce5b46a711e9e6c6747c3bc119e34152dbe9fa1ef6ec2f7

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.0.1"
+version = "0.0.2"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/src/main/scala/org/embulk/parser/xpath2/XPath2ParserPlugin.scala

@@ -11,6 +11,7 @@ import org.embulk.spi._
 import org.embulk.spi.`type`._
 import org.embulk.spi.time.TimestampParser
 import org.embulk.spi.util.FileInputInputStream
+import org.msgpack.value.{Value, Variable}
 import org.slf4j.Logger
 import org.w3c.dom.{Document, Node, NodeList}
 
@@ -61,13 +62,8 @@ class XPath2ParserPlugin extends ParserPlugin {
             val rootNodes = rootXPath.evaluate(doc, XPathConstants.NODESET).asInstanceOf[NodeList]
             (0 until rootNodes.getLength).map(rootNodes.item).foreach { node =>
               columnXPaths.zipWithIndex.foreach { case (xPath, idx) =>
-                val value: Node = xPath.evaluate(node, XPathConstants.NODE).asInstanceOf[Node]
                 val column = schema.getColumn(idx)
-                if (value == null) {
-                  pb.setNull(column)
-                } else {
-                  setColumn(pb, column, value.getTextContent, timestampParsers)
-                }
+                handleColumn(pb, node, xPath, column, timestampParsers)
               }
               pb.addRecord()
             }
@@ -94,12 +90,29 @@ class XPath2ParserPlugin extends ParserPlugin {
     }
   }
 
+  def handleColumn(pb: PageBuilder, node: Node, xPath: XPathExpression, column: Column, timestampParsers: Map[String, TimestampParser]): Unit = {
+    if (column.getType.isInstanceOf[JsonType]) {
+      val value: NodeList = xPath.evaluate(node, XPathConstants.NODESET).asInstanceOf[NodeList]
+      val values: Seq[Value] = (0 until value.getLength).map(value.item).map { valueNode =>
+        new Variable().setStringValue(valueNode.getTextContent).asStringValue()
+      }
+      val jsonValue = new Variable().setArrayValue(values.asJava).asArrayValue()
+      pb.setJson(column, jsonValue)
+    } else {
+      val value: Node = xPath.evaluate(node, XPathConstants.NODE).asInstanceOf[Node]
+      if (value == null) {
+        pb.setNull(column)
+      } else {
+        setColumn(pb, column, value.getTextContent, timestampParsers)
+      }
+    }
+  }
+
   def setColumn(pb: PageBuilder, column: Column, value: String, timestampParsers: Map[String, TimestampParser]): Unit = column.getType match {
     case _: StringType => pb.setString(column, value)
     case _: LongType => pb.setLong(column, value.toLong)
     case _: DoubleType => pb.setDouble(column, value.toDouble)
     case _: BooleanType => pb.setBoolean(column, value.toBoolean)
-    case _: JsonType => pb.setString(column, value) // treat json as string.
     case _: TimestampType => pb.setTimestamp(column, timestampParsers(column.getName).parse(value))
   }
 

data/src/test/resources/data.xml

@@ -9,5 +9,10 @@
             <ns2:author>maji-KY</ns2:author>
         </ns2:meta>
         <ns2:date>20010101</ns2:date>
+        <ns2:list>
+            <ns2:value>a</ns2:value>
+            <ns2:value>b</ns2:value>
+            <ns2:value>c</ns2:value>
+        </ns2:list>
     </ns2:entry>
 </ns1:root>

data/src/test/scala/org/embulk/parser/xpath2/XPath2ParserPluginSpec.scala

@@ -6,6 +6,7 @@ import java.util
 import org.embulk.EmbulkTestRuntime
 import org.embulk.config.{ConfigSource, TaskSource}
 import org.embulk.spi._
+import org.embulk.spi.json.JsonParser
 import org.embulk.spi.time.Timestamp
 import org.embulk.spi.util.InputStreamFileInput
 import org.junit.Assert._
@@ -31,7 +32,8 @@ class XPath2ParserPluginSpec {
         Map("path" -> "ns2:id", "name" -> "id", "type" -> "long").asJava,
         Map("path" -> "ns2:title", "name" -> "title", "type" -> "string").asJava,
         Map("path" -> "ns2:meta/ns2:author", "name" -> "author", "type" -> "string").asJava,
-        Map("path" -> "ns2:date", "name" -> "date", "type" -> "timestamp", "format" -> "%Y%m%d").asJava
+        Map("path" -> "ns2:date", "name" -> "date", "type" -> "timestamp", "format" -> "%Y%m%d").asJava,
+        Map("path" -> "ns2:list/ns2:value", "name" -> "list", "type" -> "json").asJava,
       ).asJava)
       .set("namespaces", Map[String, String]("ns1" -> "http://example.com/ns1/", "ns2" -> "http://example.com/ns2/").asJava)
       .set("out", Map[String, String]("type" -> "stdout").asJava)
@@ -108,7 +110,7 @@ class XPath2ParserPluginSpec {
                   if (reader.isNull(column)) {
                     record.put(column.getName, null)
                   } else {
-                    record.put(column.getName, reader.getString(column))
+                    record.put(column.getName, reader.getJson(column))
                   }
                 }
               })
@@ -128,7 +130,7 @@ class XPath2ParserPluginSpec {
 
     println(result)
 
-    assertEquals(ArrayBuffer(Map("date" -> Timestamp.ofEpochSecond(978307200L), "title" -> "Hello!", "author" -> "maji-KY", "id" -> 1L)), result)
+    assertEquals(ArrayBuffer(Map("date" -> Timestamp.ofEpochSecond(978307200L), "list" -> new JsonParser().parse("""["a","b","c"]"""), "title" -> "Hello!", "author" -> "maji-KY", "id" -> 1L)), result)
   }
 
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-xpath2
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - maji-KY
@@ -64,7 +64,7 @@ files:
 - src/test/resources/data.xml
 - src/test/scala/org/embulk/parser/xpath2/UnitSpec.scala
 - src/test/scala/org/embulk/parser/xpath2/XPath2ParserPluginSpec.scala
-- classpath/embulk-parser-xpath2-0.0.1.jar
+- classpath/embulk-parser-xpath2-0.0.2.jar
 - classpath/scala-library-2.12.3.jar
 homepage: https://github.com/maji-KY/embulk-parser-xpath2
 licenses: