embulk-parser-xpath2 0.0.4 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2aebf3205f36802a6d5240064e69da2e4481ab32
-  data.tar.gz: c74a3fb9310df91b0695bc83a2495bbac12860e0
+  metadata.gz: fcd8e55e48a3ea51f186ca9a66440db2a454c877
+  data.tar.gz: 8816fe3c8e734055993ecba64179c68414bccf6d
 SHA512:
-  metadata.gz: e5d7656c75d2c8c7a82d266fa26fba9730f798551d0e573c1924d9ab40d3debc40f3aec0a9219327e23c5e8282be3e9d9b206768b6783d7d95cb137e784984ea
-  data.tar.gz: 07d9a54720290242e298724c80743bb75997e6d1149fad566380195c282e6e8860c18a07bf810949ca5e522283d19cabcdc9a2062dffa6272d4d33f78302b40a
+  metadata.gz: 751d5eacce78225b0a07e2925ccfca0ee09c6bfc3a01cbaa96331881f8e503170ae3974b2548deba12395aa92c1fbe909c291ae146771c11fe8690a1da137cdc
+  data.tar.gz: 9e5c7072613b5df19dcffbb0a7d354c8b1b7c514934134683ba7ba7e268a2f22d359710d6ff51397a978f5c8709e5ee18d28bb15e1f5903c7ab5a884d833837c

data/benchmark/src/main/scala/org/embulk/parser/xpath2/ParseBenchmark.scala

@@ -41,7 +41,7 @@ class ParseBenchmark  {
 
 object ParseBenchmark {
 
-  val TestRecordSize = 1000
+  val TestRecordSize = 100 * 1000
 
   val test = new XPath2ParserPluginSpec()
   val runtime = new EmbulkTestRuntime

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.0.4"
+version = "0.1.0"
 ext {
     embulkVersion = "0.8.35"
 }
@@ -22,6 +22,7 @@ sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
 dependencies {
+    compile "com.ximpleware:vtd-xml:2.13.4"
     compile  "org.embulk:embulk-core:${embulkVersion}"
     provided "org.embulk:embulk-core:${embulkVersion}"
     testCompile "org.embulk:embulk-core:${embulkVersion}:tests"

data/build.sbt

@@ -16,6 +16,7 @@ lazy val commonSettings = Seq(
   ),
   resolvers += Resolver.jcenterRepo,
   libraryDependencies ++= Seq(
+    "com.ximpleware" % "vtd-xml" % "2.13.4",
     "org.embulk" % "embulk-core" % embulkVersion,
     "org.embulk" % "embulk-core" % embulkVersion classifier "tests",
     "junit" % "junit" % "4.+" % "test",

data/src/main/scala/org/embulk/parser/xpath2/XPath2ParserPlugin.scala

@@ -1,10 +1,7 @@
 package org.embulk.parser.xpath2
 
-import java.util
-import javax.xml.namespace.NamespaceContext
-import javax.xml.parsers.{DocumentBuilder, DocumentBuilderFactory}
-import javax.xml.xpath.{XPathConstants, XPathExpression, XPathFactory}
-
+import com.google.common.io.ByteStreams
+import com.ximpleware.{AutoPilot, VTDGen, VTDNav}
 import org.embulk.config._
 import org.embulk.parser.xpath2.config.ColumnConfig
 import org.embulk.spi._
@@ -13,21 +10,14 @@ import org.embulk.spi.time.TimestampParser
 import org.embulk.spi.util.FileInputInputStream
 import org.msgpack.value.{Value, Variable}
 import org.slf4j.Logger
-import org.w3c.dom.{Document, Node, NodeList}
 
+import scala.annotation.tailrec
 import scala.collection.JavaConverters._
-import scala.collection.immutable
 import scala.util.control.NonFatal
 
 class XPath2ParserPlugin extends ParserPlugin {
 
-  val logger: Logger = Exec.getLogger(classOf[XPath2ParserPlugin])
-
-  def docBuilder: DocumentBuilder = {
-    val factory: DocumentBuilderFactory = DocumentBuilderFactory.newInstance
-    factory.setNamespaceAware(true)
-    factory.newDocumentBuilder()
-  }
+  private[this] val logger: Logger = Exec.getLogger(classOf[XPath2ParserPlugin])
 
   override def transaction(config: ConfigSource, control: ParserPlugin.Control): Unit = {
     val task = config.loadConfig(classOf[PluginTask])
@@ -42,73 +32,83 @@ class XPath2ParserPlugin extends ParserPlugin {
     val task: PluginTask = taskSource.loadTask(classOf[PluginTask])
     val stopOnInvalidRecord: Boolean = task.getStopOnInvalidRecord
 
-    val xPathInstance = XPathFactory.newInstance.newXPath()
-    xPathInstance.setNamespaceContext(new NamespaceContext {
-      override def getPrefix(namespaceURI: String): String = task.getNamespaces.conf.asScala.collectFirst { case (_, v) if v == namespaceURI => v }.orNull
-      override def getPrefixes(namespaceURI: String): util.Iterator[_] = task.getNamespaces.conf.asScala.keys.asJava.iterator()
-      override def getNamespaceURI(prefix: String): String = task.getNamespaces.conf.asScala(prefix)
-    })
-
-    val rootXPath: XPathExpression = xPathInstance.compile(task.getRoot)
-    val columnXPaths: immutable.Seq[XPathExpression] = task.getSchema.columns.asScala.map(x => xPathInstance.compile(x.path)).toList
-
     val timestampParsers: Map[String, TimestampParser] = task.getSchema.columns.asScala
       .collect { case ColumnConfig(_, name, _, Some(timestampColumnOption), _) => (name, new TimestampParser(task, timestampColumnOption)) }.toMap
+    val columnsWithIndex: Seq[(ColumnConfig, Int)] = task.getSchema.columns.asScala.zipWithIndex
+
+    val vg = new VTDGen
 
     LoanPattern(new PageBuilder(Exec.getBufferAllocator, schema, output)) { pb =>
       while (input.nextFile()) {
-        parseXML(input) match {
-          case Right(doc) =>
-            val rootNodes = rootXPath.evaluate(doc, XPathConstants.NODESET).asInstanceOf[NodeList]
-            (0 until rootNodes.getLength).map(rootNodes.item).foreach { node =>
-              columnXPaths.zipWithIndex.foreach { case (xPath, idx) =>
+        LoanPattern(new FileInputInputStream(input)) { fiis =>
+
+          vg.setDoc(ByteStreams.toByteArray(fiis))
+          vg.parse(true)
+
+          val nav = vg.getNav
+          val rootElementAutoPilot = new AutoPilot(nav)
+          val columnElementAutoPilot = new AutoPilot(nav)
+          task.getNamespaces.conf.asScala.foreach { case (prefix, namespaceURI) =>
+            rootElementAutoPilot.declareXPathNameSpace(prefix, namespaceURI)
+            columnElementAutoPilot.declareXPathNameSpace(prefix, namespaceURI)
+          }
+
+          @tailrec
+          def execEachRecord(rootAp: AutoPilot): Unit = if (rootAp.evalXPath() != -1) {
+            nav.push()
+            try {
+              columnsWithIndex.foreach { case (columnConfig, idx) =>
+                nav.push()
+                columnElementAutoPilot.selectXPath(columnConfig.path)
                 val column = schema.getColumn(idx)
-                handleColumn(pb, node, xPath, column, timestampParsers)
+                handleColumn(pb, nav, columnElementAutoPilot, column, timestampParsers)
+                nav.pop()
               }
               pb.addRecord()
+            } catch {
+              case NonFatal(e) => if (stopOnInvalidRecord) {
+                throw new DataException(e)
+              } else {
+                logger.warn(s"Skipped invalid record $e")
+              }
             }
-          case Left(e) =>
-            if(stopOnInvalidRecord) {
-              throw new DataException(e)
-            } else {
-              logger.warn(s"Skipped invalid record $e")
-            }
+            nav.pop()
+            execEachRecord(rootAp)
+          }
+
+          rootElementAutoPilot.selectXPath(task.getRoot)
+          execEachRecord(rootElementAutoPilot)
         }
+
         pb.flush()
       }
       pb.finish()
-      pb.close()
-    }
-  }
-
-  def parseXML(input: FileInput): Either[Throwable, Document] = {
-    val stream = new FileInputInputStream(input)
-    try {
-      Right(docBuilder.parse(stream))
-    } catch {
-      case NonFatal(e) => Left(e)
     }
   }
 
-  def handleColumn(pb: PageBuilder, node: Node, xPath: XPathExpression, column: Column, timestampParsers: Map[String, TimestampParser]): Unit = {
+  final def handleColumn(pb: PageBuilder, nav: VTDNav, columnAp: AutoPilot, column: Column, timestampParsers: Map[String, TimestampParser]): Unit = {
     if (column.getType.isInstanceOf[JsonType]) {
-      val value: NodeList = xPath.evaluate(node, XPathConstants.NODESET).asInstanceOf[NodeList]
-      val values: Seq[Value] = (0 until value.getLength).map(value.item).map { valueNode =>
-        new Variable().setStringValue(valueNode.getTextContent).asStringValue()
+      val list = new java.util.ArrayList[Value]()
+      @tailrec
+      def eachJsonValue(cAp: AutoPilot): Unit = if (cAp.evalXPath() != -1) {
+        val index = nav.getText
+        if (index != -1) list.add(new Variable().setStringValue(nav.toString(index)).asStringValue())
+        eachJsonValue(cAp)
       }
-      val jsonValue = new Variable().setArrayValue(values.asJava).asArrayValue()
+      eachJsonValue(columnAp)
+      val jsonValue = new Variable().setArrayValue(list).asArrayValue()
       pb.setJson(column, jsonValue)
     } else {
-      val value: Node = xPath.evaluate(node, XPathConstants.NODE).asInstanceOf[Node]
-      if (value == null) {
+      if (columnAp.evalXPath() == -1) {
         pb.setNull(column)
       } else {
-        setColumn(pb, column, value.getTextContent, timestampParsers)
+        val index = nav.getText
+        setColumn(pb, column, nav.toString(index), timestampParsers)
       }
     }
   }
 
-  def setColumn(pb: PageBuilder, column: Column, value: String, timestampParsers: Map[String, TimestampParser]): Unit = column.getType match {
+  final def setColumn(pb: PageBuilder, column: Column, value: String, timestampParsers: Map[String, TimestampParser]): Unit = column.getType match {
     case _: StringType => pb.setString(column, value)
     case _: LongType => pb.setLong(column, value.toLong)
     case _: DoubleType => pb.setDouble(column, value.toDouble)

data/src/test/resources/invalid-data.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0"?>
+<ns1:root
+        xmlns:ns1="http://example.com/ns1/"
+        xmlns:ns2="http://example.com/ns2/">
+    <ns2:entry>
+        <ns2:id>1</ns2:id>
+        <ns2:title>Hello!</ns2:title>
+        <ns2:meta>
+            <ns2:author>maji-KY</ns2:author>
+        </ns2:meta>
+        <ns2:date>20010101</ns2:date>
+        <ns2:dateTime>2000-12-31 15:00:00</ns2:dateTime>
+        <ns2:list>
+            <ns2:value>a</ns2:value>
+            <ns2:value>b</ns2:value>
+            <ns2:value>c</ns2:value>
+        </ns2:list>
+        <ns2:rating by="subscribers">2.5</ns2:rating>
+        <ns2:rating>3.5</ns2:rating>
+        <ns2:released>true</ns2:released>
+    </ns2:entry>
+    <ns2:entry>
+        <ns2:id>2.5</ns2:id>
+        <ns2:title>Bonjour!</ns2:title>
+        <ns2:meta>
+            <ns2:author>maji-KY</ns2:author>
+        </ns2:meta>
+        <ns2:date>20010101</ns2:date>
+        <ns2:list></ns2:list>
+        <ns2:rating>3.5</ns2:rating>
+        <ns2:released>false</ns2:released>
+    </ns2:entry>
+</ns1:root>

data/src/test/scala/org/embulk/parser/xpath2/XPath2ParserPluginSpec.scala

@@ -22,6 +22,7 @@ class XPath2ParserPluginSpec {
   def runtime = new EmbulkTestRuntime
 
   val dataPath: String = classOf[XPath2ParserPlugin].getClassLoader.getResource("data.xml").getPath
+  val invalidDataPath: String = classOf[XPath2ParserPlugin].getClassLoader.getResource("invalid-data.xml").getPath
 
   def configSource: ConfigSource = Exec.newConfigSource()
     .set("in", Map[String, String]("type" -> "file", "path_prefix" -> dataPath).asJava)
@@ -39,14 +40,15 @@ class XPath2ParserPluginSpec {
     .set("namespaces", Map[String, String]("ns1" -> "http://example.com/ns1/", "ns2" -> "http://example.com/ns2/").asJava)
     .set("out", Map[String, String]("type" -> "stdout").asJava)
 
-  @Test def test() {
+  @Test def testParseXML() {
 
-    val task = configSource.loadConfig(classOf[PluginTask])
+    val cs = configSource
+    val task = cs.loadConfig(classOf[PluginTask])
 
     var schema: Schema = null
 
     val plugin = new XPath2ParserPlugin()
-    plugin.transaction(configSource, (_: TaskSource, s: Schema) => {schema = s})
+    plugin.transaction(cs, (_: TaskSource, s: Schema) => {schema = s})
 
     val result: mutable.Buffer[collection.mutable.Map[String, Any]] = mutable.Buffer()
 
@@ -83,8 +85,65 @@ class XPath2ParserPluginSpec {
     ), result)
   }
 
+  @Test(expected = classOf[DataException]) def testStopOnInvalid() {
+
+    val cs = configSource.set("stop_on_invalid_record", true)
+    val task = cs.loadConfig(classOf[PluginTask])
+
+    var schema: Schema = null
+
+    val plugin = new XPath2ParserPlugin()
+    plugin.transaction(cs, (_: TaskSource, s: Schema) => {schema = s})
+
+    val result: mutable.Buffer[collection.mutable.Map[String, Any]] = mutable.Buffer()
+
+    plugin.run(
+      task.dump(),
+      schema,
+      new InputStreamFileInput(Exec.getBufferAllocator(), new FileInputStream(new File(invalidDataPath))),
+      new TestTransactionalPageOutput(schema, result)
+    )
+
+  }
+
+  @Test() def testSkipOnInvalid() {
+
+    val cs = configSource
+    val task = cs.loadConfig(classOf[PluginTask])
+
+    var schema: Schema = null
+
+    val plugin = new XPath2ParserPlugin()
+    plugin.transaction(cs, (_: TaskSource, s: Schema) => {schema = s})
+
+    val result: mutable.Buffer[collection.mutable.Map[String, Any]] = mutable.Buffer()
+
+    plugin.run(
+      task.dump(),
+      schema,
+      new InputStreamFileInput(Exec.getBufferAllocator(), new FileInputStream(new File(invalidDataPath))),
+      new TestTransactionalPageOutput(schema, result)
+    )
+
+
+    assertEquals(ArrayBuffer(
+      Map(
+        "id" -> 1L,
+        "title" -> "Hello!",
+        "author" -> "maji-KY",
+        "date" -> Timestamp.ofEpochSecond(978274800L),
+        "date_time" -> Timestamp.ofEpochSecond(978274800L),
+        "list" -> new JsonParser().parse("""["a","b","c"]"""),
+        "rating_sub" -> 2.5d,
+        "released" -> true,
+      )
+    ), result)
+  }
+
 }
 
+
+
 class TestTransactionalPageOutput(schema: Schema, result: mutable.Buffer[collection.mutable.Map[String, Any]])
   extends TransactionalPageOutput {
   import org.embulk.spi.PageReader

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-xpath2
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.0
 platform: ruby
 authors:
 - maji-KY
@@ -66,10 +66,12 @@ files:
 - src/main/scala/org/embulk/parser/xpath2/config/NamespacesConfig.scala
 - src/main/scala/org/embulk/parser/xpath2/config/SchemaConfig.scala
 - src/test/resources/data.xml
+- src/test/resources/invalid-data.xml
 - src/test/scala/org/embulk/parser/xpath2/UnitSpec.scala
 - src/test/scala/org/embulk/parser/xpath2/XPath2ParserPluginSpec.scala
+- classpath/vtd-xml-2.13.4.jar
 - classpath/scala-library-2.12.4.jar
-- classpath/embulk-parser-xpath2-0.0.4.jar
+- classpath/embulk-parser-xpath2-0.1.0.jar
 homepage: https://github.com/maji-KY/embulk-parser-xpath2
 licenses:
 - MIT