embulk-parser-xpath2 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3c6ca2b4389dd1a0c784bd26735599b07bca338a
-  data.tar.gz: f9514cf861f316e940779590896042e8b005ab4a
+  metadata.gz: 2aebf3205f36802a6d5240064e69da2e4481ab32
+  data.tar.gz: c74a3fb9310df91b0695bc83a2495bbac12860e0
 SHA512:
-  metadata.gz: 62cb373a04138dba8e690592f23a15259eb9d95bda20c9a1cffdd3f6e4da5e36e897bd4931258f576dd5e09007e67907e0b92c98df1caba41d6eb4c4ec02ab1b
-  data.tar.gz: 44c6b99ddbdedbe24d42e4dc9fc2bdf70ee6c14032feeed191a582f190d0e8bfbf5a73236def253d7528a4fc5c7113aa3181bbe636d4c94d545a585a69b646d4
+  metadata.gz: e5d7656c75d2c8c7a82d266fa26fba9730f798551d0e573c1924d9ab40d3debc40f3aec0a9219327e23c5e8282be3e9d9b206768b6783d7d95cb137e784984ea
+  data.tar.gz: 07d9a54720290242e298724c80743bb75997e6d1149fad566380195c282e6e8860c18a07bf810949ca5e522283d19cabcdc9a2062dffa6272d4d33f78302b40a

data/.gitignore

@@ -9,8 +9,7 @@ build/
 /.settings/
 /.metadata/
 .classpath
-.project
+project/project
 /bin/
-project
 *.iml
 out

data/.travis.yml

@@ -0,0 +1,21 @@
+language: scala
+scala:
+  - 2.12.4
+jdk:
+  - oraclejdk8
+script:
+  - gradle test
+  - gradle gem
+
+before_deploy:
+  - echo '---' > ~/.gem/credentials
+  - echo ':rubygems_api_key:' ${RUBYGEMS_API_KEY} > ~/.gem/credentials
+  - chmod 0600 ~/.gem/credentials
+
+deploy:
+  provider: script
+  script:
+    - gradle gemPush
+  on:
+    tags: true
+    all_branches: true

data/README.md

@@ -1,4 +1,6 @@
 # Xml parser plugin for Embulk
+[![Gem Version](https://badge.fury.io/rb/embulk-parser-xpath2.svg)](https://badge.fury.io/rb/embulk-parser-xpath2)
+[![Build Status](https://travis-ci.org/maji-KY/embulk-parser-xpath2.svg?branch=develop)](https://travis-ci.org/maji-KY/embulk-parser-xpath2)
 
 Embulk parser plugin for parsing xml data by XPath perfectly!
 
@@ -54,3 +56,9 @@ Then you can fetch entries from the following xml:
 ```
 $ ./gradlew gem
 ```
+
+## Benchmark
+
+```
+$ sbt benchmark/jmh:run
+```

data/benchmark/src/main/scala/org/embulk/parser/xpath2/ParseBenchmark.scala

@@ -0,0 +1,99 @@
+package org.embulk.parser.xpath2
+
+import java.io.{InputStream, PipedInputStream, PipedOutputStream}
+
+import org.embulk.EmbulkTestRuntime
+import org.embulk.config.TaskSource
+import org.embulk.spi.util.InputStreamFileInput
+import org.embulk.spi.{Exec, Schema}
+import org.openjdk.jmh.annotations.Benchmark
+
+import scala.collection.mutable
+
+class ParseBenchmark  {
+  import ParseBenchmark._
+
+  @Benchmark
+  def run(): Unit = {
+    Exec.doWith(runtime.getExec, () => {
+      val configSource = test.configSource
+      val task = configSource.loadConfig(classOf[PluginTask])
+
+      var schema: Schema = null
+
+      val plugin = new XPath2ParserPlugin()
+      plugin.transaction(configSource, (_: TaskSource, s: Schema) => {schema = s})
+
+      val result: mutable.Buffer[collection.mutable.Map[String, Any]] = mutable.Buffer()
+
+      plugin.run(
+        task.dump(),
+        schema,
+        new InputStreamFileInput(Exec.getBufferAllocator(), testDataInput),
+        new TestTransactionalPageOutput(schema, result)
+      )
+
+      require(result.size == TestRecordSize)
+    })
+  }
+
+}
+
+object ParseBenchmark {
+
+  val TestRecordSize = 1000
+
+  val test = new XPath2ParserPluginSpec()
+  val runtime = new EmbulkTestRuntime
+
+  val testDataXmlEntry =
+    """    <ns2:entry>
+      |        <ns2:id>1</ns2:id>
+      |        <ns2:title>Hello!</ns2:title>
+      |        <ns2:meta>
+      |            <ns2:author>maji-KY</ns2:author>
+      |        </ns2:meta>
+      |        <ns2:date>20010101</ns2:date>
+      |        <ns2:dateTime>2000-12-31 15:00:00</ns2:dateTime>
+      |        <ns2:list>
+      |            <ns2:value>a</ns2:value>
+      |            <ns2:value>b</ns2:value>
+      |            <ns2:value>c</ns2:value>
+      |        </ns2:list>
+      |        <ns2:rating by="subscribers">2.5</ns2:rating>
+      |        <ns2:rating>3.5</ns2:rating>
+      |        <ns2:released>true</ns2:released>
+      |    </ns2:entry>
+    """.stripMargin.getBytes
+
+  def testDataInput: InputStream = {
+    val header =
+      """<?xml version="1.0"?>
+        |<ns1:root
+        |        xmlns:ns1="http://example.com/ns1/"
+        |        xmlns:ns2="http://example.com/ns2/">
+      """.stripMargin.getBytes
+    val footer =
+      """
+        |</ns1:root>""".stripMargin.getBytes
+
+    val pipedOut = new PipedOutputStream
+    val pipedIn = new PipedInputStream(pipedOut)
+    new Thread() {
+      override def run(): Unit = {
+        pipedOut.write(header)
+        1 to TestRecordSize foreach { _ =>
+          pipedOut.write(testDataXmlEntry)
+        }
+        pipedOut.write(footer)
+        pipedOut.close()
+      }
+    }.start()
+    pipedIn
+  }
+
+  def main(args: Array[String]): Unit = {
+    new ParseBenchmark().run()
+  }
+
+}

data/build.gradle

@@ -13,19 +13,22 @@ configurations {
     provided
 }
 
-version = "0.0.3"
+version = "0.0.4"
+ext {
+    embulkVersion = "0.8.35"
+}
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.32"
-    provided "org.embulk:embulk-core:0.8.32"
-    testCompile "org.embulk:embulk-core:0.8.32:tests"
-    testCompile "org.embulk:embulk-standards:0.8.32"
+    compile  "org.embulk:embulk-core:${embulkVersion}"
+    provided "org.embulk:embulk-core:${embulkVersion}"
+    testCompile "org.embulk:embulk-core:${embulkVersion}:tests"
+    testCompile "org.embulk:embulk-standards:${embulkVersion}"
     testCompile "junit:junit:4.+"
 
-    compile group: 'org.scala-lang', name: 'scala-library', version: '2.12.3'
+    compile group: 'org.scala-lang', name: 'scala-library', version: '2.12.4'
     testCompile group: 'org.scalatest', name: 'scalatest_2.12', version: '3.0.4'
 
 }
@@ -37,6 +40,13 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
 }
 clean { delete "classpath" }
 
+test {
+    testLogging {
+        events 'failed'
+        exceptionFormat 'full'
+    }
+}
+
 task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
     jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
     script "${project.name}.gemspec"

data/build.sbt

@@ -1,23 +1,34 @@
-lazy val root = (project in file(".")).
-  settings(
-    inThisBuild(List(
-      organization := "com.github.maji-KY",
-      scalaVersion := "2.12.3",
-      version := "0.0.1-SNAPSHOT"
-    )),
-    name := "embulk-parser-xpath2",
-    scalacOptions ++= Seq(
-      "-deprecation",
-      "-feature",
-      "-unchecked",
-      "-Xlint",
-      "-Ywarn-dead-code",
-      "-Ywarn-numeric-widen",
-      "-Ywarn-unused",
-      "-Ywarn-value-discard"
-    )
+val embulkVersion = "0.8.35"
+
+lazy val commonSettings = Seq(
+  organization := "com.github.maji-KY",
+  scalaVersion := "2.12.4",
+  version := "CANNOT_RELEASE",
+  scalacOptions ++= Seq(
+    "-deprecation",
+    "-feature",
+    "-unchecked",
+    "-Xlint",
+    "-Ywarn-dead-code",
+    "-Ywarn-numeric-widen",
+    "-Ywarn-unused",
+    "-Ywarn-value-discard"
+  ),
+  resolvers += Resolver.jcenterRepo,
+  libraryDependencies ++= Seq(
+    "org.embulk" % "embulk-core" % embulkVersion,
+    "org.embulk" % "embulk-core" % embulkVersion classifier "tests",
+    "junit" % "junit" % "4.+" % "test",
+    "org.scalatest" %% "scalatest" % "3.0.4" % "test"
   )
+)
+
+lazy val benchmark = (project in file("benchmark"))
+  .aggregate(main)
+  .settings(commonSettings)
+  .dependsOn(main % "compile->test")
+  .enablePlugins(JmhPlugin)
 
-resolvers += Resolver.jcenterRepo
+lazy val main = (project in file("."))
+  .settings(commonSettings)
 
-libraryDependencies ++= Dependencies.value

data/project/build.properties

@@ -0,0 +1 @@
+sbt.version=1.0.1

data/project/plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.2.27")

data/src/test/resources/data.xml

@@ -9,10 +9,25 @@
             <ns2:author>maji-KY</ns2:author>
         </ns2:meta>
         <ns2:date>20010101</ns2:date>
+        <ns2:dateTime>2000-12-31 15:00:00</ns2:dateTime>
         <ns2:list>
             <ns2:value>a</ns2:value>
             <ns2:value>b</ns2:value>
             <ns2:value>c</ns2:value>
         </ns2:list>
+        <ns2:rating by="subscribers">2.5</ns2:rating>
+        <ns2:rating>3.5</ns2:rating>
+        <ns2:released>true</ns2:released>
+    </ns2:entry>
+    <ns2:entry>
+        <ns2:id>2</ns2:id>
+        <ns2:title>Bonjour!</ns2:title>
+        <ns2:meta>
+            <ns2:author>maji-KY</ns2:author>
+        </ns2:meta>
+        <ns2:date>20010101</ns2:date>
+        <ns2:list></ns2:list>
+        <ns2:rating>3.5</ns2:rating>
+        <ns2:released>false</ns2:released>
     </ns2:entry>
 </ns1:root>

data/src/test/scala/org/embulk/parser/xpath2/XPath2ParserPluginSpec.scala

@@ -23,20 +23,23 @@ class XPath2ParserPluginSpec {
 
   val dataPath: String = classOf[XPath2ParserPlugin].getClassLoader.getResource("data.xml").getPath
 
-  @Test def test() {
+  def configSource: ConfigSource = Exec.newConfigSource()
+    .set("in", Map[String, String]("type" -> "file", "path_prefix" -> dataPath).asJava)
+    .set("root", "/ns1:root/ns2:entry")
+    .set("schema", List[util.Map[String, String]](
+      Map("path" -> "ns2:id", "name" -> "id", "type" -> "long").asJava,
+      Map("path" -> "ns2:title", "name" -> "title", "type" -> "string").asJava,
+      Map("path" -> "ns2:meta/ns2:author", "name" -> "author", "type" -> "string").asJava,
+      Map("path" -> "ns2:date", "name" -> "date", "type" -> "timestamp", "format" -> "%Y%m%d", "timezone" -> "Asia/Tokyo").asJava,
+      Map("path" -> "ns2:dateTime", "name" -> "date_time", "type" -> "timestamp", "format" -> "%Y-%m-%d %H:%M:%S", "timezone" -> "UTC").asJava,
+      Map("path" -> "ns2:list/ns2:value", "name" -> "list", "type" -> "json").asJava,
+      Map("path" -> "ns2:rating[@by='subscribers']", "name" -> "rating_sub", "type" -> "double").asJava,
+      Map("path" -> "ns2:released", "name" -> "released", "type" -> "boolean").asJava,
+    ).asJava)
+    .set("namespaces", Map[String, String]("ns1" -> "http://example.com/ns1/", "ns2" -> "http://example.com/ns2/").asJava)
+    .set("out", Map[String, String]("type" -> "stdout").asJava)
 
-    val configSource: ConfigSource = Exec.newConfigSource()
-      .set("in", Map[String, String]("type" -> "file", "path_prefix" -> dataPath).asJava)
-      .set("root", "/ns1:root/ns2:entry")
-      .set("schema", List[util.Map[String, String]](
-        Map("path" -> "ns2:id", "name" -> "id", "type" -> "long").asJava,
-        Map("path" -> "ns2:title", "name" -> "title", "type" -> "string").asJava,
-        Map("path" -> "ns2:meta/ns2:author", "name" -> "author", "type" -> "string").asJava,
-        Map("path" -> "ns2:date", "name" -> "date", "type" -> "timestamp", "format" -> "%Y%m%d", "timezone" -> "UTC").asJava,
-        Map("path" -> "ns2:list/ns2:value", "name" -> "list", "type" -> "json").asJava,
-      ).asJava)
-      .set("namespaces", Map[String, String]("ns1" -> "http://example.com/ns1/", "ns2" -> "http://example.com/ns2/").asJava)
-      .set("out", Map[String, String]("type" -> "stdout").asJava)
+  @Test def test() {
 
     val task = configSource.loadConfig(classOf[PluginTask])
 
@@ -51,86 +54,108 @@ class XPath2ParserPluginSpec {
       task.dump(),
       schema,
       new InputStreamFileInput(Exec.getBufferAllocator(), new FileInputStream(new File(dataPath))),
-      new TransactionalPageOutput() {
-
-        import org.embulk.spi.PageReader
-
-        val reader = new PageReader(schema)
-
-        override def add(page: Page) = {
-          reader.setPage(page)
-
-          while (reader.nextRecord()) {
-            val record: collection.mutable.Map[String, Any] = collection.mutable.Map()
-
-            schema.getColumns().asScala.foreach { column =>
-
-              column.visit(new ColumnVisitor() {
-                override def timestampColumn(column: Column): Unit = {
-                  if (reader.isNull(column)) {
-                    record.put(column.getName, null)
-                  } else {
-                    record.put(column.getName, reader.getTimestamp(column))
-                  }
-                }
-
-                override def stringColumn(column: Column): Unit = {
-                  if (reader.isNull(column)) {
-                    record.put(column.getName, null)
-                  } else {
-                    record.put(column.getName, reader.getString(column))
-                  }
-                }
-
-                override def longColumn(column: Column): Unit = {
-                  if (reader.isNull(column)) {
-                    record.put(column.getName, null)
-                  } else {
-                    record.put(column.getName, reader.getLong(column))
-                  }
-                }
-
-                override def doubleColumn(column: Column): Unit = {
-                  if (reader.isNull(column)) {
-                    record.put(column.getName, null)
-                  } else {
-                    record.put(column.getName, reader.getDouble(column))
-                  }
-                }
-
-                override def booleanColumn(column: Column): Unit = {
-                  if (reader.isNull(column)) {
-                    record.put(column.getName, null)
-                  } else {
-                    record.put(column.getName, reader.getBoolean(column))
-                  }
-                }
-
-                override def jsonColumn(column: Column): Unit = {
-                  if (reader.isNull(column)) {
-                    record.put(column.getName, null)
-                  } else {
-                    record.put(column.getName, reader.getJson(column))
-                  }
-                }
-              })
-
-
-            }
-            result += record
-          }
-        }
-
-        override def commit() = Exec.newTaskReport()
-        override def abort() = {}
-        override def finish() = {}
-        override def close() = {}
-      }
+      new TestTransactionalPageOutput(schema, result)
     )
 
     println(result)
 
-    assertEquals(ArrayBuffer(Map("date" -> Timestamp.ofEpochSecond(978307200L), "list" -> new JsonParser().parse("""["a","b","c"]"""), "title" -> "Hello!", "author" -> "maji-KY", "id" -> 1L)), result)
+    assertEquals(ArrayBuffer(
+      Map(
+        "id" -> 1L,
+        "title" -> "Hello!",
+        "author" -> "maji-KY",
+        "date" -> Timestamp.ofEpochSecond(978274800L),
+        "date_time" -> Timestamp.ofEpochSecond(978274800L),
+        "list" -> new JsonParser().parse("""["a","b","c"]"""),
+        "rating_sub" -> 2.5d,
+        "released" -> true,
+      ),
+      Map(
+        "id" -> 2L,
+        "title" -> "Bonjour!",
+        "author" -> "maji-KY",
+        "date" -> Timestamp.ofEpochSecond(978274800L),
+        "date_time" -> null,
+        "list" -> new JsonParser().parse("[]"),
+        "rating_sub" -> null,
+        "released" -> false,
+      )
+    ), result)
   }
 
 }
+
+class TestTransactionalPageOutput(schema: Schema, result: mutable.Buffer[collection.mutable.Map[String, Any]])
+  extends TransactionalPageOutput {
+  import org.embulk.spi.PageReader
+
+  val reader = new PageReader(schema)
+
+  override def add(page: Page) = {
+    reader.setPage(page)
+
+    while (reader.nextRecord()) {
+      val record: collection.mutable.Map[String, Any] = collection.mutable.Map()
+
+      schema.getColumns().asScala.foreach { column =>
+        column.visit(new TestColumnVisitor(reader, record))
+      }
+      result += record
+    }
+  }
+
+  override def commit() = Exec.newTaskReport()
+  override def abort() = {}
+  override def finish() = {}
+  override def close() = {}
+}
+
+class TestColumnVisitor(reader: PageReader, record: collection.mutable.Map[String, Any]) extends ColumnVisitor {
+  override def timestampColumn(column: Column): Unit = {
+    if (reader.isNull(column)) {
+      record.put(column.getName, null)
+    } else {
+      record.put(column.getName, reader.getTimestamp(column))
+    }
+  }
+
+  override def stringColumn(column: Column): Unit = {
+    if (reader.isNull(column)) {
+      record.put(column.getName, null)
+    } else {
+      record.put(column.getName, reader.getString(column))
+    }
+  }
+
+  override def longColumn(column: Column): Unit = {
+    if (reader.isNull(column)) {
+      record.put(column.getName, null)
+    } else {
+      record.put(column.getName, reader.getLong(column))
+    }
+  }
+
+  override def doubleColumn(column: Column): Unit = {
+    if (reader.isNull(column)) {
+      record.put(column.getName, null)
+    } else {
+      record.put(column.getName, reader.getDouble(column))
+    }
+  }
+
+  override def booleanColumn(column: Column): Unit = {
+    if (reader.isNull(column)) {
+      record.put(column.getName, null)
+    } else {
+      record.put(column.getName, reader.getBoolean(column))
+    }
+  }
+
+  override def jsonColumn(column: Column): Unit = {
+    if (reader.isNull(column)) {
+      record.put(column.getName, null)
+    } else {
+      record.put(column.getName, reader.getJson(column))
+    }
+  }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-xpath2
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - maji-KY
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-10-06 00:00:00.000000000 Z
+date: 2017-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -46,8 +46,10 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - LICENSE
 - README.md
+- benchmark/src/main/scala/org/embulk/parser/xpath2/ParseBenchmark.scala
 - build.gradle
 - build.sbt
 - gradle/wrapper/gradle-wrapper.jar
@@ -56,6 +58,8 @@ files:
 - gradlew.bat
 - lib/embulk/guess/xpath2.rb
 - lib/embulk/parser/xpath2.rb
+- project/build.properties
+- project/plugins.sbt
 - src/main/scala/org/embulk/parser/xpath2/LoanPattern.scala
 - src/main/scala/org/embulk/parser/xpath2/PluginTask.scala
 - src/main/scala/org/embulk/parser/xpath2/XPath2ParserPlugin.scala
@@ -64,8 +68,8 @@ files:
 - src/test/resources/data.xml
 - src/test/scala/org/embulk/parser/xpath2/UnitSpec.scala
 - src/test/scala/org/embulk/parser/xpath2/XPath2ParserPluginSpec.scala
-- classpath/embulk-parser-xpath2-0.0.3.jar
-- classpath/scala-library-2.12.3.jar
+- classpath/scala-library-2.12.4.jar
+- classpath/embulk-parser-xpath2-0.0.4.jar
 homepage: https://github.com/maji-KY/embulk-parser-xpath2
 licenses:
 - MIT