embulk-parser-xpath2 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -3
- data/README.md +101 -0
- data/benchmark/src/main/scala/org/embulk/parser/xpath2/ParseBenchmark.scala +1 -1
- data/build.gradle +1 -1
- data/src/main/scala/org/embulk/parser/xpath2/MsgPackEncoder.scala +139 -0
- data/src/main/scala/org/embulk/parser/xpath2/VTD.scala +12 -0
- data/src/main/scala/org/embulk/parser/xpath2/XPath2ParserPlugin.scala +12 -18
- data/src/main/scala/org/embulk/parser/xpath2/config/SchemaConfig.scala +25 -4
- data/src/test/resources/json_config.yml +30 -0
- data/src/test/resources/json_data.xml +27 -0
- data/src/test/scala/org/embulk/parser/xpath2/TestColumnVisitor.scala +53 -0
- data/src/test/scala/org/embulk/parser/xpath2/TestTransactionalPageOutput.scala +31 -0
- data/src/test/scala/org/embulk/parser/xpath2/XPath2ParserPluginJsonSpec.scala +86 -0
- data/src/test/scala/org/embulk/parser/xpath2/XPath2ParserPluginSpec.scala +0 -77
- metadata +10 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d739e0975d864c0920c7af2f4bdbaac2d4978925
|
|
4
|
+
data.tar.gz: 07d69b8ad2cefa9e25edd06c98952e4735b45498
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 8edf4e7762787940f0addf22fe650e5d5912d030cac7d86d0c4bf00e70b275b641c903b6ab65088a7858a38caed425b0d0bdb99df2be51d5cfc417497ea2896e
|
|
7
|
+
data.tar.gz: 0ddc27b1bfab2f31e3992e008cdcedfde8c288e287409551e52a5ecbae37086582ee4b5659e8fe131f1149e97a3212d50fd38f44b05a0747e1693b85002ef8c9
|
data/.gitignore
CHANGED
data/README.md
CHANGED
|
@@ -11,6 +11,7 @@ Embulk parser plugin for parsing xml data by XPath perfectly!
|
|
|
11
11
|
|
|
12
12
|
- namespace awareness
|
|
13
13
|
- nullable columns
|
|
14
|
+
- complex json array columns (with restrictions)
|
|
14
15
|
|
|
15
16
|
## Overview
|
|
16
17
|
|
|
@@ -61,6 +62,106 @@ Then you can fetch entries from the following xml:
|
|
|
61
62
|
</ns2:entry>
|
|
62
63
|
</ns1:root>
|
|
63
64
|
```
|
|
65
|
+
|
|
66
|
+
## complex json array column
|
|
67
|
+
|
|
68
|
+
### Usage
|
|
69
|
+
|
|
70
|
+
```yaml
|
|
71
|
+
parser:
|
|
72
|
+
type: xpath2
|
|
73
|
+
root: '/ns1:root/ns2:entry'
|
|
74
|
+
schema:
|
|
75
|
+
- { path: 'ns2:id', name: id, type: long }
|
|
76
|
+
- path: 'ns2:list'
|
|
77
|
+
name: list
|
|
78
|
+
type: json
|
|
79
|
+
structure: # adding structure key to enabling complex json array column
|
|
80
|
+
- path: 'ns2:list'
|
|
81
|
+
name: list
|
|
82
|
+
type: array
|
|
83
|
+
- path: 'ns2:list/ns2:elements'
|
|
84
|
+
name: elements
|
|
85
|
+
type: array
|
|
86
|
+
- path: 'ns2:list/ns2:elements/ns2:name'
|
|
87
|
+
name: elementName
|
|
88
|
+
type: string
|
|
89
|
+
- path: 'ns2:list/ns2:elements/ns2:value'
|
|
90
|
+
name: elementValue
|
|
91
|
+
type: long
|
|
92
|
+
- path: 'ns2:list/ns2:elements/ns2:active'
|
|
93
|
+
name: elementActive
|
|
94
|
+
type: boolean
|
|
95
|
+
namespaces: {ns1: 'http://example.com/ns1/', ns2: 'http://example.com/ns2/'}
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
### Structure configuration
|
|
99
|
+
- **path**: specify path from the XPath of the column (string, required)
|
|
100
|
+
- **name**: json key name (string)
|
|
101
|
+
- **type**: json data type (One of array, string, long, boolean., required)
|
|
102
|
+
|
|
103
|
+
Then you can fetch entries from the following xml:
|
|
104
|
+
```xml
|
|
105
|
+
<?xml version="1.0"?>
|
|
106
|
+
<ns1:root
|
|
107
|
+
xmlns:ns1="http://example.com/ns1/"
|
|
108
|
+
xmlns:ns2="http://example.com/ns2/">
|
|
109
|
+
<ns2:entry>
|
|
110
|
+
<ns2:id>1</ns2:id>
|
|
111
|
+
<ns2:list>
|
|
112
|
+
<ns2:elements>
|
|
113
|
+
<ns2:name>foo1</ns2:name>
|
|
114
|
+
<ns2:value>1</ns2:value>
|
|
115
|
+
<ns2:active>true</ns2:active>
|
|
116
|
+
</ns2:elements>
|
|
117
|
+
<ns2:elements>
|
|
118
|
+
<ns2:name>foo2</ns2:name>
|
|
119
|
+
<ns2:value>2</ns2:value>
|
|
120
|
+
<ns2:active>false</ns2:active>
|
|
121
|
+
</ns2:elements>
|
|
122
|
+
</ns2:list>
|
|
123
|
+
<ns2:list>
|
|
124
|
+
<ns2:elements>
|
|
125
|
+
<ns2:name>bar1</ns2:name>
|
|
126
|
+
<ns2:value>3</ns2:value>
|
|
127
|
+
<ns2:active>true</ns2:active>
|
|
128
|
+
</ns2:elements>
|
|
129
|
+
</ns2:list>
|
|
130
|
+
</ns2:entry>
|
|
131
|
+
</ns1:root>
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
result of `list` column:
|
|
135
|
+
```json
|
|
136
|
+
{
|
|
137
|
+
"list": [
|
|
138
|
+
{
|
|
139
|
+
"elements": [
|
|
140
|
+
{
|
|
141
|
+
"elementActive": true,
|
|
142
|
+
"elementName": "foo1",
|
|
143
|
+
"elementValue": 1
|
|
144
|
+
},
|
|
145
|
+
{
|
|
146
|
+
"elementActive": false,
|
|
147
|
+
"elementName": "foo2",
|
|
148
|
+
"elementValue": 2
|
|
149
|
+
}
|
|
150
|
+
]
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
"elements": [
|
|
154
|
+
{
|
|
155
|
+
"elementActive": true,
|
|
156
|
+
"elementName": "bar1",
|
|
157
|
+
"elementValue": 3
|
|
158
|
+
}
|
|
159
|
+
]
|
|
160
|
+
}
|
|
161
|
+
]
|
|
162
|
+
}
|
|
163
|
+
```
|
|
164
|
+
|
|
64
165
|
## Build
|
|
65
166
|
|
|
66
167
|
```
|
data/build.gradle
CHANGED
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
package org.embulk.parser.xpath2
|
|
2
|
+
|
|
3
|
+
import com.ximpleware.{AutoPilot, VTDNav}
|
|
4
|
+
import org.embulk.parser.xpath2.config.JsonStructureElement
|
|
5
|
+
import org.embulk.spi.Column
|
|
6
|
+
import org.msgpack.value.{Value, Variable}
|
|
7
|
+
|
|
8
|
+
import scala.annotation.tailrec
|
|
9
|
+
import scala.collection.JavaConverters._
|
|
10
|
+
import scala.collection.immutable.Queue
|
|
11
|
+
|
|
12
|
+
sealed trait Direction
|
|
13
|
+
case object Parent extends Direction
|
|
14
|
+
case object Sibling extends Direction
|
|
15
|
+
case object Child extends Direction
|
|
16
|
+
|
|
17
|
+
private case class Path(depth: Int, pathFragments: Seq[String], moveDirection: Direction) {
|
|
18
|
+
|
|
19
|
+
def next(depth: Int, elementName: String): Path = {
|
|
20
|
+
if (this.depth > depth) {
|
|
21
|
+
val (rest :+ _) = pathFragments
|
|
22
|
+
Path(depth, rest, Parent)
|
|
23
|
+
} else if (this.depth == depth) {
|
|
24
|
+
val (rest :+ _) = pathFragments
|
|
25
|
+
Path(depth, rest :+ elementName, Sibling)
|
|
26
|
+
} else {
|
|
27
|
+
Path(depth, pathFragments :+ elementName, Child)
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
val path: String = pathFragments.mkString("/")
|
|
32
|
+
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
object MsgPackEncoder {
|
|
36
|
+
|
|
37
|
+
def encode(nav: VTDNav, columnAp: AutoPilot, column: Column, maybeStructure: Option[Seq[JsonStructureElement]]): Value = maybeStructure.map { structure =>
|
|
38
|
+
// complex json array
|
|
39
|
+
val keyValues = Iterator.continually(columnAp.evalXPath()).takeWhile(_ != -1).flatMap { _ =>
|
|
40
|
+
VTD.withinContext(nav) {
|
|
41
|
+
constructJsonMap(nav, columnAp, column, structure).toSeq
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
val mergedMap = keyValues.toSeq.groupBy { case (k, _) => k }.map { case (k, v)=>
|
|
45
|
+
val mergedValues = v.flatMap {
|
|
46
|
+
case (_, x: Seq[Any]) => x
|
|
47
|
+
case _ => sys.error("Root element supports array only. Please reconsider the configuration.")
|
|
48
|
+
}
|
|
49
|
+
(k, mergedValues)
|
|
50
|
+
}
|
|
51
|
+
convertToValue(mergedMap)
|
|
52
|
+
} getOrElse {
|
|
53
|
+
// simple string[]
|
|
54
|
+
@tailrec
|
|
55
|
+
def eachJsonValue(cAp: AutoPilot, queue: Queue[Value]): Queue[Value] = if (cAp.evalXPath() != -1) {
|
|
56
|
+
val index = nav.getText
|
|
57
|
+
val nextQueue = if (index != -1) queue :+ new Variable().setStringValue(nav.toString(index)).asStringValue() else queue
|
|
58
|
+
eachJsonValue(cAp, nextQueue)
|
|
59
|
+
} else queue
|
|
60
|
+
asArrayValue(eachJsonValue(columnAp, Queue.empty[Value]))
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
private def constructJsonMap(nav: VTDNav, columnAp: AutoPilot, column: Column, structure: Seq[JsonStructureElement]): Map[String, Any] = {
|
|
64
|
+
|
|
65
|
+
@tailrec
|
|
66
|
+
def eachElement(eAp: AutoPilot, previousPath: Path, obj: Map[String, Any]): Map[String, Any] = if (eAp.iterate()) {
|
|
67
|
+
val current = previousPath.next(nav.getCurrentDepth, nav.toString(nav.getCurrentIndex))
|
|
68
|
+
if (current.moveDirection == Parent) {
|
|
69
|
+
obj
|
|
70
|
+
} else {
|
|
71
|
+
val updated = structure.find(_.path == current.path).map { x =>
|
|
72
|
+
x.`type` match {
|
|
73
|
+
case "array" =>
|
|
74
|
+
val targetArray = obj.getOrElse(x.name, Queue[Any]()).asInstanceOf[Seq[Any]]
|
|
75
|
+
val childStructure = eachArrayElement(eAp, current)
|
|
76
|
+
obj.updated(x.name, targetArray ++ childStructure)
|
|
77
|
+
case "string" => obj.updated(x.name, nav.toNormalizedString(nav.getText))
|
|
78
|
+
case "long" => obj.updated(x.name, nav.toNormalizedString(nav.getText).toLong)
|
|
79
|
+
case "boolean" => obj.updated(x.name, nav.toNormalizedString(nav.getText).toBoolean)
|
|
80
|
+
case notSupported@_ => sys.error(s"type=$notSupported is notSupported")
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
eachElement(eAp, current, updated.getOrElse(obj))
|
|
84
|
+
}
|
|
85
|
+
} else obj
|
|
86
|
+
|
|
87
|
+
def isArrayElement(current: Path): Boolean =
|
|
88
|
+
structure.exists(x => x.path == current.path && x.`type` == "array")
|
|
89
|
+
|
|
90
|
+
def eachArrayElement(eAp: AutoPilot, previousPath: Path): Seq[Map[String, Any]] = {
|
|
91
|
+
@tailrec
|
|
92
|
+
def loop(eAp: AutoPilot, previousPath: Path, obj: Map[String, Any], elements: Seq[Map[String, Any]]): Seq[Map[String, Any]] = {
|
|
93
|
+
val arrayContent = eachElement(eAp, previousPath, obj)
|
|
94
|
+
val currentPath = previousPath.next(nav.getCurrentDepth, nav.toString(nav.getCurrentIndex))
|
|
95
|
+
if (isArrayElement(currentPath)) {
|
|
96
|
+
loop(eAp, currentPath, obj, elements :+ arrayContent)
|
|
97
|
+
} else elements :+ arrayContent
|
|
98
|
+
}
|
|
99
|
+
loop(eAp, previousPath, Map.empty[String, Any], Queue.empty[Map[String, Any]])
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
val eachElementAp = new AutoPilot(nav)
|
|
103
|
+
eachElementAp.selectElement("*")
|
|
104
|
+
|
|
105
|
+
val initialPath = Path(-1, Vector.empty, Sibling)
|
|
106
|
+
|
|
107
|
+
eachElement(eachElementAp, initialPath, Map[String, Any]())
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
private def convertToValue(obj: Map[String, Any]): Value = {
|
|
111
|
+
val map = obj.map {
|
|
112
|
+
case (k, v: Seq[_]) => (asStringValue(k), convertToValue(v))
|
|
113
|
+
case (k, v: Map[_, _]) => (asStringValue(k), convertToValue(v.asInstanceOf[Map[String, Any]]))
|
|
114
|
+
case (k, v: String) => (asStringValue(k), asStringValue(v))
|
|
115
|
+
case (k, v: Boolean) => (asStringValue(k), asBooleanValue(v))
|
|
116
|
+
case (k, v: Long) => (asStringValue(k), asLongValue(v))
|
|
117
|
+
case (k, v) => sys.error(s"can't convert: key=$k, value=$v")
|
|
118
|
+
}
|
|
119
|
+
asMapValue(map)
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
private def convertToValue(seq: Seq[Any]): Value = {
|
|
123
|
+
val list = seq.map {
|
|
124
|
+
case v: Seq[_] => convertToValue(v)
|
|
125
|
+
case v: Map[_, _] => convertToValue(v.asInstanceOf[Map[String, Any]])
|
|
126
|
+
case v: String => asStringValue(v)
|
|
127
|
+
case v: Boolean => asBooleanValue(v)
|
|
128
|
+
case v: Long => asLongValue(v)
|
|
129
|
+
}
|
|
130
|
+
asArrayValue(list)
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
private final def asStringValue(value: String): Value = new Variable().setStringValue(value).asStringValue()
|
|
134
|
+
private final def asBooleanValue(value: Boolean): Value = new Variable().setBooleanValue(value).asBooleanValue()
|
|
135
|
+
private final def asLongValue(value: Long): Value = new Variable().setIntegerValue(value).asNumberValue()
|
|
136
|
+
private final def asArrayValue(value: Seq[Value]): Value = new Variable().setArrayValue(value.asJava).asArrayValue()
|
|
137
|
+
private final def asMapValue(value: Map[Value, Value]): Value = new Variable().setMapValue(value.asJava).asMapValue()
|
|
138
|
+
|
|
139
|
+
}
|
|
@@ -3,12 +3,11 @@ package org.embulk.parser.xpath2
|
|
|
3
3
|
import com.google.common.io.ByteStreams
|
|
4
4
|
import com.ximpleware.{AutoPilot, VTDGen, VTDNav}
|
|
5
5
|
import org.embulk.config._
|
|
6
|
-
import org.embulk.parser.xpath2.config.ColumnConfig
|
|
6
|
+
import org.embulk.parser.xpath2.config.{ColumnConfig, JsonStructureElement}
|
|
7
7
|
import org.embulk.spi._
|
|
8
8
|
import org.embulk.spi.`type`._
|
|
9
9
|
import org.embulk.spi.time.TimestampParser
|
|
10
10
|
import org.embulk.spi.util.FileInputInputStream
|
|
11
|
-
import org.msgpack.value.{Value, Variable}
|
|
12
11
|
import org.slf4j.Logger
|
|
13
12
|
|
|
14
13
|
import scala.annotation.tailrec
|
|
@@ -33,7 +32,10 @@ class XPath2ParserPlugin extends ParserPlugin {
|
|
|
33
32
|
val stopOnInvalidRecord: Boolean = task.getStopOnInvalidRecord
|
|
34
33
|
|
|
35
34
|
val timestampParsers: Map[String, TimestampParser] = task.getSchema.columns.asScala
|
|
36
|
-
.collect { case ColumnConfig(_, name, _, Some(timestampColumnOption), _) => (name, new TimestampParser(task, timestampColumnOption)) }.toMap
|
|
35
|
+
.collect { case ColumnConfig(_, name, _, Some(timestampColumnOption), _, _) => (name, new TimestampParser(task, timestampColumnOption)) }.toMap
|
|
36
|
+
|
|
37
|
+
val jsonStructures: Map[String, Seq[JsonStructureElement]] = task.getSchema.columns.asScala
|
|
38
|
+
.collect { case ColumnConfig(_, name, _, _, Some(jsonColumnOption), _) => (name, jsonColumnOption.structure.asScala) }.toMap
|
|
37
39
|
|
|
38
40
|
def declareXPathNS(ap: AutoPilot): Unit = {
|
|
39
41
|
task.getNamespaces.conf.asScala.foreach { case (prefix, namespaceURI) =>
|
|
@@ -67,11 +69,11 @@ class XPath2ParserPlugin extends ParserPlugin {
|
|
|
67
69
|
nav.push()
|
|
68
70
|
try {
|
|
69
71
|
columnElementAutoPilots.zipWithIndex.foreach { case (columnElementAutoPilot, idx) =>
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
72
|
+
VTD.withinContext(nav) {
|
|
73
|
+
columnElementAutoPilot.resetXPath()
|
|
74
|
+
val column = schema.getColumn(idx)
|
|
75
|
+
handleColumn(pb, nav, columnElementAutoPilot, column, timestampParsers, jsonStructures)
|
|
76
|
+
}
|
|
75
77
|
}
|
|
76
78
|
pb.addRecord()
|
|
77
79
|
} catch {
|
|
@@ -95,17 +97,9 @@ class XPath2ParserPlugin extends ParserPlugin {
|
|
|
95
97
|
}
|
|
96
98
|
}
|
|
97
99
|
|
|
98
|
-
final def handleColumn(pb: PageBuilder, nav: VTDNav, columnAp: AutoPilot, column: Column, timestampParsers: Map[String, TimestampParser]): Unit = {
|
|
100
|
+
final def handleColumn(pb: PageBuilder, nav: VTDNav, columnAp: AutoPilot, column: Column, timestampParsers: Map[String, TimestampParser], jsonStructures: Map[String, Seq[JsonStructureElement]]): Unit = {
|
|
99
101
|
if (column.getType.isInstanceOf[JsonType]) {
|
|
100
|
-
val
|
|
101
|
-
@tailrec
|
|
102
|
-
def eachJsonValue(cAp: AutoPilot): Unit = if (cAp.evalXPath() != -1) {
|
|
103
|
-
val index = nav.getText
|
|
104
|
-
if (index != -1) list.add(new Variable().setStringValue(nav.toString(index)).asStringValue())
|
|
105
|
-
eachJsonValue(cAp)
|
|
106
|
-
}
|
|
107
|
-
eachJsonValue(columnAp)
|
|
108
|
-
val jsonValue = new Variable().setArrayValue(list).asArrayValue()
|
|
102
|
+
val jsonValue = MsgPackEncoder.encode(nav, columnAp, column, jsonStructures.get(column.getName))
|
|
109
103
|
pb.setJson(column, jsonValue)
|
|
110
104
|
} else {
|
|
111
105
|
if (columnAp.evalXPath() == -1) {
|
|
@@ -2,10 +2,10 @@ package org.embulk.parser.xpath2.config
|
|
|
2
2
|
|
|
3
3
|
import java.util
|
|
4
4
|
|
|
5
|
-
import com.fasterxml.jackson.annotation.{JsonCreator, JsonValue}
|
|
5
|
+
import com.fasterxml.jackson.annotation.{JsonCreator, JsonProperty, JsonValue}
|
|
6
6
|
import com.google.common.base.Optional
|
|
7
7
|
import org.embulk.config.{Config, ConfigDefault, ConfigSource}
|
|
8
|
-
import org.embulk.spi.`type`.{TimestampType, Type}
|
|
8
|
+
import org.embulk.spi.`type`.{JsonType, TimestampType, Type}
|
|
9
9
|
import org.embulk.spi.time.TimestampParser.TimestampColumnOption
|
|
10
10
|
import org.joda.time.DateTimeZone
|
|
11
11
|
|
|
@@ -14,11 +14,18 @@ case class SchemaConfig @JsonCreator()(columns: java.util.List[ColumnConfig]) {
|
|
|
14
14
|
def getColumns: util.List[ColumnConfig] = columns
|
|
15
15
|
}
|
|
16
16
|
|
|
17
|
-
case class ColumnConfig(path: String, name: String, `type`: Type, timestampOption: Option[TimestampColumnOption], option: ConfigSource) {
|
|
17
|
+
case class ColumnConfig(path: String, name: String, `type`: Type, timestampOption: Option[TimestampColumnOption], jsonOption: Option[JsonColumnOption], option: ConfigSource) {
|
|
18
18
|
|
|
19
19
|
@JsonCreator()
|
|
20
20
|
def this(src: ConfigSource) = {
|
|
21
|
-
this(
|
|
21
|
+
this(
|
|
22
|
+
src.get(classOf[String], "path"),
|
|
23
|
+
src.get(classOf[String], "name"),
|
|
24
|
+
src.get(classOf[Type], "type"),
|
|
25
|
+
ColumnConfig.getTimestampOption(src, src.get(classOf[Type], "type")),
|
|
26
|
+
ColumnConfig.getJsonOption(src, src.get(classOf[Type], "type")),
|
|
27
|
+
src
|
|
28
|
+
)
|
|
22
29
|
}
|
|
23
30
|
|
|
24
31
|
@JsonValue()
|
|
@@ -49,11 +56,25 @@ private class TimestampColumnOptionImpl(timezone: Optional[DateTimeZone], format
|
|
|
49
56
|
override val getDate = date
|
|
50
57
|
}
|
|
51
58
|
|
|
59
|
+
class JsonStructureElement(@JsonProperty("path") val path: String, @JsonProperty("type") val `type`: String) {
|
|
60
|
+
@JsonProperty("name")
|
|
61
|
+
val name: String = path
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
case class JsonColumnOption(@JsonProperty("structure") structure: java.util.List[JsonStructureElement])
|
|
65
|
+
|
|
52
66
|
object ColumnConfig {
|
|
53
67
|
private def getTimestampOption(src: ConfigSource, `type`: Type): Option[TimestampColumnOption] = `type` match {
|
|
54
68
|
case _: TimestampType => Some(getOption(src).loadConfig(classOf[TimestampColumnOptionImpl]))
|
|
55
69
|
case _ => None
|
|
56
70
|
}
|
|
57
71
|
|
|
72
|
+
private def getJsonOption(src: ConfigSource, `type`: Type): Option[JsonColumnOption] = `type` match {
|
|
73
|
+
case _: JsonType =>
|
|
74
|
+
val option = getOption(src)
|
|
75
|
+
if (option.has("structure")) Some(option.loadConfig(classOf[JsonColumnOption])) else None
|
|
76
|
+
case _ => None
|
|
77
|
+
}
|
|
78
|
+
|
|
58
79
|
private def getOption(src: ConfigSource) = src.deepCopy().remove("path").remove("name").remove("type")
|
|
59
80
|
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
in:
|
|
2
|
+
type: file
|
|
3
|
+
path_prefix: src/test/resources/data.xml
|
|
4
|
+
parser:
|
|
5
|
+
type: xpath2
|
|
6
|
+
root: '/ns1:root/ns2:entry'
|
|
7
|
+
schema:
|
|
8
|
+
- { path: 'ns2:id', name: id, type: long }
|
|
9
|
+
- path: 'ns2:list'
|
|
10
|
+
name: list
|
|
11
|
+
type: json
|
|
12
|
+
structure:
|
|
13
|
+
- path: 'ns2:list'
|
|
14
|
+
name: list
|
|
15
|
+
type: array
|
|
16
|
+
- path: 'ns2:list/ns2:elements'
|
|
17
|
+
name: elements
|
|
18
|
+
type: array
|
|
19
|
+
- path: 'ns2:list/ns2:elements/ns2:name'
|
|
20
|
+
name: elementName
|
|
21
|
+
type: string
|
|
22
|
+
- path: 'ns2:list/ns2:elements/ns2:value'
|
|
23
|
+
name: elementValue
|
|
24
|
+
type: long
|
|
25
|
+
- path: 'ns2:list/ns2:elements/ns2:active'
|
|
26
|
+
name: elementActive
|
|
27
|
+
type: boolean
|
|
28
|
+
namespaces: {ns1: 'http://example.com/ns1/', ns2: 'http://example.com/ns2/'}
|
|
29
|
+
out:
|
|
30
|
+
type: stdout
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
<?xml version="1.0"?>
|
|
2
|
+
<ns1:root
|
|
3
|
+
xmlns:ns1="http://example.com/ns1/"
|
|
4
|
+
xmlns:ns2="http://example.com/ns2/">
|
|
5
|
+
<ns2:entry>
|
|
6
|
+
<ns2:id>1</ns2:id>
|
|
7
|
+
<ns2:list>
|
|
8
|
+
<ns2:elements>
|
|
9
|
+
<ns2:name>foo1</ns2:name>
|
|
10
|
+
<ns2:value>1</ns2:value>
|
|
11
|
+
<ns2:active>true</ns2:active>
|
|
12
|
+
</ns2:elements>
|
|
13
|
+
<ns2:elements>
|
|
14
|
+
<ns2:name>foo2</ns2:name>
|
|
15
|
+
<ns2:value>2</ns2:value>
|
|
16
|
+
<ns2:active>false</ns2:active>
|
|
17
|
+
</ns2:elements>
|
|
18
|
+
</ns2:list>
|
|
19
|
+
<ns2:list>
|
|
20
|
+
<ns2:elements>
|
|
21
|
+
<ns2:name>bar1</ns2:name>
|
|
22
|
+
<ns2:value>3</ns2:value>
|
|
23
|
+
<ns2:active>true</ns2:active>
|
|
24
|
+
</ns2:elements>
|
|
25
|
+
</ns2:list>
|
|
26
|
+
</ns2:entry>
|
|
27
|
+
</ns1:root>
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
package org.embulk.parser.xpath2
|
|
2
|
+
|
|
3
|
+
import org.embulk.spi.{Column, ColumnVisitor, PageReader}
|
|
4
|
+
|
|
5
|
+
class TestColumnVisitor(reader: PageReader, record: collection.mutable.Map[String, Any]) extends ColumnVisitor {
|
|
6
|
+
override def timestampColumn(column: Column): Unit = {
|
|
7
|
+
if (reader.isNull(column)) {
|
|
8
|
+
record.put(column.getName, null)
|
|
9
|
+
} else {
|
|
10
|
+
record.put(column.getName, reader.getTimestamp(column))
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
override def stringColumn(column: Column): Unit = {
|
|
15
|
+
if (reader.isNull(column)) {
|
|
16
|
+
record.put(column.getName, null)
|
|
17
|
+
} else {
|
|
18
|
+
record.put(column.getName, reader.getString(column))
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
override def longColumn(column: Column): Unit = {
|
|
23
|
+
if (reader.isNull(column)) {
|
|
24
|
+
record.put(column.getName, null)
|
|
25
|
+
} else {
|
|
26
|
+
record.put(column.getName, reader.getLong(column))
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
override def doubleColumn(column: Column): Unit = {
|
|
31
|
+
if (reader.isNull(column)) {
|
|
32
|
+
record.put(column.getName, null)
|
|
33
|
+
} else {
|
|
34
|
+
record.put(column.getName, reader.getDouble(column))
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
override def booleanColumn(column: Column): Unit = {
|
|
39
|
+
if (reader.isNull(column)) {
|
|
40
|
+
record.put(column.getName, null)
|
|
41
|
+
} else {
|
|
42
|
+
record.put(column.getName, reader.getBoolean(column))
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
override def jsonColumn(column: Column): Unit = {
|
|
47
|
+
if (reader.isNull(column)) {
|
|
48
|
+
record.put(column.getName, null)
|
|
49
|
+
} else {
|
|
50
|
+
record.put(column.getName, reader.getJson(column))
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
package org.embulk.parser.xpath2
|
|
2
|
+
|
|
3
|
+
import org.embulk.spi.{Exec, Page, Schema, TransactionalPageOutput}
|
|
4
|
+
|
|
5
|
+
import scala.collection.mutable
|
|
6
|
+
import scala.collection.JavaConverters._
|
|
7
|
+
|
|
8
|
+
class TestTransactionalPageOutput(schema: Schema, result: mutable.Buffer[collection.mutable.Map[String, Any]])
|
|
9
|
+
extends TransactionalPageOutput {
|
|
10
|
+
import org.embulk.spi.PageReader
|
|
11
|
+
|
|
12
|
+
val reader = new PageReader(schema)
|
|
13
|
+
|
|
14
|
+
override def add(page: Page) = {
|
|
15
|
+
reader.setPage(page)
|
|
16
|
+
|
|
17
|
+
while (reader.nextRecord()) {
|
|
18
|
+
val record: collection.mutable.Map[String, Any] = collection.mutable.Map()
|
|
19
|
+
|
|
20
|
+
schema.getColumns().asScala.foreach { column =>
|
|
21
|
+
column.visit(new TestColumnVisitor(reader, record))
|
|
22
|
+
}
|
|
23
|
+
result += record
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
override def commit() = Exec.newTaskReport()
|
|
28
|
+
override def abort() = {}
|
|
29
|
+
override def finish() = {}
|
|
30
|
+
override def close() = {}
|
|
31
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
package org.embulk.parser.xpath2
|
|
2
|
+
|
|
3
|
+
import java.io.{File, FileInputStream}
|
|
4
|
+
import java.nio.file
|
|
5
|
+
import java.nio.file.Paths
|
|
6
|
+
|
|
7
|
+
import org.embulk.EmbulkTestRuntime
|
|
8
|
+
import org.embulk.config.{ConfigLoader, ConfigSource, TaskSource}
|
|
9
|
+
import org.embulk.spi.json.JsonParser
|
|
10
|
+
import org.embulk.spi.util.InputStreamFileInput
|
|
11
|
+
import org.embulk.spi.{Exec, _}
|
|
12
|
+
import org.junit.Assert._
|
|
13
|
+
import org.junit.{Rule, Test}
|
|
14
|
+
|
|
15
|
+
import scala.collection.mutable
|
|
16
|
+
import scala.collection.mutable.ArrayBuffer
|
|
17
|
+
|
|
18
|
+
class XPath2ParserPluginJsonSpec {
|
|
19
|
+
|
|
20
|
+
@Rule
|
|
21
|
+
def runtime = new EmbulkTestRuntime
|
|
22
|
+
|
|
23
|
+
val yamlPath: file.Path = Paths.get(classOf[XPath2ParserPlugin].getClassLoader.getResource("json_config.yml").toURI)
|
|
24
|
+
val dataPath: String = classOf[XPath2ParserPlugin].getClassLoader.getResource("json_data.xml").getPath
|
|
25
|
+
|
|
26
|
+
def configSource: ConfigSource = new ConfigLoader(Exec.getModelManager).fromYamlFile(yamlPath.toFile).getNested("in").getNested("parser")
|
|
27
|
+
|
|
28
|
+
@Test def testParseJsonArrayXML() {
|
|
29
|
+
|
|
30
|
+
val cs = configSource
|
|
31
|
+
val task = cs.loadConfig(classOf[PluginTask])
|
|
32
|
+
|
|
33
|
+
var schema: Schema = null
|
|
34
|
+
|
|
35
|
+
val plugin = new XPath2ParserPlugin()
|
|
36
|
+
plugin.transaction(cs, (_: TaskSource, s: Schema) => {schema = s})
|
|
37
|
+
|
|
38
|
+
val result: mutable.Buffer[collection.mutable.Map[String, Any]] = mutable.Buffer()
|
|
39
|
+
|
|
40
|
+
plugin.run(
|
|
41
|
+
task.dump(),
|
|
42
|
+
schema,
|
|
43
|
+
new InputStreamFileInput(Exec.getBufferAllocator, new FileInputStream(new File(dataPath))),
|
|
44
|
+
new TestTransactionalPageOutput(schema, result)
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
println(result)
|
|
48
|
+
|
|
49
|
+
val expectedJson =
|
|
50
|
+
"""{
|
|
51
|
+
"list": [
|
|
52
|
+
{
|
|
53
|
+
"elements": [
|
|
54
|
+
{
|
|
55
|
+
"elementActive": true,
|
|
56
|
+
"elementName": "foo1",
|
|
57
|
+
"elementValue": 1
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"elementActive": false,
|
|
61
|
+
"elementName": "foo2",
|
|
62
|
+
"elementValue": 2
|
|
63
|
+
}
|
|
64
|
+
]
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
"elements": [
|
|
68
|
+
{
|
|
69
|
+
"elementActive": true,
|
|
70
|
+
"elementName": "bar1",
|
|
71
|
+
"elementValue": 3
|
|
72
|
+
}
|
|
73
|
+
]
|
|
74
|
+
}
|
|
75
|
+
]
|
|
76
|
+
}"""
|
|
77
|
+
|
|
78
|
+
assertEquals(ArrayBuffer(
|
|
79
|
+
Map(
|
|
80
|
+
"id" -> 1L,
|
|
81
|
+
"list" -> new JsonParser().parse(expectedJson)
|
|
82
|
+
)
|
|
83
|
+
), result)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
}
|
|
@@ -141,80 +141,3 @@ class XPath2ParserPluginSpec {
|
|
|
141
141
|
}
|
|
142
142
|
|
|
143
143
|
}
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
class TestTransactionalPageOutput(schema: Schema, result: mutable.Buffer[collection.mutable.Map[String, Any]])
|
|
148
|
-
extends TransactionalPageOutput {
|
|
149
|
-
import org.embulk.spi.PageReader
|
|
150
|
-
|
|
151
|
-
val reader = new PageReader(schema)
|
|
152
|
-
|
|
153
|
-
override def add(page: Page) = {
|
|
154
|
-
reader.setPage(page)
|
|
155
|
-
|
|
156
|
-
while (reader.nextRecord()) {
|
|
157
|
-
val record: collection.mutable.Map[String, Any] = collection.mutable.Map()
|
|
158
|
-
|
|
159
|
-
schema.getColumns().asScala.foreach { column =>
|
|
160
|
-
column.visit(new TestColumnVisitor(reader, record))
|
|
161
|
-
}
|
|
162
|
-
result += record
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
override def commit() = Exec.newTaskReport()
|
|
167
|
-
override def abort() = {}
|
|
168
|
-
override def finish() = {}
|
|
169
|
-
override def close() = {}
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
class TestColumnVisitor(reader: PageReader, record: collection.mutable.Map[String, Any]) extends ColumnVisitor {
|
|
173
|
-
override def timestampColumn(column: Column): Unit = {
|
|
174
|
-
if (reader.isNull(column)) {
|
|
175
|
-
record.put(column.getName, null)
|
|
176
|
-
} else {
|
|
177
|
-
record.put(column.getName, reader.getTimestamp(column))
|
|
178
|
-
}
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
override def stringColumn(column: Column): Unit = {
|
|
182
|
-
if (reader.isNull(column)) {
|
|
183
|
-
record.put(column.getName, null)
|
|
184
|
-
} else {
|
|
185
|
-
record.put(column.getName, reader.getString(column))
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
override def longColumn(column: Column): Unit = {
|
|
190
|
-
if (reader.isNull(column)) {
|
|
191
|
-
record.put(column.getName, null)
|
|
192
|
-
} else {
|
|
193
|
-
record.put(column.getName, reader.getLong(column))
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
override def doubleColumn(column: Column): Unit = {
|
|
198
|
-
if (reader.isNull(column)) {
|
|
199
|
-
record.put(column.getName, null)
|
|
200
|
-
} else {
|
|
201
|
-
record.put(column.getName, reader.getDouble(column))
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
override def booleanColumn(column: Column): Unit = {
|
|
206
|
-
if (reader.isNull(column)) {
|
|
207
|
-
record.put(column.getName, null)
|
|
208
|
-
} else {
|
|
209
|
-
record.put(column.getName, reader.getBoolean(column))
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
override def jsonColumn(column: Column): Unit = {
|
|
214
|
-
if (reader.isNull(column)) {
|
|
215
|
-
record.put(column.getName, null)
|
|
216
|
-
} else {
|
|
217
|
-
record.put(column.getName, reader.getJson(column))
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
}
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-parser-xpath2
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- maji-KY
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2018-01-12 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -61,18 +61,25 @@ files:
|
|
|
61
61
|
- project/build.properties
|
|
62
62
|
- project/plugins.sbt
|
|
63
63
|
- src/main/scala/org/embulk/parser/xpath2/LoanPattern.scala
|
|
64
|
+
- src/main/scala/org/embulk/parser/xpath2/MsgPackEncoder.scala
|
|
64
65
|
- src/main/scala/org/embulk/parser/xpath2/PluginTask.scala
|
|
66
|
+
- src/main/scala/org/embulk/parser/xpath2/VTD.scala
|
|
65
67
|
- src/main/scala/org/embulk/parser/xpath2/XPath2ParserPlugin.scala
|
|
66
68
|
- src/main/scala/org/embulk/parser/xpath2/config/NamespacesConfig.scala
|
|
67
69
|
- src/main/scala/org/embulk/parser/xpath2/config/SchemaConfig.scala
|
|
68
70
|
- src/test/resources/config.yml
|
|
69
71
|
- src/test/resources/data.xml
|
|
70
72
|
- src/test/resources/invalid-data.xml
|
|
73
|
+
- src/test/resources/json_config.yml
|
|
74
|
+
- src/test/resources/json_data.xml
|
|
75
|
+
- src/test/scala/org/embulk/parser/xpath2/TestColumnVisitor.scala
|
|
76
|
+
- src/test/scala/org/embulk/parser/xpath2/TestTransactionalPageOutput.scala
|
|
71
77
|
- src/test/scala/org/embulk/parser/xpath2/UnitSpec.scala
|
|
78
|
+
- src/test/scala/org/embulk/parser/xpath2/XPath2ParserPluginJsonSpec.scala
|
|
72
79
|
- src/test/scala/org/embulk/parser/xpath2/XPath2ParserPluginSpec.scala
|
|
73
80
|
- classpath/vtd-xml-2.13.4.jar
|
|
74
81
|
- classpath/scala-library-2.12.4.jar
|
|
75
|
-
- classpath/embulk-parser-xpath2-0.
|
|
82
|
+
- classpath/embulk-parser-xpath2-0.2.0.jar
|
|
76
83
|
homepage: https://github.com/maji-KY/embulk-parser-xpath2
|
|
77
84
|
licenses:
|
|
78
85
|
- GPL-2.0
|