embulk-filter-expand_json 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +9 -0
- data/README.md +4 -1
- data/build.gradle +13 -3
- data/classpath/embulk-filter-expand_json-0.0.3.jar +0 -0
- data/src/main/java/org/embulk/filter/expand_json/ExpandJsonFilterPlugin.java +1 -18
- data/src/main/java/org/embulk/filter/expand_json/FilteredPageOutput.java +15 -7
- data/src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java +390 -0
- metadata +4 -3
- data/classpath/embulk-filter-expand_json-0.0.2.jar +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a7c9576abd91635ad11888a88fba5f14e576383e
|
|
4
|
+
data.tar.gz: 6a80bd358b9056620075fc16f1f7ae59a5a911c9
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 63ae1b2d8c541497ba9bc0c892a403b30c32e6ac8222eb6bc70d09675c0fc9ec2e26a6b5fbd9443e7d5e2ee479b6bbaa8efaf8d8e1f4995b839f9fd1d4600089
|
|
7
|
+
data.tar.gz: 73e53f220990bfe91249eceef14450039c29461a20821963e281d99462fb76fdf6583784f4270301c0a46f6b805259143dd1473d787f21a9c18306718885f6d8
|
data/.travis.yml
ADDED
data/README.md
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# Expand Json filter plugin for Embulk
|
|
2
2
|
|
|
3
|
+
[](https://travis-ci.org/civitaspo/embulk-filter-expand_json)
|
|
4
|
+
[](https://coveralls.io/github/civitaspo/embulk-filter-expand_json?branch=master)
|
|
5
|
+
|
|
3
6
|
expand columns having json into multiple columns
|
|
4
7
|
|
|
5
8
|
## Overview
|
|
@@ -9,7 +12,7 @@ expand columns having json into multiple columns
|
|
|
9
12
|
## Configuration
|
|
10
13
|
|
|
11
14
|
- **json_column_name**: a column name having json to be expanded (string, required)
|
|
12
|
-
- **root**: root property to start fetching each entries, specify in [JsonPath](http://goessner.net/articles/JsonPath/) style (string, default:
|
|
15
|
+
- **root**: root property to start fetching each entries, specify in [JsonPath](http://goessner.net/articles/JsonPath/) style (string, default: `"$."`)
|
|
13
16
|
- **expanded_columns**: columns expanded into multiple columns (array of hash, required)
|
|
14
17
|
- **name**: name of the column. you can define [JsonPath](http://goessner.net/articles/JsonPath/) style.
|
|
15
18
|
- **type**: type of the column (see below)
|
data/build.gradle
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
plugins {
|
|
2
2
|
id "com.jfrog.bintray" version "1.1"
|
|
3
3
|
id "com.github.jruby-gradle.base" version "0.1.5"
|
|
4
|
+
id "com.github.kt3k.coveralls" version "2.4.0"
|
|
5
|
+
id "jacoco"
|
|
4
6
|
id "java"
|
|
5
7
|
}
|
|
6
8
|
import com.github.jrubygradle.JRubyExec
|
|
@@ -12,15 +14,23 @@ configurations {
|
|
|
12
14
|
provided
|
|
13
15
|
}
|
|
14
16
|
|
|
15
|
-
version = "0.0.
|
|
17
|
+
version = "0.0.3"
|
|
16
18
|
sourceCompatibility = 1.7
|
|
17
19
|
targetCompatibility = 1.7
|
|
18
20
|
|
|
19
21
|
dependencies {
|
|
20
|
-
compile "org.embulk:embulk-core:0.7
|
|
21
|
-
provided "org.embulk:embulk-core:0.7
|
|
22
|
+
compile "org.embulk:embulk-core:0.7.+"
|
|
23
|
+
provided "org.embulk:embulk-core:0.7.+"
|
|
22
24
|
compile "com.jayway.jsonpath:json-path:2.+"
|
|
23
25
|
testCompile "junit:junit:4.+"
|
|
26
|
+
testCompile "org.embulk:embulk-core:0.7.+:tests"
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
jacocoTestReport {
|
|
30
|
+
reports {
|
|
31
|
+
xml.enabled = true // coveralls plugin depends on xml format report
|
|
32
|
+
html.enabled = true
|
|
33
|
+
}
|
|
24
34
|
}
|
|
25
35
|
|
|
26
36
|
task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
Binary file
|
|
@@ -1,14 +1,6 @@
|
|
|
1
1
|
package org.embulk.filter.expand_json;
|
|
2
2
|
|
|
3
|
-
import com.fasterxml.jackson.core.JsonProcessingException;
|
|
4
|
-
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
5
|
-
import com.google.common.base.Throwables;
|
|
6
3
|
import com.google.common.collect.ImmutableList;
|
|
7
|
-
import com.google.common.collect.Maps;
|
|
8
|
-
import com.jayway.jsonpath.Configuration;
|
|
9
|
-
import com.jayway.jsonpath.JsonPath;
|
|
10
|
-
import com.jayway.jsonpath.Option;
|
|
11
|
-
import com.jayway.jsonpath.ReadContext;
|
|
12
4
|
import org.embulk.config.Config;
|
|
13
5
|
import org.embulk.config.ConfigDefault;
|
|
14
6
|
import org.embulk.config.ConfigSource;
|
|
@@ -18,21 +10,12 @@ import org.embulk.spi.Column;
|
|
|
18
10
|
import org.embulk.spi.ColumnConfig;
|
|
19
11
|
import org.embulk.spi.Exec;
|
|
20
12
|
import org.embulk.spi.FilterPlugin;
|
|
21
|
-
import org.embulk.spi.Page;
|
|
22
|
-
import org.embulk.spi.PageBuilder;
|
|
23
13
|
import org.embulk.spi.PageOutput;
|
|
24
|
-
import org.embulk.spi.PageReader;
|
|
25
14
|
import org.embulk.spi.Schema;
|
|
26
15
|
import org.embulk.spi.time.TimestampParser;
|
|
27
|
-
import org.embulk.spi.type.Types;
|
|
28
|
-
import org.joda.time.DateTimeZone;
|
|
29
|
-
import org.jruby.embed.ScriptingContainer;
|
|
30
16
|
import org.slf4j.Logger;
|
|
31
17
|
|
|
32
|
-
import java.util.ArrayList;
|
|
33
|
-
import java.util.HashMap;
|
|
34
18
|
import java.util.List;
|
|
35
|
-
import java.util.Map;
|
|
36
19
|
|
|
37
20
|
public class ExpandJsonFilterPlugin
|
|
38
21
|
implements FilterPlugin
|
|
@@ -52,10 +35,10 @@ public class ExpandJsonFilterPlugin
|
|
|
52
35
|
@Config("expanded_columns")
|
|
53
36
|
public List<ColumnConfig> getExpandedColumns();
|
|
54
37
|
|
|
38
|
+
// Time zone of timestamp columns if the value itself doesn’t include time zone description (eg. Asia/Tokyo)
|
|
55
39
|
@Config("time_zone")
|
|
56
40
|
@ConfigDefault("\"UTC\"")
|
|
57
41
|
public String getTimeZone();
|
|
58
|
-
|
|
59
42
|
}
|
|
60
43
|
|
|
61
44
|
@Override
|
|
@@ -22,12 +22,11 @@ import org.embulk.spi.type.Types;
|
|
|
22
22
|
import org.joda.time.DateTimeZone;
|
|
23
23
|
import org.slf4j.Logger;
|
|
24
24
|
|
|
25
|
-
import java.util.ArrayList;
|
|
26
25
|
import java.util.HashMap;
|
|
27
26
|
import java.util.List;
|
|
28
27
|
import java.util.Map;
|
|
29
28
|
|
|
30
|
-
import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin
|
|
29
|
+
import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin.PluginTask;
|
|
31
30
|
|
|
32
31
|
/**
|
|
33
32
|
* Created by takahiro.nakayama on 10/19/15.
|
|
@@ -44,6 +43,7 @@ public class FilteredPageOutput
|
|
|
44
43
|
private final PageReader pageReader;
|
|
45
44
|
private final Schema inputSchema;
|
|
46
45
|
private final Schema outputSchema;
|
|
46
|
+
private final PageBuilder pageBuilder;
|
|
47
47
|
private final PageOutput pageOutput;
|
|
48
48
|
|
|
49
49
|
FilteredPageOutput(PluginTask task, Schema inputSchema, Schema outputSchema, PageOutput pageOutput)
|
|
@@ -76,20 +76,20 @@ public class FilteredPageOutput
|
|
|
76
76
|
this.inputSchema = inputSchema;
|
|
77
77
|
this.outputSchema = outputSchema;
|
|
78
78
|
this.pageOutput = pageOutput;
|
|
79
|
+
this.pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, pageOutput);
|
|
79
80
|
}
|
|
80
81
|
|
|
81
82
|
@Override
|
|
82
83
|
public void add(Page page)
|
|
83
84
|
{
|
|
84
|
-
try
|
|
85
|
+
try {
|
|
85
86
|
pageReader.setPage(page);
|
|
86
87
|
|
|
87
88
|
while (pageReader.nextRecord()) {
|
|
88
|
-
|
|
89
|
+
setInputColumnsExceptExpandedJsonColumns(pageBuilder, inputColumnsExceptExpandedJsonColumn);
|
|
89
90
|
setExpandedJsonColumns(pageBuilder, jsonColumn, expandedJsonColumns, timestampParserHashMap);
|
|
90
91
|
pageBuilder.addRecord();
|
|
91
92
|
}
|
|
92
|
-
pageBuilder.finish();
|
|
93
93
|
}
|
|
94
94
|
catch (JsonProcessingException e) {
|
|
95
95
|
logger.error(e.getMessage());
|
|
@@ -100,6 +100,7 @@ public class FilteredPageOutput
|
|
|
100
100
|
@Override
|
|
101
101
|
public void finish()
|
|
102
102
|
{
|
|
103
|
+
pageBuilder.finish();
|
|
103
104
|
pageOutput.finish();
|
|
104
105
|
}
|
|
105
106
|
|
|
@@ -107,6 +108,7 @@ public class FilteredPageOutput
|
|
|
107
108
|
public void close()
|
|
108
109
|
{
|
|
109
110
|
pageReader.close();
|
|
111
|
+
pageBuilder.close();
|
|
110
112
|
pageOutput.close();
|
|
111
113
|
}
|
|
112
114
|
|
|
@@ -115,7 +117,13 @@ public class FilteredPageOutput
|
|
|
115
117
|
final HashMap<String, TimestampParser> timestampParserHashMap = Maps.newHashMap();
|
|
116
118
|
for (ColumnConfig expandedColumnConfig: task.getExpandedColumns()) {
|
|
117
119
|
if (Types.TIMESTAMP.equals(expandedColumnConfig.getType())) {
|
|
118
|
-
String format
|
|
120
|
+
String format;
|
|
121
|
+
if (expandedColumnConfig.getOption().has("format")) {
|
|
122
|
+
format = expandedColumnConfig.getOption().get(String.class, "format");
|
|
123
|
+
}
|
|
124
|
+
else {
|
|
125
|
+
format = task.getDefaultTimestampFormat();
|
|
126
|
+
}
|
|
119
127
|
DateTimeZone timezone = DateTimeZone.forID(task.getTimeZone());
|
|
120
128
|
TimestampParser parser = new TimestampParser(task.getJRuby(), format, timezone);
|
|
121
129
|
|
|
@@ -128,7 +136,7 @@ public class FilteredPageOutput
|
|
|
128
136
|
return timestampParserHashMap;
|
|
129
137
|
}
|
|
130
138
|
|
|
131
|
-
private void
|
|
139
|
+
private void setInputColumnsExceptExpandedJsonColumns(PageBuilder pageBuilder, List<Column> inputColumnsExceptExpandedJsonColumn) {
|
|
132
140
|
for (Column inputColumn: inputColumnsExceptExpandedJsonColumn) {
|
|
133
141
|
if (pageReader.isNull(inputColumn)) {
|
|
134
142
|
pageBuilder.setNull(inputColumn);
|
|
@@ -1,5 +1,395 @@
|
|
|
1
1
|
package org.embulk.filter.expand_json;
|
|
2
2
|
|
|
3
|
+
import com.fasterxml.jackson.core.JsonProcessingException;
|
|
4
|
+
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
5
|
+
import com.google.common.base.Throwables;
|
|
6
|
+
import com.google.common.collect.ImmutableList;
|
|
7
|
+
import com.google.common.collect.ImmutableMap;
|
|
8
|
+
import com.jayway.jsonpath.InvalidJsonException;
|
|
9
|
+
import org.embulk.EmbulkTestRuntime;
|
|
10
|
+
import org.embulk.config.ConfigException;
|
|
11
|
+
import org.embulk.config.ConfigLoader;
|
|
12
|
+
import org.embulk.config.ConfigSource;
|
|
13
|
+
import org.embulk.config.TaskSource;
|
|
14
|
+
import org.embulk.spi.Column;
|
|
15
|
+
import org.embulk.spi.Exec;
|
|
16
|
+
import org.embulk.spi.Page;
|
|
17
|
+
import org.embulk.spi.PageOutput;
|
|
18
|
+
import org.embulk.spi.PageReader;
|
|
19
|
+
import org.embulk.spi.PageTestUtils;
|
|
20
|
+
import org.embulk.spi.Schema;
|
|
21
|
+
import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
|
|
22
|
+
import org.junit.Before;
|
|
23
|
+
import org.junit.Rule;
|
|
24
|
+
import org.junit.Test;
|
|
25
|
+
import org.junit.rules.ExpectedException;
|
|
26
|
+
|
|
27
|
+
import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin.Control;
|
|
28
|
+
import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin.PluginTask;
|
|
29
|
+
import static org.embulk.spi.type.Types.*;
|
|
30
|
+
import static org.junit.Assert.assertEquals;
|
|
31
|
+
|
|
3
32
|
public class TestExpandJsonFilterPlugin
|
|
4
33
|
{
|
|
34
|
+
@Rule
|
|
35
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
|
36
|
+
|
|
37
|
+
@Rule
|
|
38
|
+
public ExpectedException exception = ExpectedException.none();
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
private final Schema schema = Schema.builder()
|
|
42
|
+
.add("_c0", STRING)
|
|
43
|
+
.build();
|
|
44
|
+
private ExpandJsonFilterPlugin expandJsonFilterPlugin;
|
|
45
|
+
|
|
46
|
+
@Before
|
|
47
|
+
public void createResources()
|
|
48
|
+
{
|
|
49
|
+
expandJsonFilterPlugin = new ExpandJsonFilterPlugin();
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
private ConfigSource getConfigFromYaml(String yaml)
|
|
53
|
+
{
|
|
54
|
+
ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
|
|
55
|
+
return loader.fromYamlString(yaml);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
private String convertToJsonString(Object object)
|
|
59
|
+
{
|
|
60
|
+
ObjectMapper mapper = new ObjectMapper();
|
|
61
|
+
try {
|
|
62
|
+
return mapper.writeValueAsString(object);
|
|
63
|
+
}
|
|
64
|
+
catch (JsonProcessingException e) {
|
|
65
|
+
throw Throwables.propagate(e);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
private String getBrokenJsonString()
|
|
70
|
+
{
|
|
71
|
+
return "{\"_j0\": \"te\"\n";
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/*
|
|
75
|
+
Config test
|
|
76
|
+
*/
|
|
77
|
+
|
|
78
|
+
@Test
|
|
79
|
+
public void testThrowExceptionAbsentJsonColumnName()
|
|
80
|
+
{
|
|
81
|
+
String configYaml = "" +
|
|
82
|
+
"type: expand_json\n" +
|
|
83
|
+
"expanded_columns:\n" +
|
|
84
|
+
" - {name: _c1, type: string}";
|
|
85
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
|
86
|
+
|
|
87
|
+
exception.expect(ConfigException.class);
|
|
88
|
+
exception.expectMessage("Field 'json_column_name' is required but not set");
|
|
89
|
+
config.loadConfig(PluginTask.class);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
@Test
|
|
93
|
+
public void testThrowExceptionAbsentExpandedColumns()
|
|
94
|
+
{
|
|
95
|
+
String configYaml = "" +
|
|
96
|
+
"type: expand_json\n" +
|
|
97
|
+
"json_column_name: _c0\n";
|
|
98
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
|
99
|
+
|
|
100
|
+
exception.expect(ConfigException.class);
|
|
101
|
+
exception.expectMessage("Field 'expanded_columns' is required but not set");
|
|
102
|
+
config.loadConfig(PluginTask.class);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
@Test
|
|
106
|
+
public void testDefaultValue()
|
|
107
|
+
{
|
|
108
|
+
String configYaml = "" +
|
|
109
|
+
"type: expand_json\n" +
|
|
110
|
+
"json_column_name: _c0\n" +
|
|
111
|
+
"expanded_columns:\n" +
|
|
112
|
+
" - {name: _j1, type: boolean}\n" +
|
|
113
|
+
" - {name: _j2, type: long}\n" +
|
|
114
|
+
" - {name: _j3, type: timestamp}\n" +
|
|
115
|
+
" - {name: _j4, type: double}\n" +
|
|
116
|
+
" - {name: _j5, type: string}\n";
|
|
117
|
+
|
|
118
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
|
119
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
|
120
|
+
|
|
121
|
+
assertEquals("$.", task.getRoot());
|
|
122
|
+
assertEquals("UTC", task.getTimeZone());
|
|
123
|
+
assertEquals("%Y-%m-%d %H:%M:%S.%N %z", task.getDefaultTimestampFormat());
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/*
|
|
127
|
+
Expand Test
|
|
128
|
+
*/
|
|
129
|
+
|
|
130
|
+
@Test
|
|
131
|
+
public void testExpandJsonKeyToSchema()
|
|
132
|
+
{
|
|
133
|
+
String configYaml = "" +
|
|
134
|
+
"type: expand_json\n" +
|
|
135
|
+
"json_column_name: _c0\n" +
|
|
136
|
+
"root: $.\n" +
|
|
137
|
+
"expanded_columns:\n" +
|
|
138
|
+
" - {name: _j1, type: boolean}\n" +
|
|
139
|
+
" - {name: _j2, type: long}\n" +
|
|
140
|
+
" - {name: _j3, type: timestamp}\n" +
|
|
141
|
+
" - {name: _j4, type: double}\n" +
|
|
142
|
+
" - {name: _j5, type: string}\n";
|
|
143
|
+
|
|
144
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
|
145
|
+
|
|
146
|
+
expandJsonFilterPlugin.transaction(config, schema, new Control()
|
|
147
|
+
{
|
|
148
|
+
@Override
|
|
149
|
+
public void run(TaskSource taskSource, Schema outputSchema)
|
|
150
|
+
{
|
|
151
|
+
assertEquals(5, outputSchema.getColumnCount());
|
|
152
|
+
|
|
153
|
+
Column new_j1 = outputSchema.getColumn(0);
|
|
154
|
+
Column new_j2 = outputSchema.getColumn(1);
|
|
155
|
+
Column new_j3 = outputSchema.getColumn(2);
|
|
156
|
+
Column new_j4 = outputSchema.getColumn(3);
|
|
157
|
+
Column new_j5 = outputSchema.getColumn(4);
|
|
158
|
+
|
|
159
|
+
assertEquals("_j1", new_j1.getName());
|
|
160
|
+
assertEquals(BOOLEAN, new_j1.getType());
|
|
161
|
+
assertEquals("_j2", new_j2.getName());
|
|
162
|
+
assertEquals(LONG, new_j2.getType());
|
|
163
|
+
assertEquals("_j3", new_j3.getName());
|
|
164
|
+
assertEquals(TIMESTAMP, new_j3.getType());
|
|
165
|
+
assertEquals("_j4", new_j4.getName());
|
|
166
|
+
assertEquals(DOUBLE, new_j4.getType());
|
|
167
|
+
assertEquals("_j5", new_j5.getName());
|
|
168
|
+
assertEquals(STRING, new_j5.getType());
|
|
169
|
+
}
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
@Test
|
|
174
|
+
public void testExpandJsonValues()
|
|
175
|
+
{
|
|
176
|
+
String configYaml = "" +
|
|
177
|
+
"type: expand_json\n" +
|
|
178
|
+
"json_column_name: _c0\n" +
|
|
179
|
+
"root: $.\n" +
|
|
180
|
+
"time_zone: Asia/Tokyo\n" +
|
|
181
|
+
"expanded_columns:\n" +
|
|
182
|
+
" - {name: _j0, type: boolean}\n" +
|
|
183
|
+
" - {name: _j1, type: long}\n" +
|
|
184
|
+
" - {name: _j2, type: timestamp, format: '%Y-%m-%d %H:%M:%S %z'}\n" +
|
|
185
|
+
" - {name: _j3, type: double}\n" +
|
|
186
|
+
" - {name: _j4, type: string}\n" +
|
|
187
|
+
" - {name: _j5, type: timestamp, format: '%Y-%m-%d %H:%M:%S %z'}\n" +
|
|
188
|
+
" - {name: _j6, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}\n" +
|
|
189
|
+
// JsonPath: https://github.com/jayway/JsonPath
|
|
190
|
+
" - {name: '_j7.store.book[*].author', type: string}\n" +
|
|
191
|
+
" - {name: '_j7..book[?(@.price <= $[''_j7''][''expensive''])].author', type: string}\n" +
|
|
192
|
+
" - {name: '_j7..book[?(@.isbn)]', type: string}\n" +
|
|
193
|
+
" - {name: '_j7..book[?(@.author =~ /.*REES/i)].title', type: string}\n" +
|
|
194
|
+
" - {name: '_j7.store.book[2].author', type: string}\n";
|
|
195
|
+
|
|
196
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
|
197
|
+
|
|
198
|
+
expandJsonFilterPlugin.transaction(config, schema, new Control()
|
|
199
|
+
{
|
|
200
|
+
@Override
|
|
201
|
+
public void run(TaskSource taskSource, Schema outputSchema)
|
|
202
|
+
{
|
|
203
|
+
MockPageOutput mockPageOutput = new MockPageOutput();
|
|
204
|
+
PageOutput pageOutput = expandJsonFilterPlugin.open(taskSource,
|
|
205
|
+
schema,
|
|
206
|
+
outputSchema,
|
|
207
|
+
mockPageOutput);
|
|
208
|
+
|
|
209
|
+
ImmutableMap.Builder<String,Object> builder = ImmutableMap.builder();
|
|
210
|
+
builder.put("_j0", true);
|
|
211
|
+
builder.put("_j1", 2);
|
|
212
|
+
builder.put("_j2", "2014-10-21 04:44:33 +0900");
|
|
213
|
+
builder.put("_j3", 4.4);
|
|
214
|
+
builder.put("_j4", "v5");
|
|
215
|
+
builder.put("_j5", "2014-10-21 04:44:33 +0000");
|
|
216
|
+
builder.put("_j6", "2014-10-21 04:44:33");
|
|
217
|
+
builder.put("_j7",
|
|
218
|
+
ImmutableMap.of("store",
|
|
219
|
+
ImmutableMap.of("book",
|
|
220
|
+
ImmutableList.of(ImmutableMap.of("author",
|
|
221
|
+
"Nigel Rees",
|
|
222
|
+
"title",
|
|
223
|
+
"Sayings of the Century",
|
|
224
|
+
"price",
|
|
225
|
+
8.95),
|
|
226
|
+
ImmutableMap.of("author",
|
|
227
|
+
"Evelyn Waugh",
|
|
228
|
+
"title",
|
|
229
|
+
"Sword of Honour",
|
|
230
|
+
"price",
|
|
231
|
+
12.99),
|
|
232
|
+
ImmutableMap.of("author",
|
|
233
|
+
"Herman Melville",
|
|
234
|
+
"title",
|
|
235
|
+
"Moby Dick",
|
|
236
|
+
"isbn",
|
|
237
|
+
"0-553-21311-3",
|
|
238
|
+
"price",
|
|
239
|
+
8.99),
|
|
240
|
+
ImmutableMap.of("author",
|
|
241
|
+
"J. R. R. Tolkien",
|
|
242
|
+
"title",
|
|
243
|
+
"The Lord of the Rings",
|
|
244
|
+
"isbn",
|
|
245
|
+
"0-395-19395-8",
|
|
246
|
+
"price",
|
|
247
|
+
22.99)
|
|
248
|
+
),
|
|
249
|
+
"bicycle",
|
|
250
|
+
ImmutableMap.of("color",
|
|
251
|
+
"red",
|
|
252
|
+
"price",
|
|
253
|
+
19.95
|
|
254
|
+
)
|
|
255
|
+
),
|
|
256
|
+
"expensive",
|
|
257
|
+
10
|
|
258
|
+
)
|
|
259
|
+
/*
|
|
260
|
+
{
|
|
261
|
+
"store": {
|
|
262
|
+
"book": [
|
|
263
|
+
{
|
|
264
|
+
"author": "Nigel Rees",
|
|
265
|
+
"title": "Sayings of the Century",
|
|
266
|
+
"price": 8.95
|
|
267
|
+
},
|
|
268
|
+
{
|
|
269
|
+
"author": "Evelyn Waugh",
|
|
270
|
+
"title": "Sword of Honour",
|
|
271
|
+
"price": 12.99
|
|
272
|
+
},
|
|
273
|
+
{
|
|
274
|
+
"author": "Herman Melville",
|
|
275
|
+
"title": "Moby Dick",
|
|
276
|
+
"isbn": "0-553-21311-3",
|
|
277
|
+
"price": 8.99
|
|
278
|
+
},
|
|
279
|
+
{
|
|
280
|
+
"author": "J. R. R. Tolkien",
|
|
281
|
+
"title": "The Lord of the Rings",
|
|
282
|
+
"isbn": "0-395-19395-8",
|
|
283
|
+
"price": 22.99
|
|
284
|
+
}
|
|
285
|
+
],
|
|
286
|
+
"bicycle": {
|
|
287
|
+
"color": "red",
|
|
288
|
+
"price": 19.95
|
|
289
|
+
}
|
|
290
|
+
},
|
|
291
|
+
"expensive": 10
|
|
292
|
+
}
|
|
293
|
+
*/
|
|
294
|
+
);
|
|
295
|
+
|
|
296
|
+
String data = convertToJsonString(builder.build());
|
|
297
|
+
|
|
298
|
+
for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(),
|
|
299
|
+
schema,
|
|
300
|
+
data)) {
|
|
301
|
+
pageOutput.add(page);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
pageOutput.finish();
|
|
305
|
+
pageOutput.close();
|
|
306
|
+
|
|
307
|
+
PageReader pageReader = new PageReader(outputSchema);
|
|
308
|
+
|
|
309
|
+
for (Page page : mockPageOutput.pages) {
|
|
310
|
+
pageReader.setPage(page);
|
|
311
|
+
assertEquals(true, pageReader.getBoolean(outputSchema.getColumn(0)));
|
|
312
|
+
assertEquals(2, pageReader.getLong(outputSchema.getColumn(1)));
|
|
313
|
+
assertEquals("2014-10-20 19:44:33 UTC",
|
|
314
|
+
pageReader.getTimestamp(outputSchema.getColumn(2)).toString());
|
|
315
|
+
assertEquals(String.valueOf(4.4),
|
|
316
|
+
String.valueOf(pageReader.getDouble(outputSchema.getColumn(3))));
|
|
317
|
+
assertEquals("v5", pageReader.getString(outputSchema.getColumn(4)));
|
|
318
|
+
assertEquals("2014-10-21 04:44:33 UTC",
|
|
319
|
+
pageReader.getTimestamp(outputSchema.getColumn(5)).toString());
|
|
320
|
+
assertEquals("2014-10-20 19:44:33 UTC",
|
|
321
|
+
pageReader.getTimestamp(outputSchema.getColumn(6)).toString());
|
|
322
|
+
assertEquals("[\"Nigel Rees\",\"Evelyn Waugh\",\"Herman Melville\",\"J. R. R. Tolkien\"]",
|
|
323
|
+
pageReader.getString(outputSchema.getColumn(7)));
|
|
324
|
+
assertEquals("[\"Nigel Rees\",\"Herman Melville\"]",
|
|
325
|
+
pageReader.getString(outputSchema.getColumn(8)));
|
|
326
|
+
assertEquals("[" +
|
|
327
|
+
"{" +
|
|
328
|
+
"\"author\":\"Herman Melville\"," +
|
|
329
|
+
"\"title\":\"Moby Dick\"," +
|
|
330
|
+
"\"isbn\":\"0-553-21311-3\"," +
|
|
331
|
+
"\"price\":8.99" +
|
|
332
|
+
"}," +
|
|
333
|
+
"{" +
|
|
334
|
+
"\"author\":\"J. R. R. Tolkien\"," +
|
|
335
|
+
"\"title\":\"The Lord of the Rings\"," +
|
|
336
|
+
"\"isbn\":\"0-395-19395-8\"," +
|
|
337
|
+
"\"price\":22.99" +
|
|
338
|
+
"}" +
|
|
339
|
+
"]",
|
|
340
|
+
pageReader.getString(outputSchema.getColumn(9)));
|
|
341
|
+
assertEquals("[\"Sayings of the Century\"]",
|
|
342
|
+
pageReader.getString(outputSchema.getColumn(10)));
|
|
343
|
+
assertEquals("Herman Melville",
|
|
344
|
+
pageReader.getString(outputSchema.getColumn(11)));
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
});
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
@Test
|
|
351
|
+
public void testAbortBrokenJsonString()
|
|
352
|
+
{
|
|
353
|
+
String configYaml = "" +
|
|
354
|
+
"type: expand_json\n" +
|
|
355
|
+
"json_column_name: _c0\n" +
|
|
356
|
+
"root: $.\n" +
|
|
357
|
+
"time_zone: Asia/Tokyo\n" +
|
|
358
|
+
"expanded_columns:\n" +
|
|
359
|
+
" - {name: _j0, type: string}\n";
|
|
360
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
|
361
|
+
|
|
362
|
+
expandJsonFilterPlugin.transaction(config, schema, new Control()
|
|
363
|
+
{
|
|
364
|
+
@Override
|
|
365
|
+
public void run(TaskSource taskSource, Schema outputSchema)
|
|
366
|
+
{
|
|
367
|
+
MockPageOutput mockPageOutput = new MockPageOutput();
|
|
368
|
+
PageOutput pageOutput = expandJsonFilterPlugin.open(taskSource,
|
|
369
|
+
schema,
|
|
370
|
+
outputSchema,
|
|
371
|
+
mockPageOutput);
|
|
372
|
+
|
|
373
|
+
String data = getBrokenJsonString();
|
|
374
|
+
for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(),
|
|
375
|
+
schema,
|
|
376
|
+
data)) {
|
|
377
|
+
exception.expect(InvalidJsonException.class);
|
|
378
|
+
exception.expectMessage("Unexpected End Of File position 12: null");
|
|
379
|
+
pageOutput.add(page);
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
pageOutput.finish();
|
|
383
|
+
pageOutput.close();
|
|
384
|
+
|
|
385
|
+
PageReader pageReader = new PageReader(outputSchema);
|
|
386
|
+
|
|
387
|
+
for (Page page : mockPageOutput.pages) {
|
|
388
|
+
pageReader.setPage(page);
|
|
389
|
+
assertEquals("te", pageReader.getString(outputSchema.getColumn(0)));
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
});
|
|
393
|
+
}
|
|
394
|
+
|
|
5
395
|
}
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-filter-expand_json
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Civitaspo
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2015-
|
|
11
|
+
date: 2015-11-16 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -46,6 +46,7 @@ extensions: []
|
|
|
46
46
|
extra_rdoc_files: []
|
|
47
47
|
files:
|
|
48
48
|
- .gitignore
|
|
49
|
+
- .travis.yml
|
|
49
50
|
- LICENSE.txt
|
|
50
51
|
- README.md
|
|
51
52
|
- build.gradle
|
|
@@ -61,7 +62,7 @@ files:
|
|
|
61
62
|
- src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java
|
|
62
63
|
- classpath/asm-1.0.2.jar
|
|
63
64
|
- classpath/asm-3.3.1.jar
|
|
64
|
-
- classpath/embulk-filter-expand_json-0.0.
|
|
65
|
+
- classpath/embulk-filter-expand_json-0.0.3.jar
|
|
65
66
|
- classpath/json-path-2.0.0.jar
|
|
66
67
|
- classpath/json-smart-2.1.1.jar
|
|
67
68
|
homepage: https://github.com/civitaspo/embulk-filter-expand_json
|
|
Binary file
|