embulk-filter-expand_json 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +9 -0
- data/README.md +4 -1
- data/build.gradle +13 -3
- data/classpath/embulk-filter-expand_json-0.0.3.jar +0 -0
- data/src/main/java/org/embulk/filter/expand_json/ExpandJsonFilterPlugin.java +1 -18
- data/src/main/java/org/embulk/filter/expand_json/FilteredPageOutput.java +15 -7
- data/src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java +390 -0
- metadata +4 -3
- data/classpath/embulk-filter-expand_json-0.0.2.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a7c9576abd91635ad11888a88fba5f14e576383e
|
4
|
+
data.tar.gz: 6a80bd358b9056620075fc16f1f7ae59a5a911c9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 63ae1b2d8c541497ba9bc0c892a403b30c32e6ac8222eb6bc70d09675c0fc9ec2e26a6b5fbd9443e7d5e2ee479b6bbaa8efaf8d8e1f4995b839f9fd1d4600089
|
7
|
+
data.tar.gz: 73e53f220990bfe91249eceef14450039c29461a20821963e281d99462fb76fdf6583784f4270301c0a46f6b805259143dd1473d787f21a9c18306718885f6d8
|
data/.travis.yml
ADDED
data/README.md
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
# Expand Json filter plugin for Embulk
|
2
2
|
|
3
|
+
[![Build Status](https://travis-ci.org/civitaspo/embulk-filter-expand_json.svg)](https://travis-ci.org/civitaspo/embulk-filter-expand_json)
|
4
|
+
[![Coverage Status](https://coveralls.io/repos/civitaspo/embulk-filter-expand_json/badge.svg?branch=master&service=github)](https://coveralls.io/github/civitaspo/embulk-filter-expand_json?branch=master)
|
5
|
+
|
3
6
|
expand columns having json into multiple columns
|
4
7
|
|
5
8
|
## Overview
|
@@ -9,7 +12,7 @@ expand columns having json into multiple columns
|
|
9
12
|
## Configuration
|
10
13
|
|
11
14
|
- **json_column_name**: a column name having json to be expanded (string, required)
|
12
|
-
- **root**: root property to start fetching each entries, specify in [JsonPath](http://goessner.net/articles/JsonPath/) style (string, default:
|
15
|
+
- **root**: root property to start fetching each entries, specify in [JsonPath](http://goessner.net/articles/JsonPath/) style (string, default: `"$."`)
|
13
16
|
- **expanded_columns**: columns expanded into multiple columns (array of hash, required)
|
14
17
|
- **name**: name of the column. you can define [JsonPath](http://goessner.net/articles/JsonPath/) style.
|
15
18
|
- **type**: type of the column (see below)
|
data/build.gradle
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
plugins {
|
2
2
|
id "com.jfrog.bintray" version "1.1"
|
3
3
|
id "com.github.jruby-gradle.base" version "0.1.5"
|
4
|
+
id "com.github.kt3k.coveralls" version "2.4.0"
|
5
|
+
id "jacoco"
|
4
6
|
id "java"
|
5
7
|
}
|
6
8
|
import com.github.jrubygradle.JRubyExec
|
@@ -12,15 +14,23 @@ configurations {
|
|
12
14
|
provided
|
13
15
|
}
|
14
16
|
|
15
|
-
version = "0.0.
|
17
|
+
version = "0.0.3"
|
16
18
|
sourceCompatibility = 1.7
|
17
19
|
targetCompatibility = 1.7
|
18
20
|
|
19
21
|
dependencies {
|
20
|
-
compile "org.embulk:embulk-core:0.7
|
21
|
-
provided "org.embulk:embulk-core:0.7
|
22
|
+
compile "org.embulk:embulk-core:0.7.+"
|
23
|
+
provided "org.embulk:embulk-core:0.7.+"
|
22
24
|
compile "com.jayway.jsonpath:json-path:2.+"
|
23
25
|
testCompile "junit:junit:4.+"
|
26
|
+
testCompile "org.embulk:embulk-core:0.7.+:tests"
|
27
|
+
}
|
28
|
+
|
29
|
+
jacocoTestReport {
|
30
|
+
reports {
|
31
|
+
xml.enabled = true // coveralls plugin depends on xml format report
|
32
|
+
html.enabled = true
|
33
|
+
}
|
24
34
|
}
|
25
35
|
|
26
36
|
task classpath(type: Copy, dependsOn: ["jar"]) {
|
Binary file
|
@@ -1,14 +1,6 @@
|
|
1
1
|
package org.embulk.filter.expand_json;
|
2
2
|
|
3
|
-
import com.fasterxml.jackson.core.JsonProcessingException;
|
4
|
-
import com.fasterxml.jackson.databind.ObjectMapper;
|
5
|
-
import com.google.common.base.Throwables;
|
6
3
|
import com.google.common.collect.ImmutableList;
|
7
|
-
import com.google.common.collect.Maps;
|
8
|
-
import com.jayway.jsonpath.Configuration;
|
9
|
-
import com.jayway.jsonpath.JsonPath;
|
10
|
-
import com.jayway.jsonpath.Option;
|
11
|
-
import com.jayway.jsonpath.ReadContext;
|
12
4
|
import org.embulk.config.Config;
|
13
5
|
import org.embulk.config.ConfigDefault;
|
14
6
|
import org.embulk.config.ConfigSource;
|
@@ -18,21 +10,12 @@ import org.embulk.spi.Column;
|
|
18
10
|
import org.embulk.spi.ColumnConfig;
|
19
11
|
import org.embulk.spi.Exec;
|
20
12
|
import org.embulk.spi.FilterPlugin;
|
21
|
-
import org.embulk.spi.Page;
|
22
|
-
import org.embulk.spi.PageBuilder;
|
23
13
|
import org.embulk.spi.PageOutput;
|
24
|
-
import org.embulk.spi.PageReader;
|
25
14
|
import org.embulk.spi.Schema;
|
26
15
|
import org.embulk.spi.time.TimestampParser;
|
27
|
-
import org.embulk.spi.type.Types;
|
28
|
-
import org.joda.time.DateTimeZone;
|
29
|
-
import org.jruby.embed.ScriptingContainer;
|
30
16
|
import org.slf4j.Logger;
|
31
17
|
|
32
|
-
import java.util.ArrayList;
|
33
|
-
import java.util.HashMap;
|
34
18
|
import java.util.List;
|
35
|
-
import java.util.Map;
|
36
19
|
|
37
20
|
public class ExpandJsonFilterPlugin
|
38
21
|
implements FilterPlugin
|
@@ -52,10 +35,10 @@ public class ExpandJsonFilterPlugin
|
|
52
35
|
@Config("expanded_columns")
|
53
36
|
public List<ColumnConfig> getExpandedColumns();
|
54
37
|
|
38
|
+
// Time zone of timestamp columns if the value itself doesn’t include time zone description (eg. Asia/Tokyo)
|
55
39
|
@Config("time_zone")
|
56
40
|
@ConfigDefault("\"UTC\"")
|
57
41
|
public String getTimeZone();
|
58
|
-
|
59
42
|
}
|
60
43
|
|
61
44
|
@Override
|
@@ -22,12 +22,11 @@ import org.embulk.spi.type.Types;
|
|
22
22
|
import org.joda.time.DateTimeZone;
|
23
23
|
import org.slf4j.Logger;
|
24
24
|
|
25
|
-
import java.util.ArrayList;
|
26
25
|
import java.util.HashMap;
|
27
26
|
import java.util.List;
|
28
27
|
import java.util.Map;
|
29
28
|
|
30
|
-
import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin
|
29
|
+
import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin.PluginTask;
|
31
30
|
|
32
31
|
/**
|
33
32
|
* Created by takahiro.nakayama on 10/19/15.
|
@@ -44,6 +43,7 @@ public class FilteredPageOutput
|
|
44
43
|
private final PageReader pageReader;
|
45
44
|
private final Schema inputSchema;
|
46
45
|
private final Schema outputSchema;
|
46
|
+
private final PageBuilder pageBuilder;
|
47
47
|
private final PageOutput pageOutput;
|
48
48
|
|
49
49
|
FilteredPageOutput(PluginTask task, Schema inputSchema, Schema outputSchema, PageOutput pageOutput)
|
@@ -76,20 +76,20 @@ public class FilteredPageOutput
|
|
76
76
|
this.inputSchema = inputSchema;
|
77
77
|
this.outputSchema = outputSchema;
|
78
78
|
this.pageOutput = pageOutput;
|
79
|
+
this.pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, pageOutput);
|
79
80
|
}
|
80
81
|
|
81
82
|
@Override
|
82
83
|
public void add(Page page)
|
83
84
|
{
|
84
|
-
try
|
85
|
+
try {
|
85
86
|
pageReader.setPage(page);
|
86
87
|
|
87
88
|
while (pageReader.nextRecord()) {
|
88
|
-
|
89
|
+
setInputColumnsExceptExpandedJsonColumns(pageBuilder, inputColumnsExceptExpandedJsonColumn);
|
89
90
|
setExpandedJsonColumns(pageBuilder, jsonColumn, expandedJsonColumns, timestampParserHashMap);
|
90
91
|
pageBuilder.addRecord();
|
91
92
|
}
|
92
|
-
pageBuilder.finish();
|
93
93
|
}
|
94
94
|
catch (JsonProcessingException e) {
|
95
95
|
logger.error(e.getMessage());
|
@@ -100,6 +100,7 @@ public class FilteredPageOutput
|
|
100
100
|
@Override
|
101
101
|
public void finish()
|
102
102
|
{
|
103
|
+
pageBuilder.finish();
|
103
104
|
pageOutput.finish();
|
104
105
|
}
|
105
106
|
|
@@ -107,6 +108,7 @@ public class FilteredPageOutput
|
|
107
108
|
public void close()
|
108
109
|
{
|
109
110
|
pageReader.close();
|
111
|
+
pageBuilder.close();
|
110
112
|
pageOutput.close();
|
111
113
|
}
|
112
114
|
|
@@ -115,7 +117,13 @@ public class FilteredPageOutput
|
|
115
117
|
final HashMap<String, TimestampParser> timestampParserHashMap = Maps.newHashMap();
|
116
118
|
for (ColumnConfig expandedColumnConfig: task.getExpandedColumns()) {
|
117
119
|
if (Types.TIMESTAMP.equals(expandedColumnConfig.getType())) {
|
118
|
-
String format
|
120
|
+
String format;
|
121
|
+
if (expandedColumnConfig.getOption().has("format")) {
|
122
|
+
format = expandedColumnConfig.getOption().get(String.class, "format");
|
123
|
+
}
|
124
|
+
else {
|
125
|
+
format = task.getDefaultTimestampFormat();
|
126
|
+
}
|
119
127
|
DateTimeZone timezone = DateTimeZone.forID(task.getTimeZone());
|
120
128
|
TimestampParser parser = new TimestampParser(task.getJRuby(), format, timezone);
|
121
129
|
|
@@ -128,7 +136,7 @@ public class FilteredPageOutput
|
|
128
136
|
return timestampParserHashMap;
|
129
137
|
}
|
130
138
|
|
131
|
-
private void
|
139
|
+
private void setInputColumnsExceptExpandedJsonColumns(PageBuilder pageBuilder, List<Column> inputColumnsExceptExpandedJsonColumn) {
|
132
140
|
for (Column inputColumn: inputColumnsExceptExpandedJsonColumn) {
|
133
141
|
if (pageReader.isNull(inputColumn)) {
|
134
142
|
pageBuilder.setNull(inputColumn);
|
@@ -1,5 +1,395 @@
|
|
1
1
|
package org.embulk.filter.expand_json;
|
2
2
|
|
3
|
+
import com.fasterxml.jackson.core.JsonProcessingException;
|
4
|
+
import com.fasterxml.jackson.databind.ObjectMapper;
|
5
|
+
import com.google.common.base.Throwables;
|
6
|
+
import com.google.common.collect.ImmutableList;
|
7
|
+
import com.google.common.collect.ImmutableMap;
|
8
|
+
import com.jayway.jsonpath.InvalidJsonException;
|
9
|
+
import org.embulk.EmbulkTestRuntime;
|
10
|
+
import org.embulk.config.ConfigException;
|
11
|
+
import org.embulk.config.ConfigLoader;
|
12
|
+
import org.embulk.config.ConfigSource;
|
13
|
+
import org.embulk.config.TaskSource;
|
14
|
+
import org.embulk.spi.Column;
|
15
|
+
import org.embulk.spi.Exec;
|
16
|
+
import org.embulk.spi.Page;
|
17
|
+
import org.embulk.spi.PageOutput;
|
18
|
+
import org.embulk.spi.PageReader;
|
19
|
+
import org.embulk.spi.PageTestUtils;
|
20
|
+
import org.embulk.spi.Schema;
|
21
|
+
import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
|
22
|
+
import org.junit.Before;
|
23
|
+
import org.junit.Rule;
|
24
|
+
import org.junit.Test;
|
25
|
+
import org.junit.rules.ExpectedException;
|
26
|
+
|
27
|
+
import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin.Control;
|
28
|
+
import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin.PluginTask;
|
29
|
+
import static org.embulk.spi.type.Types.*;
|
30
|
+
import static org.junit.Assert.assertEquals;
|
31
|
+
|
3
32
|
public class TestExpandJsonFilterPlugin
|
4
33
|
{
|
34
|
+
@Rule
|
35
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
36
|
+
|
37
|
+
@Rule
|
38
|
+
public ExpectedException exception = ExpectedException.none();
|
39
|
+
|
40
|
+
|
41
|
+
private final Schema schema = Schema.builder()
|
42
|
+
.add("_c0", STRING)
|
43
|
+
.build();
|
44
|
+
private ExpandJsonFilterPlugin expandJsonFilterPlugin;
|
45
|
+
|
46
|
+
@Before
|
47
|
+
public void createResources()
|
48
|
+
{
|
49
|
+
expandJsonFilterPlugin = new ExpandJsonFilterPlugin();
|
50
|
+
}
|
51
|
+
|
52
|
+
private ConfigSource getConfigFromYaml(String yaml)
|
53
|
+
{
|
54
|
+
ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
|
55
|
+
return loader.fromYamlString(yaml);
|
56
|
+
}
|
57
|
+
|
58
|
+
private String convertToJsonString(Object object)
|
59
|
+
{
|
60
|
+
ObjectMapper mapper = new ObjectMapper();
|
61
|
+
try {
|
62
|
+
return mapper.writeValueAsString(object);
|
63
|
+
}
|
64
|
+
catch (JsonProcessingException e) {
|
65
|
+
throw Throwables.propagate(e);
|
66
|
+
}
|
67
|
+
}
|
68
|
+
|
69
|
+
private String getBrokenJsonString()
|
70
|
+
{
|
71
|
+
return "{\"_j0\": \"te\"\n";
|
72
|
+
}
|
73
|
+
|
74
|
+
/*
|
75
|
+
Config test
|
76
|
+
*/
|
77
|
+
|
78
|
+
@Test
|
79
|
+
public void testThrowExceptionAbsentJsonColumnName()
|
80
|
+
{
|
81
|
+
String configYaml = "" +
|
82
|
+
"type: expand_json\n" +
|
83
|
+
"expanded_columns:\n" +
|
84
|
+
" - {name: _c1, type: string}";
|
85
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
86
|
+
|
87
|
+
exception.expect(ConfigException.class);
|
88
|
+
exception.expectMessage("Field 'json_column_name' is required but not set");
|
89
|
+
config.loadConfig(PluginTask.class);
|
90
|
+
}
|
91
|
+
|
92
|
+
@Test
|
93
|
+
public void testThrowExceptionAbsentExpandedColumns()
|
94
|
+
{
|
95
|
+
String configYaml = "" +
|
96
|
+
"type: expand_json\n" +
|
97
|
+
"json_column_name: _c0\n";
|
98
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
99
|
+
|
100
|
+
exception.expect(ConfigException.class);
|
101
|
+
exception.expectMessage("Field 'expanded_columns' is required but not set");
|
102
|
+
config.loadConfig(PluginTask.class);
|
103
|
+
}
|
104
|
+
|
105
|
+
@Test
|
106
|
+
public void testDefaultValue()
|
107
|
+
{
|
108
|
+
String configYaml = "" +
|
109
|
+
"type: expand_json\n" +
|
110
|
+
"json_column_name: _c0\n" +
|
111
|
+
"expanded_columns:\n" +
|
112
|
+
" - {name: _j1, type: boolean}\n" +
|
113
|
+
" - {name: _j2, type: long}\n" +
|
114
|
+
" - {name: _j3, type: timestamp}\n" +
|
115
|
+
" - {name: _j4, type: double}\n" +
|
116
|
+
" - {name: _j5, type: string}\n";
|
117
|
+
|
118
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
119
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
120
|
+
|
121
|
+
assertEquals("$.", task.getRoot());
|
122
|
+
assertEquals("UTC", task.getTimeZone());
|
123
|
+
assertEquals("%Y-%m-%d %H:%M:%S.%N %z", task.getDefaultTimestampFormat());
|
124
|
+
}
|
125
|
+
|
126
|
+
/*
|
127
|
+
Expand Test
|
128
|
+
*/
|
129
|
+
|
130
|
+
@Test
|
131
|
+
public void testExpandJsonKeyToSchema()
|
132
|
+
{
|
133
|
+
String configYaml = "" +
|
134
|
+
"type: expand_json\n" +
|
135
|
+
"json_column_name: _c0\n" +
|
136
|
+
"root: $.\n" +
|
137
|
+
"expanded_columns:\n" +
|
138
|
+
" - {name: _j1, type: boolean}\n" +
|
139
|
+
" - {name: _j2, type: long}\n" +
|
140
|
+
" - {name: _j3, type: timestamp}\n" +
|
141
|
+
" - {name: _j4, type: double}\n" +
|
142
|
+
" - {name: _j5, type: string}\n";
|
143
|
+
|
144
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
145
|
+
|
146
|
+
expandJsonFilterPlugin.transaction(config, schema, new Control()
|
147
|
+
{
|
148
|
+
@Override
|
149
|
+
public void run(TaskSource taskSource, Schema outputSchema)
|
150
|
+
{
|
151
|
+
assertEquals(5, outputSchema.getColumnCount());
|
152
|
+
|
153
|
+
Column new_j1 = outputSchema.getColumn(0);
|
154
|
+
Column new_j2 = outputSchema.getColumn(1);
|
155
|
+
Column new_j3 = outputSchema.getColumn(2);
|
156
|
+
Column new_j4 = outputSchema.getColumn(3);
|
157
|
+
Column new_j5 = outputSchema.getColumn(4);
|
158
|
+
|
159
|
+
assertEquals("_j1", new_j1.getName());
|
160
|
+
assertEquals(BOOLEAN, new_j1.getType());
|
161
|
+
assertEquals("_j2", new_j2.getName());
|
162
|
+
assertEquals(LONG, new_j2.getType());
|
163
|
+
assertEquals("_j3", new_j3.getName());
|
164
|
+
assertEquals(TIMESTAMP, new_j3.getType());
|
165
|
+
assertEquals("_j4", new_j4.getName());
|
166
|
+
assertEquals(DOUBLE, new_j4.getType());
|
167
|
+
assertEquals("_j5", new_j5.getName());
|
168
|
+
assertEquals(STRING, new_j5.getType());
|
169
|
+
}
|
170
|
+
});
|
171
|
+
}
|
172
|
+
|
173
|
+
@Test
|
174
|
+
public void testExpandJsonValues()
|
175
|
+
{
|
176
|
+
String configYaml = "" +
|
177
|
+
"type: expand_json\n" +
|
178
|
+
"json_column_name: _c0\n" +
|
179
|
+
"root: $.\n" +
|
180
|
+
"time_zone: Asia/Tokyo\n" +
|
181
|
+
"expanded_columns:\n" +
|
182
|
+
" - {name: _j0, type: boolean}\n" +
|
183
|
+
" - {name: _j1, type: long}\n" +
|
184
|
+
" - {name: _j2, type: timestamp, format: '%Y-%m-%d %H:%M:%S %z'}\n" +
|
185
|
+
" - {name: _j3, type: double}\n" +
|
186
|
+
" - {name: _j4, type: string}\n" +
|
187
|
+
" - {name: _j5, type: timestamp, format: '%Y-%m-%d %H:%M:%S %z'}\n" +
|
188
|
+
" - {name: _j6, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}\n" +
|
189
|
+
// JsonPath: https://github.com/jayway/JsonPath
|
190
|
+
" - {name: '_j7.store.book[*].author', type: string}\n" +
|
191
|
+
" - {name: '_j7..book[?(@.price <= $[''_j7''][''expensive''])].author', type: string}\n" +
|
192
|
+
" - {name: '_j7..book[?(@.isbn)]', type: string}\n" +
|
193
|
+
" - {name: '_j7..book[?(@.author =~ /.*REES/i)].title', type: string}\n" +
|
194
|
+
" - {name: '_j7.store.book[2].author', type: string}\n";
|
195
|
+
|
196
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
197
|
+
|
198
|
+
expandJsonFilterPlugin.transaction(config, schema, new Control()
|
199
|
+
{
|
200
|
+
@Override
|
201
|
+
public void run(TaskSource taskSource, Schema outputSchema)
|
202
|
+
{
|
203
|
+
MockPageOutput mockPageOutput = new MockPageOutput();
|
204
|
+
PageOutput pageOutput = expandJsonFilterPlugin.open(taskSource,
|
205
|
+
schema,
|
206
|
+
outputSchema,
|
207
|
+
mockPageOutput);
|
208
|
+
|
209
|
+
ImmutableMap.Builder<String,Object> builder = ImmutableMap.builder();
|
210
|
+
builder.put("_j0", true);
|
211
|
+
builder.put("_j1", 2);
|
212
|
+
builder.put("_j2", "2014-10-21 04:44:33 +0900");
|
213
|
+
builder.put("_j3", 4.4);
|
214
|
+
builder.put("_j4", "v5");
|
215
|
+
builder.put("_j5", "2014-10-21 04:44:33 +0000");
|
216
|
+
builder.put("_j6", "2014-10-21 04:44:33");
|
217
|
+
builder.put("_j7",
|
218
|
+
ImmutableMap.of("store",
|
219
|
+
ImmutableMap.of("book",
|
220
|
+
ImmutableList.of(ImmutableMap.of("author",
|
221
|
+
"Nigel Rees",
|
222
|
+
"title",
|
223
|
+
"Sayings of the Century",
|
224
|
+
"price",
|
225
|
+
8.95),
|
226
|
+
ImmutableMap.of("author",
|
227
|
+
"Evelyn Waugh",
|
228
|
+
"title",
|
229
|
+
"Sword of Honour",
|
230
|
+
"price",
|
231
|
+
12.99),
|
232
|
+
ImmutableMap.of("author",
|
233
|
+
"Herman Melville",
|
234
|
+
"title",
|
235
|
+
"Moby Dick",
|
236
|
+
"isbn",
|
237
|
+
"0-553-21311-3",
|
238
|
+
"price",
|
239
|
+
8.99),
|
240
|
+
ImmutableMap.of("author",
|
241
|
+
"J. R. R. Tolkien",
|
242
|
+
"title",
|
243
|
+
"The Lord of the Rings",
|
244
|
+
"isbn",
|
245
|
+
"0-395-19395-8",
|
246
|
+
"price",
|
247
|
+
22.99)
|
248
|
+
),
|
249
|
+
"bicycle",
|
250
|
+
ImmutableMap.of("color",
|
251
|
+
"red",
|
252
|
+
"price",
|
253
|
+
19.95
|
254
|
+
)
|
255
|
+
),
|
256
|
+
"expensive",
|
257
|
+
10
|
258
|
+
)
|
259
|
+
/*
|
260
|
+
{
|
261
|
+
"store": {
|
262
|
+
"book": [
|
263
|
+
{
|
264
|
+
"author": "Nigel Rees",
|
265
|
+
"title": "Sayings of the Century",
|
266
|
+
"price": 8.95
|
267
|
+
},
|
268
|
+
{
|
269
|
+
"author": "Evelyn Waugh",
|
270
|
+
"title": "Sword of Honour",
|
271
|
+
"price": 12.99
|
272
|
+
},
|
273
|
+
{
|
274
|
+
"author": "Herman Melville",
|
275
|
+
"title": "Moby Dick",
|
276
|
+
"isbn": "0-553-21311-3",
|
277
|
+
"price": 8.99
|
278
|
+
},
|
279
|
+
{
|
280
|
+
"author": "J. R. R. Tolkien",
|
281
|
+
"title": "The Lord of the Rings",
|
282
|
+
"isbn": "0-395-19395-8",
|
283
|
+
"price": 22.99
|
284
|
+
}
|
285
|
+
],
|
286
|
+
"bicycle": {
|
287
|
+
"color": "red",
|
288
|
+
"price": 19.95
|
289
|
+
}
|
290
|
+
},
|
291
|
+
"expensive": 10
|
292
|
+
}
|
293
|
+
*/
|
294
|
+
);
|
295
|
+
|
296
|
+
String data = convertToJsonString(builder.build());
|
297
|
+
|
298
|
+
for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(),
|
299
|
+
schema,
|
300
|
+
data)) {
|
301
|
+
pageOutput.add(page);
|
302
|
+
}
|
303
|
+
|
304
|
+
pageOutput.finish();
|
305
|
+
pageOutput.close();
|
306
|
+
|
307
|
+
PageReader pageReader = new PageReader(outputSchema);
|
308
|
+
|
309
|
+
for (Page page : mockPageOutput.pages) {
|
310
|
+
pageReader.setPage(page);
|
311
|
+
assertEquals(true, pageReader.getBoolean(outputSchema.getColumn(0)));
|
312
|
+
assertEquals(2, pageReader.getLong(outputSchema.getColumn(1)));
|
313
|
+
assertEquals("2014-10-20 19:44:33 UTC",
|
314
|
+
pageReader.getTimestamp(outputSchema.getColumn(2)).toString());
|
315
|
+
assertEquals(String.valueOf(4.4),
|
316
|
+
String.valueOf(pageReader.getDouble(outputSchema.getColumn(3))));
|
317
|
+
assertEquals("v5", pageReader.getString(outputSchema.getColumn(4)));
|
318
|
+
assertEquals("2014-10-21 04:44:33 UTC",
|
319
|
+
pageReader.getTimestamp(outputSchema.getColumn(5)).toString());
|
320
|
+
assertEquals("2014-10-20 19:44:33 UTC",
|
321
|
+
pageReader.getTimestamp(outputSchema.getColumn(6)).toString());
|
322
|
+
assertEquals("[\"Nigel Rees\",\"Evelyn Waugh\",\"Herman Melville\",\"J. R. R. Tolkien\"]",
|
323
|
+
pageReader.getString(outputSchema.getColumn(7)));
|
324
|
+
assertEquals("[\"Nigel Rees\",\"Herman Melville\"]",
|
325
|
+
pageReader.getString(outputSchema.getColumn(8)));
|
326
|
+
assertEquals("[" +
|
327
|
+
"{" +
|
328
|
+
"\"author\":\"Herman Melville\"," +
|
329
|
+
"\"title\":\"Moby Dick\"," +
|
330
|
+
"\"isbn\":\"0-553-21311-3\"," +
|
331
|
+
"\"price\":8.99" +
|
332
|
+
"}," +
|
333
|
+
"{" +
|
334
|
+
"\"author\":\"J. R. R. Tolkien\"," +
|
335
|
+
"\"title\":\"The Lord of the Rings\"," +
|
336
|
+
"\"isbn\":\"0-395-19395-8\"," +
|
337
|
+
"\"price\":22.99" +
|
338
|
+
"}" +
|
339
|
+
"]",
|
340
|
+
pageReader.getString(outputSchema.getColumn(9)));
|
341
|
+
assertEquals("[\"Sayings of the Century\"]",
|
342
|
+
pageReader.getString(outputSchema.getColumn(10)));
|
343
|
+
assertEquals("Herman Melville",
|
344
|
+
pageReader.getString(outputSchema.getColumn(11)));
|
345
|
+
}
|
346
|
+
}
|
347
|
+
});
|
348
|
+
}
|
349
|
+
|
350
|
+
@Test
|
351
|
+
public void testAbortBrokenJsonString()
|
352
|
+
{
|
353
|
+
String configYaml = "" +
|
354
|
+
"type: expand_json\n" +
|
355
|
+
"json_column_name: _c0\n" +
|
356
|
+
"root: $.\n" +
|
357
|
+
"time_zone: Asia/Tokyo\n" +
|
358
|
+
"expanded_columns:\n" +
|
359
|
+
" - {name: _j0, type: string}\n";
|
360
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
361
|
+
|
362
|
+
expandJsonFilterPlugin.transaction(config, schema, new Control()
|
363
|
+
{
|
364
|
+
@Override
|
365
|
+
public void run(TaskSource taskSource, Schema outputSchema)
|
366
|
+
{
|
367
|
+
MockPageOutput mockPageOutput = new MockPageOutput();
|
368
|
+
PageOutput pageOutput = expandJsonFilterPlugin.open(taskSource,
|
369
|
+
schema,
|
370
|
+
outputSchema,
|
371
|
+
mockPageOutput);
|
372
|
+
|
373
|
+
String data = getBrokenJsonString();
|
374
|
+
for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(),
|
375
|
+
schema,
|
376
|
+
data)) {
|
377
|
+
exception.expect(InvalidJsonException.class);
|
378
|
+
exception.expectMessage("Unexpected End Of File position 12: null");
|
379
|
+
pageOutput.add(page);
|
380
|
+
}
|
381
|
+
|
382
|
+
pageOutput.finish();
|
383
|
+
pageOutput.close();
|
384
|
+
|
385
|
+
PageReader pageReader = new PageReader(outputSchema);
|
386
|
+
|
387
|
+
for (Page page : mockPageOutput.pages) {
|
388
|
+
pageReader.setPage(page);
|
389
|
+
assertEquals("te", pageReader.getString(outputSchema.getColumn(0)));
|
390
|
+
}
|
391
|
+
}
|
392
|
+
});
|
393
|
+
}
|
394
|
+
|
5
395
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-filter-expand_json
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Civitaspo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-11-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -46,6 +46,7 @@ extensions: []
|
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
48
|
- .gitignore
|
49
|
+
- .travis.yml
|
49
50
|
- LICENSE.txt
|
50
51
|
- README.md
|
51
52
|
- build.gradle
|
@@ -61,7 +62,7 @@ files:
|
|
61
62
|
- src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java
|
62
63
|
- classpath/asm-1.0.2.jar
|
63
64
|
- classpath/asm-3.3.1.jar
|
64
|
-
- classpath/embulk-filter-expand_json-0.0.
|
65
|
+
- classpath/embulk-filter-expand_json-0.0.3.jar
|
65
66
|
- classpath/json-path-2.0.0.jar
|
66
67
|
- classpath/json-smart-2.1.1.jar
|
67
68
|
homepage: https://github.com/civitaspo/embulk-filter-expand_json
|
Binary file
|