embulk-filter-expand_json 0.0.2 → 0.0.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 021e96f2218ac3326f0c470f860bde498866eff9
4
- data.tar.gz: dd085b630641bed9a2829b26af1bb03e5ad75680
3
+ metadata.gz: a7c9576abd91635ad11888a88fba5f14e576383e
4
+ data.tar.gz: 6a80bd358b9056620075fc16f1f7ae59a5a911c9
5
5
  SHA512:
6
- metadata.gz: 4d6a8fc2b5b53e7b31d8246b231a48ff796cb4d74fcc0cadb713ff22d990327f623e337e56a97c0e0c0ffeb3b1027171199a1193962a6d6854ca552c0de78a2a
7
- data.tar.gz: d3ad561c853b5677aa1e968007086dd1bcaf7cf6e116119b0b31906d7dd5922bf8c9920bdbdd74f2c6fcfdaa6dc0c8ffcf88c2545611189b2b734d8fcf416756
6
+ metadata.gz: 63ae1b2d8c541497ba9bc0c892a403b30c32e6ac8222eb6bc70d09675c0fc9ec2e26a6b5fbd9443e7d5e2ee479b6bbaa8efaf8d8e1f4995b839f9fd1d4600089
7
+ data.tar.gz: 73e53f220990bfe91249eceef14450039c29461a20821963e281d99462fb76fdf6583784f4270301c0a46f6b805259143dd1473d787f21a9c18306718885f6d8
data/.travis.yml ADDED
@@ -0,0 +1,9 @@
1
+ language: java
2
+ jdk:
3
+ - openjdk7
4
+ - oraclejdk7
5
+ - oraclejdk8
6
+ script:
7
+ - ./gradlew test
8
+ after_success:
9
+ - ./gradlew jacocoTestReport coveralls
data/README.md CHANGED
@@ -1,5 +1,8 @@
1
1
  # Expand Json filter plugin for Embulk
2
2
 
3
+ [![Build Status](https://travis-ci.org/civitaspo/embulk-filter-expand_json.svg)](https://travis-ci.org/civitaspo/embulk-filter-expand_json)
4
+ [![Coverage Status](https://coveralls.io/repos/civitaspo/embulk-filter-expand_json/badge.svg?branch=master&service=github)](https://coveralls.io/github/civitaspo/embulk-filter-expand_json?branch=master)
5
+
3
6
  expand columns having json into multiple columns
4
7
 
5
8
  ## Overview
@@ -9,7 +12,7 @@ expand columns having json into multiple columns
9
12
  ## Configuration
10
13
 
11
14
  - **json_column_name**: a column name having json to be expanded (string, required)
12
- - **root**: root property to start fetching each entries, specify in [JsonPath](http://goessner.net/articles/JsonPath/) style (string, default: `\"$.\"`)
15
+ - **root**: root property to start fetching each entries, specify in [JsonPath](http://goessner.net/articles/JsonPath/) style (string, default: `"$."`)
13
16
  - **expanded_columns**: columns expanded into multiple columns (array of hash, required)
14
17
  - **name**: name of the column. you can define [JsonPath](http://goessner.net/articles/JsonPath/) style.
15
18
  - **type**: type of the column (see below)
data/build.gradle CHANGED
@@ -1,6 +1,8 @@
1
1
  plugins {
2
2
  id "com.jfrog.bintray" version "1.1"
3
3
  id "com.github.jruby-gradle.base" version "0.1.5"
4
+ id "com.github.kt3k.coveralls" version "2.4.0"
5
+ id "jacoco"
4
6
  id "java"
5
7
  }
6
8
  import com.github.jrubygradle.JRubyExec
@@ -12,15 +14,23 @@ configurations {
12
14
  provided
13
15
  }
14
16
 
15
- version = "0.0.2"
17
+ version = "0.0.3"
16
18
  sourceCompatibility = 1.7
17
19
  targetCompatibility = 1.7
18
20
 
19
21
  dependencies {
20
- compile "org.embulk:embulk-core:0.7.4"
21
- provided "org.embulk:embulk-core:0.7.4"
22
+ compile "org.embulk:embulk-core:0.7.+"
23
+ provided "org.embulk:embulk-core:0.7.+"
22
24
  compile "com.jayway.jsonpath:json-path:2.+"
23
25
  testCompile "junit:junit:4.+"
26
+ testCompile "org.embulk:embulk-core:0.7.+:tests"
27
+ }
28
+
29
+ jacocoTestReport {
30
+ reports {
31
+ xml.enabled = true // coveralls plugin depends on xml format report
32
+ html.enabled = true
33
+ }
24
34
  }
25
35
 
26
36
  task classpath(type: Copy, dependsOn: ["jar"]) {
@@ -1,14 +1,6 @@
1
1
  package org.embulk.filter.expand_json;
2
2
 
3
- import com.fasterxml.jackson.core.JsonProcessingException;
4
- import com.fasterxml.jackson.databind.ObjectMapper;
5
- import com.google.common.base.Throwables;
6
3
  import com.google.common.collect.ImmutableList;
7
- import com.google.common.collect.Maps;
8
- import com.jayway.jsonpath.Configuration;
9
- import com.jayway.jsonpath.JsonPath;
10
- import com.jayway.jsonpath.Option;
11
- import com.jayway.jsonpath.ReadContext;
12
4
  import org.embulk.config.Config;
13
5
  import org.embulk.config.ConfigDefault;
14
6
  import org.embulk.config.ConfigSource;
@@ -18,21 +10,12 @@ import org.embulk.spi.Column;
18
10
  import org.embulk.spi.ColumnConfig;
19
11
  import org.embulk.spi.Exec;
20
12
  import org.embulk.spi.FilterPlugin;
21
- import org.embulk.spi.Page;
22
- import org.embulk.spi.PageBuilder;
23
13
  import org.embulk.spi.PageOutput;
24
- import org.embulk.spi.PageReader;
25
14
  import org.embulk.spi.Schema;
26
15
  import org.embulk.spi.time.TimestampParser;
27
- import org.embulk.spi.type.Types;
28
- import org.joda.time.DateTimeZone;
29
- import org.jruby.embed.ScriptingContainer;
30
16
  import org.slf4j.Logger;
31
17
 
32
- import java.util.ArrayList;
33
- import java.util.HashMap;
34
18
  import java.util.List;
35
- import java.util.Map;
36
19
 
37
20
  public class ExpandJsonFilterPlugin
38
21
  implements FilterPlugin
@@ -52,10 +35,10 @@ public class ExpandJsonFilterPlugin
52
35
  @Config("expanded_columns")
53
36
  public List<ColumnConfig> getExpandedColumns();
54
37
 
38
+ // Time zone of timestamp columns if the value itself doesn’t include time zone description (eg. Asia/Tokyo)
55
39
  @Config("time_zone")
56
40
  @ConfigDefault("\"UTC\"")
57
41
  public String getTimeZone();
58
-
59
42
  }
60
43
 
61
44
  @Override
@@ -22,12 +22,11 @@ import org.embulk.spi.type.Types;
22
22
  import org.joda.time.DateTimeZone;
23
23
  import org.slf4j.Logger;
24
24
 
25
- import java.util.ArrayList;
26
25
  import java.util.HashMap;
27
26
  import java.util.List;
28
27
  import java.util.Map;
29
28
 
30
- import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin.*;
29
+ import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin.PluginTask;
31
30
 
32
31
  /**
33
32
  * Created by takahiro.nakayama on 10/19/15.
@@ -44,6 +43,7 @@ public class FilteredPageOutput
44
43
  private final PageReader pageReader;
45
44
  private final Schema inputSchema;
46
45
  private final Schema outputSchema;
46
+ private final PageBuilder pageBuilder;
47
47
  private final PageOutput pageOutput;
48
48
 
49
49
  FilteredPageOutput(PluginTask task, Schema inputSchema, Schema outputSchema, PageOutput pageOutput)
@@ -76,20 +76,20 @@ public class FilteredPageOutput
76
76
  this.inputSchema = inputSchema;
77
77
  this.outputSchema = outputSchema;
78
78
  this.pageOutput = pageOutput;
79
+ this.pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, pageOutput);
79
80
  }
80
81
 
81
82
  @Override
82
83
  public void add(Page page)
83
84
  {
84
- try (PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, pageOutput)) {
85
+ try {
85
86
  pageReader.setPage(page);
86
87
 
87
88
  while (pageReader.nextRecord()) {
88
- setInputColumnsExceptFlattenJsonColumns(pageBuilder, inputColumnsExceptExpandedJsonColumn);
89
+ setInputColumnsExceptExpandedJsonColumns(pageBuilder, inputColumnsExceptExpandedJsonColumn);
89
90
  setExpandedJsonColumns(pageBuilder, jsonColumn, expandedJsonColumns, timestampParserHashMap);
90
91
  pageBuilder.addRecord();
91
92
  }
92
- pageBuilder.finish();
93
93
  }
94
94
  catch (JsonProcessingException e) {
95
95
  logger.error(e.getMessage());
@@ -100,6 +100,7 @@ public class FilteredPageOutput
100
100
  @Override
101
101
  public void finish()
102
102
  {
103
+ pageBuilder.finish();
103
104
  pageOutput.finish();
104
105
  }
105
106
 
@@ -107,6 +108,7 @@ public class FilteredPageOutput
107
108
  public void close()
108
109
  {
109
110
  pageReader.close();
111
+ pageBuilder.close();
110
112
  pageOutput.close();
111
113
  }
112
114
 
@@ -115,7 +117,13 @@ public class FilteredPageOutput
115
117
  final HashMap<String, TimestampParser> timestampParserHashMap = Maps.newHashMap();
116
118
  for (ColumnConfig expandedColumnConfig: task.getExpandedColumns()) {
117
119
  if (Types.TIMESTAMP.equals(expandedColumnConfig.getType())) {
118
- String format = expandedColumnConfig.getOption().get(String.class, "format");
120
+ String format;
121
+ if (expandedColumnConfig.getOption().has("format")) {
122
+ format = expandedColumnConfig.getOption().get(String.class, "format");
123
+ }
124
+ else {
125
+ format = task.getDefaultTimestampFormat();
126
+ }
119
127
  DateTimeZone timezone = DateTimeZone.forID(task.getTimeZone());
120
128
  TimestampParser parser = new TimestampParser(task.getJRuby(), format, timezone);
121
129
 
@@ -128,7 +136,7 @@ public class FilteredPageOutput
128
136
  return timestampParserHashMap;
129
137
  }
130
138
 
131
- private void setInputColumnsExceptFlattenJsonColumns(PageBuilder pageBuilder, List<Column> inputColumnsExceptExpandedJsonColumn) {
139
+ private void setInputColumnsExceptExpandedJsonColumns(PageBuilder pageBuilder, List<Column> inputColumnsExceptExpandedJsonColumn) {
132
140
  for (Column inputColumn: inputColumnsExceptExpandedJsonColumn) {
133
141
  if (pageReader.isNull(inputColumn)) {
134
142
  pageBuilder.setNull(inputColumn);
@@ -1,5 +1,395 @@
1
1
  package org.embulk.filter.expand_json;
2
2
 
3
+ import com.fasterxml.jackson.core.JsonProcessingException;
4
+ import com.fasterxml.jackson.databind.ObjectMapper;
5
+ import com.google.common.base.Throwables;
6
+ import com.google.common.collect.ImmutableList;
7
+ import com.google.common.collect.ImmutableMap;
8
+ import com.jayway.jsonpath.InvalidJsonException;
9
+ import org.embulk.EmbulkTestRuntime;
10
+ import org.embulk.config.ConfigException;
11
+ import org.embulk.config.ConfigLoader;
12
+ import org.embulk.config.ConfigSource;
13
+ import org.embulk.config.TaskSource;
14
+ import org.embulk.spi.Column;
15
+ import org.embulk.spi.Exec;
16
+ import org.embulk.spi.Page;
17
+ import org.embulk.spi.PageOutput;
18
+ import org.embulk.spi.PageReader;
19
+ import org.embulk.spi.PageTestUtils;
20
+ import org.embulk.spi.Schema;
21
+ import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
22
+ import org.junit.Before;
23
+ import org.junit.Rule;
24
+ import org.junit.Test;
25
+ import org.junit.rules.ExpectedException;
26
+
27
+ import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin.Control;
28
+ import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin.PluginTask;
29
+ import static org.embulk.spi.type.Types.*;
30
+ import static org.junit.Assert.assertEquals;
31
+
3
32
  public class TestExpandJsonFilterPlugin
4
33
  {
34
+ @Rule
35
+ public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
36
+
37
+ @Rule
38
+ public ExpectedException exception = ExpectedException.none();
39
+
40
+
41
+ private final Schema schema = Schema.builder()
42
+ .add("_c0", STRING)
43
+ .build();
44
+ private ExpandJsonFilterPlugin expandJsonFilterPlugin;
45
+
46
+ @Before
47
+ public void createResources()
48
+ {
49
+ expandJsonFilterPlugin = new ExpandJsonFilterPlugin();
50
+ }
51
+
52
+ private ConfigSource getConfigFromYaml(String yaml)
53
+ {
54
+ ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
55
+ return loader.fromYamlString(yaml);
56
+ }
57
+
58
+ private String convertToJsonString(Object object)
59
+ {
60
+ ObjectMapper mapper = new ObjectMapper();
61
+ try {
62
+ return mapper.writeValueAsString(object);
63
+ }
64
+ catch (JsonProcessingException e) {
65
+ throw Throwables.propagate(e);
66
+ }
67
+ }
68
+
69
+ private String getBrokenJsonString()
70
+ {
71
+ return "{\"_j0\": \"te\"\n";
72
+ }
73
+
74
+ /*
75
+ Config test
76
+ */
77
+
78
+ @Test
79
+ public void testThrowExceptionAbsentJsonColumnName()
80
+ {
81
+ String configYaml = "" +
82
+ "type: expand_json\n" +
83
+ "expanded_columns:\n" +
84
+ " - {name: _c1, type: string}";
85
+ ConfigSource config = getConfigFromYaml(configYaml);
86
+
87
+ exception.expect(ConfigException.class);
88
+ exception.expectMessage("Field 'json_column_name' is required but not set");
89
+ config.loadConfig(PluginTask.class);
90
+ }
91
+
92
+ @Test
93
+ public void testThrowExceptionAbsentExpandedColumns()
94
+ {
95
+ String configYaml = "" +
96
+ "type: expand_json\n" +
97
+ "json_column_name: _c0\n";
98
+ ConfigSource config = getConfigFromYaml(configYaml);
99
+
100
+ exception.expect(ConfigException.class);
101
+ exception.expectMessage("Field 'expanded_columns' is required but not set");
102
+ config.loadConfig(PluginTask.class);
103
+ }
104
+
105
+ @Test
106
+ public void testDefaultValue()
107
+ {
108
+ String configYaml = "" +
109
+ "type: expand_json\n" +
110
+ "json_column_name: _c0\n" +
111
+ "expanded_columns:\n" +
112
+ " - {name: _j1, type: boolean}\n" +
113
+ " - {name: _j2, type: long}\n" +
114
+ " - {name: _j3, type: timestamp}\n" +
115
+ " - {name: _j4, type: double}\n" +
116
+ " - {name: _j5, type: string}\n";
117
+
118
+ ConfigSource config = getConfigFromYaml(configYaml);
119
+ PluginTask task = config.loadConfig(PluginTask.class);
120
+
121
+ assertEquals("$.", task.getRoot());
122
+ assertEquals("UTC", task.getTimeZone());
123
+ assertEquals("%Y-%m-%d %H:%M:%S.%N %z", task.getDefaultTimestampFormat());
124
+ }
125
+
126
+ /*
127
+ Expand Test
128
+ */
129
+
130
+ @Test
131
+ public void testExpandJsonKeyToSchema()
132
+ {
133
+ String configYaml = "" +
134
+ "type: expand_json\n" +
135
+ "json_column_name: _c0\n" +
136
+ "root: $.\n" +
137
+ "expanded_columns:\n" +
138
+ " - {name: _j1, type: boolean}\n" +
139
+ " - {name: _j2, type: long}\n" +
140
+ " - {name: _j3, type: timestamp}\n" +
141
+ " - {name: _j4, type: double}\n" +
142
+ " - {name: _j5, type: string}\n";
143
+
144
+ ConfigSource config = getConfigFromYaml(configYaml);
145
+
146
+ expandJsonFilterPlugin.transaction(config, schema, new Control()
147
+ {
148
+ @Override
149
+ public void run(TaskSource taskSource, Schema outputSchema)
150
+ {
151
+ assertEquals(5, outputSchema.getColumnCount());
152
+
153
+ Column new_j1 = outputSchema.getColumn(0);
154
+ Column new_j2 = outputSchema.getColumn(1);
155
+ Column new_j3 = outputSchema.getColumn(2);
156
+ Column new_j4 = outputSchema.getColumn(3);
157
+ Column new_j5 = outputSchema.getColumn(4);
158
+
159
+ assertEquals("_j1", new_j1.getName());
160
+ assertEquals(BOOLEAN, new_j1.getType());
161
+ assertEquals("_j2", new_j2.getName());
162
+ assertEquals(LONG, new_j2.getType());
163
+ assertEquals("_j3", new_j3.getName());
164
+ assertEquals(TIMESTAMP, new_j3.getType());
165
+ assertEquals("_j4", new_j4.getName());
166
+ assertEquals(DOUBLE, new_j4.getType());
167
+ assertEquals("_j5", new_j5.getName());
168
+ assertEquals(STRING, new_j5.getType());
169
+ }
170
+ });
171
+ }
172
+
173
+ @Test
174
+ public void testExpandJsonValues()
175
+ {
176
+ String configYaml = "" +
177
+ "type: expand_json\n" +
178
+ "json_column_name: _c0\n" +
179
+ "root: $.\n" +
180
+ "time_zone: Asia/Tokyo\n" +
181
+ "expanded_columns:\n" +
182
+ " - {name: _j0, type: boolean}\n" +
183
+ " - {name: _j1, type: long}\n" +
184
+ " - {name: _j2, type: timestamp, format: '%Y-%m-%d %H:%M:%S %z'}\n" +
185
+ " - {name: _j3, type: double}\n" +
186
+ " - {name: _j4, type: string}\n" +
187
+ " - {name: _j5, type: timestamp, format: '%Y-%m-%d %H:%M:%S %z'}\n" +
188
+ " - {name: _j6, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}\n" +
189
+ // JsonPath: https://github.com/jayway/JsonPath
190
+ " - {name: '_j7.store.book[*].author', type: string}\n" +
191
+ " - {name: '_j7..book[?(@.price <= $[''_j7''][''expensive''])].author', type: string}\n" +
192
+ " - {name: '_j7..book[?(@.isbn)]', type: string}\n" +
193
+ " - {name: '_j7..book[?(@.author =~ /.*REES/i)].title', type: string}\n" +
194
+ " - {name: '_j7.store.book[2].author', type: string}\n";
195
+
196
+ ConfigSource config = getConfigFromYaml(configYaml);
197
+
198
+ expandJsonFilterPlugin.transaction(config, schema, new Control()
199
+ {
200
+ @Override
201
+ public void run(TaskSource taskSource, Schema outputSchema)
202
+ {
203
+ MockPageOutput mockPageOutput = new MockPageOutput();
204
+ PageOutput pageOutput = expandJsonFilterPlugin.open(taskSource,
205
+ schema,
206
+ outputSchema,
207
+ mockPageOutput);
208
+
209
+ ImmutableMap.Builder<String,Object> builder = ImmutableMap.builder();
210
+ builder.put("_j0", true);
211
+ builder.put("_j1", 2);
212
+ builder.put("_j2", "2014-10-21 04:44:33 +0900");
213
+ builder.put("_j3", 4.4);
214
+ builder.put("_j4", "v5");
215
+ builder.put("_j5", "2014-10-21 04:44:33 +0000");
216
+ builder.put("_j6", "2014-10-21 04:44:33");
217
+ builder.put("_j7",
218
+ ImmutableMap.of("store",
219
+ ImmutableMap.of("book",
220
+ ImmutableList.of(ImmutableMap.of("author",
221
+ "Nigel Rees",
222
+ "title",
223
+ "Sayings of the Century",
224
+ "price",
225
+ 8.95),
226
+ ImmutableMap.of("author",
227
+ "Evelyn Waugh",
228
+ "title",
229
+ "Sword of Honour",
230
+ "price",
231
+ 12.99),
232
+ ImmutableMap.of("author",
233
+ "Herman Melville",
234
+ "title",
235
+ "Moby Dick",
236
+ "isbn",
237
+ "0-553-21311-3",
238
+ "price",
239
+ 8.99),
240
+ ImmutableMap.of("author",
241
+ "J. R. R. Tolkien",
242
+ "title",
243
+ "The Lord of the Rings",
244
+ "isbn",
245
+ "0-395-19395-8",
246
+ "price",
247
+ 22.99)
248
+ ),
249
+ "bicycle",
250
+ ImmutableMap.of("color",
251
+ "red",
252
+ "price",
253
+ 19.95
254
+ )
255
+ ),
256
+ "expensive",
257
+ 10
258
+ )
259
+ /*
260
+ {
261
+ "store": {
262
+ "book": [
263
+ {
264
+ "author": "Nigel Rees",
265
+ "title": "Sayings of the Century",
266
+ "price": 8.95
267
+ },
268
+ {
269
+ "author": "Evelyn Waugh",
270
+ "title": "Sword of Honour",
271
+ "price": 12.99
272
+ },
273
+ {
274
+ "author": "Herman Melville",
275
+ "title": "Moby Dick",
276
+ "isbn": "0-553-21311-3",
277
+ "price": 8.99
278
+ },
279
+ {
280
+ "author": "J. R. R. Tolkien",
281
+ "title": "The Lord of the Rings",
282
+ "isbn": "0-395-19395-8",
283
+ "price": 22.99
284
+ }
285
+ ],
286
+ "bicycle": {
287
+ "color": "red",
288
+ "price": 19.95
289
+ }
290
+ },
291
+ "expensive": 10
292
+ }
293
+ */
294
+ );
295
+
296
+ String data = convertToJsonString(builder.build());
297
+
298
+ for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(),
299
+ schema,
300
+ data)) {
301
+ pageOutput.add(page);
302
+ }
303
+
304
+ pageOutput.finish();
305
+ pageOutput.close();
306
+
307
+ PageReader pageReader = new PageReader(outputSchema);
308
+
309
+ for (Page page : mockPageOutput.pages) {
310
+ pageReader.setPage(page);
311
+ assertEquals(true, pageReader.getBoolean(outputSchema.getColumn(0)));
312
+ assertEquals(2, pageReader.getLong(outputSchema.getColumn(1)));
313
+ assertEquals("2014-10-20 19:44:33 UTC",
314
+ pageReader.getTimestamp(outputSchema.getColumn(2)).toString());
315
+ assertEquals(String.valueOf(4.4),
316
+ String.valueOf(pageReader.getDouble(outputSchema.getColumn(3))));
317
+ assertEquals("v5", pageReader.getString(outputSchema.getColumn(4)));
318
+ assertEquals("2014-10-21 04:44:33 UTC",
319
+ pageReader.getTimestamp(outputSchema.getColumn(5)).toString());
320
+ assertEquals("2014-10-20 19:44:33 UTC",
321
+ pageReader.getTimestamp(outputSchema.getColumn(6)).toString());
322
+ assertEquals("[\"Nigel Rees\",\"Evelyn Waugh\",\"Herman Melville\",\"J. R. R. Tolkien\"]",
323
+ pageReader.getString(outputSchema.getColumn(7)));
324
+ assertEquals("[\"Nigel Rees\",\"Herman Melville\"]",
325
+ pageReader.getString(outputSchema.getColumn(8)));
326
+ assertEquals("[" +
327
+ "{" +
328
+ "\"author\":\"Herman Melville\"," +
329
+ "\"title\":\"Moby Dick\"," +
330
+ "\"isbn\":\"0-553-21311-3\"," +
331
+ "\"price\":8.99" +
332
+ "}," +
333
+ "{" +
334
+ "\"author\":\"J. R. R. Tolkien\"," +
335
+ "\"title\":\"The Lord of the Rings\"," +
336
+ "\"isbn\":\"0-395-19395-8\"," +
337
+ "\"price\":22.99" +
338
+ "}" +
339
+ "]",
340
+ pageReader.getString(outputSchema.getColumn(9)));
341
+ assertEquals("[\"Sayings of the Century\"]",
342
+ pageReader.getString(outputSchema.getColumn(10)));
343
+ assertEquals("Herman Melville",
344
+ pageReader.getString(outputSchema.getColumn(11)));
345
+ }
346
+ }
347
+ });
348
+ }
349
+
350
+ @Test
351
+ public void testAbortBrokenJsonString()
352
+ {
353
+ String configYaml = "" +
354
+ "type: expand_json\n" +
355
+ "json_column_name: _c0\n" +
356
+ "root: $.\n" +
357
+ "time_zone: Asia/Tokyo\n" +
358
+ "expanded_columns:\n" +
359
+ " - {name: _j0, type: string}\n";
360
+ ConfigSource config = getConfigFromYaml(configYaml);
361
+
362
+ expandJsonFilterPlugin.transaction(config, schema, new Control()
363
+ {
364
+ @Override
365
+ public void run(TaskSource taskSource, Schema outputSchema)
366
+ {
367
+ MockPageOutput mockPageOutput = new MockPageOutput();
368
+ PageOutput pageOutput = expandJsonFilterPlugin.open(taskSource,
369
+ schema,
370
+ outputSchema,
371
+ mockPageOutput);
372
+
373
+ String data = getBrokenJsonString();
374
+ for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(),
375
+ schema,
376
+ data)) {
377
+ exception.expect(InvalidJsonException.class);
378
+ exception.expectMessage("Unexpected End Of File position 12: null");
379
+ pageOutput.add(page);
380
+ }
381
+
382
+ pageOutput.finish();
383
+ pageOutput.close();
384
+
385
+ PageReader pageReader = new PageReader(outputSchema);
386
+
387
+ for (Page page : mockPageOutput.pages) {
388
+ pageReader.setPage(page);
389
+ assertEquals("te", pageReader.getString(outputSchema.getColumn(0)));
390
+ }
391
+ }
392
+ });
393
+ }
394
+
5
395
  }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-filter-expand_json
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Civitaspo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-10-19 00:00:00.000000000 Z
11
+ date: 2015-11-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -46,6 +46,7 @@ extensions: []
46
46
  extra_rdoc_files: []
47
47
  files:
48
48
  - .gitignore
49
+ - .travis.yml
49
50
  - LICENSE.txt
50
51
  - README.md
51
52
  - build.gradle
@@ -61,7 +62,7 @@ files:
61
62
  - src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java
62
63
  - classpath/asm-1.0.2.jar
63
64
  - classpath/asm-3.3.1.jar
64
- - classpath/embulk-filter-expand_json-0.0.2.jar
65
+ - classpath/embulk-filter-expand_json-0.0.3.jar
65
66
  - classpath/json-path-2.0.0.jar
66
67
  - classpath/json-smart-2.1.1.jar
67
68
  homepage: https://github.com/civitaspo/embulk-filter-expand_json