embulk-parser-jsonl 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: eed57dc612f9af90f47c438130dbe3469e086ed0
4
- data.tar.gz: 7c2bd81c5b583eae4ed38a7afbfdd06fa5cb99bc
3
+ metadata.gz: 7f4944036ca4cacbe9f2b8a0943503f77c92aca1
4
+ data.tar.gz: a11616b53812123915ccdd8e88938471f81b2aea
5
5
  SHA512:
6
- metadata.gz: b0a751469cdc1bb4ac979e898dd2a83b7cf9110d729fbc6a65bdd37a4215cd06e88eb64f345b2fd0d4e0bb99b7d49822db7b83f0f8a658062507a49b06e7bca4
7
- data.tar.gz: 2da25aefb298cb41351f0201055797a56efaf77be11781a8daa2eff08d0c910492cb2bf2448bb63306e137ab74b347e335c860473b696fe5427e7e8b78fba8cc
6
+ metadata.gz: bf717db902192c17124a393b063972067e2e9d46b66dfc7247cf427f03bcec551d10c799c2ec6923ca486ba44ec0017d72598280106d356eb3f9464fdbf94d0f
7
+ data.tar.gz: be23fbef9cba986f15fecebd91ba15059f1fea6fe13aaebd62373488aa092e6f42b7fb13ab471bf125c38f8ab587a7a0bfe62dc6b465dece554a80e6e0792e25
data/.gitignore CHANGED
@@ -6,3 +6,6 @@ build/
6
6
  /classpath/
7
7
  /.gradle
8
8
  /Gemfile.lock
9
+
10
+ /.idea
11
+ *.iml
data/CHANGELOG.md CHANGED
@@ -1,3 +1,8 @@
1
+ ## 0.1.2 - 2016-03-27
2
+
3
+ [new feature] Support column_options option [#4](https://github.com/shun0102/embulk-parser-jsonl/pull/4)
4
+ [maintenance] Upgrade Embulk v0.8.8 [#6](https://github.com/shun0102/embulk-parser-jsonl/pull/6)
5
+
1
6
  ## 0.1.1 - 2016-03-17
2
7
 
3
8
  [fix bug] Avoid org.embulk.spi.json.JsonParseException: Unable to parse empty string [#5](https://github.com/shun0102/embulk-parser-jsonl/pull/5)
data/build.gradle CHANGED
@@ -13,19 +13,19 @@ configurations {
13
13
  provided
14
14
  }
15
15
 
16
- version = "0.1.1"
16
+ version = "0.1.2"
17
17
 
18
18
  compileJava.options.encoding = 'UTF-8' // source encoding
19
19
  sourceCompatibility = 1.7
20
20
  targetCompatibility = 1.7
21
21
 
22
22
  dependencies {
23
- compile "org.embulk:embulk-core:0.8.2"
24
- provided "org.embulk:embulk-core:0.8.2"
23
+ compile "org.embulk:embulk-core:0.8.8"
24
+ provided "org.embulk:embulk-core:0.8.8"
25
25
 
26
26
  testCompile "junit:junit:4.+"
27
- testCompile "org.embulk:embulk-core:0.8.2:tests"
28
- testCompile "org.embulk:embulk-standards:0.8.2"
27
+ testCompile "org.embulk:embulk-core:0.8.8:tests"
28
+ testCompile "org.embulk:embulk-standards:0.8.8"
29
29
  }
30
30
 
31
31
  task classpath(type: Copy, dependsOn: ["jar"]) {
@@ -1,7 +1,7 @@
1
1
 
2
2
  Gem::Specification.new do |spec|
3
3
  spec.name = "embulk-parser-jsonl"
4
- spec.version = "0.1.1"
4
+ spec.version = "0.1.2"
5
5
  spec.authors = ["Shunsuke Mikami"]
6
6
  spec.summary = "Jsonl parser plugin for Embulk"
7
7
  spec.description = "Parses Jsonl files read by other file input plugins."
@@ -0,0 +1,23 @@
1
+ in:
2
+ type: file
3
+ path_prefix: "sample"
4
+ parser:
5
+ type: jsonl
6
+ columns:
7
+ - {name: "foo", type: "string"}
8
+ - {name: "bool", type: "boolean"}
9
+ - {name: "bool_str", type: "string"}
10
+ - {name: "int", type: "long"}
11
+ - {name: "int_str", type: "string"}
12
+ - {name: "time", type: "timestamp", format: '%Y-%m-%d %H:%M:%S'}
13
+ - {name: "double", type: "double"}
14
+ - {name: "double_str", type: "string"}
15
+ - {name: "array", type: "json"}
16
+ column_options:
17
+ bool_str: {type: "boolean"}
18
+ int_str: {type: "long"}
19
+ double_str: {type: "double"}
20
+
21
+
22
+ out:
23
+ type: stdout
@@ -0,0 +1,2 @@
1
+ {"foo": "bar", "bool": true, "bool_str": "true", "int": 1, "int_str": "42", "time": "2016-3-2 00:39:18", "double": 1.2, "double_str": "2.4", "array": [1, 2, 3]}
2
+ {"foo": null, "bool": false, "bool_str": "false", "int": 1, "int_str": "42", "time": "2016-3-2 00:39:18", "double": 1.2, "double_str": "2.4", "array": [{"inner": "hoge"}, {"inner": 1.5}]}
@@ -0,0 +1,14 @@
1
+ package org.embulk.parser.jsonl;
2
+
3
+ import org.embulk.spi.DataException;
4
+
5
+ public class JsonRecordValidateException
6
+ extends DataException {
7
+ JsonRecordValidateException(String message) {
8
+ super(message);
9
+ }
10
+
11
+ public JsonRecordValidateException(Throwable cause) {
12
+ super(cause);
13
+ }
14
+ }
@@ -0,0 +1,16 @@
1
+ package org.embulk.parser.jsonl;
2
+
3
+ import org.embulk.config.Config;
4
+ import org.embulk.config.ConfigDefault;
5
+ import org.embulk.config.Task;
6
+ import org.embulk.spi.type.Type;
7
+
8
+ import com.google.common.base.Optional;
9
+
10
+ public interface JsonlColumnOption
11
+ extends Task
12
+ {
13
+ @Config("type")
14
+ @ConfigDefault("null")
15
+ Optional<Type> getType();
16
+ }
@@ -1,6 +1,8 @@
1
1
  package org.embulk.parser.jsonl;
2
2
 
3
3
  import com.google.common.base.Optional;
4
+ import com.google.common.base.Supplier;
5
+ import com.google.common.collect.ImmutableList;
4
6
  import com.google.common.collect.ImmutableMap;
5
7
  import org.embulk.config.Config;
6
8
  import org.embulk.config.ConfigDefault;
@@ -8,8 +10,10 @@ import org.embulk.config.ConfigException;
8
10
  import org.embulk.config.ConfigSource;
9
11
  import org.embulk.config.Task;
10
12
  import org.embulk.config.TaskSource;
13
+ import org.embulk.parser.jsonl.getter.CommonColumnGetter;
14
+ import org.embulk.parser.jsonl.getter.ColumnGetterFactory;
11
15
  import org.embulk.spi.Column;
12
- import org.embulk.spi.ColumnVisitor;
16
+ import org.embulk.spi.ColumnConfig;
13
17
  import org.embulk.spi.DataException;
14
18
  import org.embulk.spi.Exec;
15
19
  import org.embulk.spi.FileInput;
@@ -23,10 +27,6 @@ import org.embulk.spi.json.JsonParser;
23
27
  import org.embulk.spi.time.TimestampParser;
24
28
  import org.embulk.spi.util.LineDecoder;
25
29
  import org.embulk.spi.util.Timestamps;
26
- import org.msgpack.core.MessageTypeException;
27
- import org.msgpack.value.BooleanValue;
28
- import org.msgpack.value.FloatValue;
29
- import org.msgpack.value.IntegerValue;
30
30
  import org.msgpack.value.Value;
31
31
  import org.slf4j.Logger;
32
32
 
@@ -52,6 +52,10 @@ public class JsonlParserPlugin
52
52
  @Config("stop_on_invalid_record")
53
53
  @ConfigDefault("false")
54
54
  boolean getStopOnInvalidRecord();
55
+
56
+ @Config("column_options")
57
+ @ConfigDefault("{}")
58
+ Map<String, JsonlColumnOption> getColumnOptions();
55
59
  }
56
60
 
57
61
  private final Logger log;
@@ -69,7 +73,14 @@ public class JsonlParserPlugin
69
73
  public void transaction(ConfigSource configSource, Control control)
70
74
  {
71
75
  PluginTask task = configSource.loadConfig(PluginTask.class);
72
- control.run(task.dump(), getSchemaConfig(task).toSchema());
76
+ SchemaConfig schemaConfig = getSchemaConfig(task);
77
+ ImmutableList.Builder<Column> columns = ImmutableList.builder();
78
+ for (int i = 0; i < schemaConfig.getColumnCount(); i++) {
79
+ ColumnConfig columnConfig = schemaConfig.getColumn(i);
80
+ JsonlColumnOption columnOption = columnOptionOf(task.getColumnOptions(), columnConfig.getName());
81
+ columns.add(new Column(i, columnConfig.getName(), columnOption.getType().or(columnConfig.getType())));
82
+ }
83
+ control.run(task.dump(), new Schema(columns.build()));
73
84
  }
74
85
 
75
86
  // this method is to keep the backward compatibility of 'schema' option.
@@ -97,12 +108,21 @@ public class JsonlParserPlugin
97
108
 
98
109
  setColumnNameValues(schema);
99
110
 
100
- final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, getSchemaConfig(task));
111
+ final SchemaConfig schemaConfig = getSchemaConfig(task);
112
+ final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, schemaConfig);
101
113
  final LineDecoder decoder = newLineDecoder(input, task);
102
114
  final JsonParser jsonParser = newJsonParser();
103
115
  final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
104
116
 
105
117
  try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
118
+ ColumnGetterFactory factory = new ColumnGetterFactory(pageBuilder, timestampParsers);
119
+ ImmutableMap.Builder<String, CommonColumnGetter> columnGettersBuilder = ImmutableMap.builder();
120
+ for (ColumnConfig columnConfig : schemaConfig.getColumns()) {
121
+ CommonColumnGetter columnGetter = factory.newColumnGetter(columnConfig);
122
+ columnGettersBuilder.put(columnConfig.getName(), columnGetter);
123
+ }
124
+ ImmutableMap<String, CommonColumnGetter> columnGetters = columnGettersBuilder.build();
125
+
106
126
  while (decoder.nextFile()) { // TODO this implementation should be improved with new JsonParser API on Embulk v0.8.3
107
127
  lineNumber = 0;
108
128
 
@@ -117,115 +137,12 @@ public class JsonlParserPlugin
117
137
  }
118
138
 
119
139
  final Map<Value, Value> record = value.asMapValue().map();
120
-
121
- schema.visitColumns(new ColumnVisitor() {
122
- @Override
123
- public void booleanColumn(Column column)
124
- {
125
- Value v = record.get(getColumnNameValue(column));
126
- if (isNil(v)) {
127
- pageBuilder.setNull(column);
128
- }
129
- else {
130
- try {
131
- pageBuilder.setBoolean(column, ((BooleanValue) v).getBoolean());
132
- }
133
- catch (MessageTypeException e) {
134
- throw new JsonRecordValidateException(e);
135
- }
136
- }
137
- }
138
-
139
- @Override
140
- public void longColumn(Column column)
141
- {
142
- Value v = record.get(getColumnNameValue(column));
143
- if (isNil(v)) {
144
- pageBuilder.setNull(column);
145
- }
146
- else {
147
- try {
148
- pageBuilder.setLong(column, ((IntegerValue) v).asLong());
149
- }
150
- catch (MessageTypeException e) {
151
- throw new JsonRecordValidateException(e);
152
- }
153
- }
154
- }
155
-
156
- @Override
157
- public void doubleColumn(Column column)
158
- {
159
- Value v = record.get(getColumnNameValue(column));
160
- if (isNil(v)) {
161
- pageBuilder.setNull(column);
162
- }
163
- else {
164
- try {
165
- pageBuilder.setDouble(column, ((FloatValue) v).toDouble());
166
- }
167
- catch (MessageTypeException e) {
168
- throw new JsonRecordValidateException(e);
169
- }
170
- }
171
- }
172
-
173
- @Override
174
- public void stringColumn(Column column)
175
- {
176
- Value v = record.get(getColumnNameValue(column));
177
- if (isNil(v)) {
178
- pageBuilder.setNull(column);
179
- }
180
- else {
181
- try {
182
- pageBuilder.setString(column, v.toString());
183
- }
184
- catch (MessageTypeException e) {
185
- throw new JsonRecordValidateException(e);
186
- }
187
- }
188
- }
189
-
190
- @Override
191
- public void timestampColumn(Column column)
192
- {
193
- Value v = record.get(getColumnNameValue(column));
194
- if (isNil(v)) {
195
- pageBuilder.setNull(column);
196
- }
197
- else {
198
- try {
199
- pageBuilder.setTimestamp(column, timestampParsers[column.getIndex()].parse(v.toString()));
200
- }
201
- catch (MessageTypeException e) {
202
- throw new JsonRecordValidateException(e);
203
- }
204
- }
205
- }
206
-
207
- @Override
208
- public void jsonColumn(Column column)
209
- {
210
- Value v = record.get(getColumnNameValue(column));
211
- if (isNil(v)) {
212
- pageBuilder.setNull(column);
213
- }
214
- else {
215
- try {
216
- pageBuilder.setJson(column, v);
217
- }
218
- catch (MessageTypeException e) {
219
- throw new JsonRecordValidateException(e);
220
- }
221
- }
222
- }
223
-
224
- private boolean isNil(Value v)
225
- {
226
- return v == null || v.isNilValue();
227
- }
228
- });
140
+ for (Column column : schema.getColumns()) {
141
+ Value v = record.get(getColumnNameValue(column));
142
+ CommonColumnGetter columnGetter = columnGetters.get(column.getName());
143
+ columnGetter.setValue(v);
144
+ column.visit(columnGetter);
145
+ }
229
146
 
230
147
  pageBuilder.addRecord();
231
148
  }
@@ -267,17 +184,17 @@ public class JsonlParserPlugin
267
184
  return new JsonParser();
268
185
  }
269
186
 
270
- static class JsonRecordValidateException
271
- extends DataException
187
+ private static JsonlColumnOption columnOptionOf(Map<String, JsonlColumnOption> columnOptions, String columnName)
272
188
  {
273
- JsonRecordValidateException(String message)
274
- {
275
- super(message);
276
- }
277
-
278
- JsonRecordValidateException(Throwable cause)
279
- {
280
- super(cause);
281
- }
189
+ return Optional.fromNullable(columnOptions.get(columnName)).or(
190
+ // default column option
191
+ new Supplier<JsonlColumnOption>()
192
+ {
193
+ public JsonlColumnOption get()
194
+ {
195
+ return Exec.newConfigSource().loadConfig(JsonlColumnOption.class);
196
+ }
197
+ });
282
198
  }
199
+
283
200
  }
@@ -0,0 +1,24 @@
1
+ package org.embulk.parser.jsonl.getter;
2
+
3
+ import org.embulk.spi.ColumnConfig;
4
+ import org.embulk.spi.PageBuilder;
5
+ import org.embulk.spi.time.TimestampParser;
6
+
7
+ public class ColumnGetterFactory {
8
+ private PageBuilder pageBuilder;
9
+ private TimestampParser[] timestampParsers;
10
+
11
+ public ColumnGetterFactory(PageBuilder pageBuilder, TimestampParser[] timestampParsers) {
12
+ this.pageBuilder = pageBuilder;
13
+ this.timestampParsers = timestampParsers;
14
+ }
15
+
16
+ public CommonColumnGetter newColumnGetter(ColumnConfig columnConfig) {
17
+ switch (columnConfig.getType().getName()) {
18
+ case "string":
19
+ return new StringColumnGetter(pageBuilder, timestampParsers);
20
+ default:
21
+ return new CommonColumnGetter(pageBuilder, timestampParsers);
22
+ }
23
+ }
24
+ }
@@ -0,0 +1,131 @@
1
+ package org.embulk.parser.jsonl.getter;
2
+
3
+ import org.embulk.parser.jsonl.JsonRecordValidateException;
4
+ import org.embulk.spi.Column;
5
+ import org.embulk.spi.ColumnVisitor;
6
+ import org.embulk.spi.PageBuilder;
7
+ import org.embulk.spi.time.TimestampParser;
8
+ import org.msgpack.core.MessageTypeException;
9
+ import org.msgpack.value.BooleanValue;
10
+ import org.msgpack.value.FloatValue;
11
+ import org.msgpack.value.IntegerValue;
12
+ import org.msgpack.value.Value;
13
+
14
+ public class CommonColumnGetter implements ColumnVisitor {
15
+ protected final PageBuilder pageBuilder;
16
+ protected final TimestampParser[] timestampParsers;
17
+
18
+ protected Value value;
19
+
20
+ public CommonColumnGetter(PageBuilder pageBuilder, TimestampParser[] timestampParsers)
21
+ {
22
+ this.pageBuilder = pageBuilder;
23
+ this.timestampParsers = timestampParsers;
24
+ }
25
+
26
+ public void setValue(Value value)
27
+ {
28
+ this.value = value;
29
+ }
30
+
31
+ @Override
32
+ public void booleanColumn(Column column)
33
+ {
34
+ if (isNil(value)) {
35
+ pageBuilder.setNull(column);
36
+ }
37
+ else {
38
+ try {
39
+ pageBuilder.setBoolean(column, ((BooleanValue) value).getBoolean());
40
+ }
41
+ catch (MessageTypeException e) {
42
+ throw new JsonRecordValidateException(e);
43
+ }
44
+ }
45
+ }
46
+
47
+ @Override
48
+ public void longColumn(Column column)
49
+ {
50
+ if (isNil(value)) {
51
+ pageBuilder.setNull(column);
52
+ }
53
+ else {
54
+ try {
55
+ pageBuilder.setLong(column, ((IntegerValue) value).asLong());
56
+ }
57
+ catch (MessageTypeException e) {
58
+ throw new JsonRecordValidateException(e);
59
+ }
60
+ }
61
+ }
62
+
63
+ @Override
64
+ public void doubleColumn(Column column)
65
+ {
66
+ if (isNil(value)) {
67
+ pageBuilder.setNull(column);
68
+ }
69
+ else {
70
+ try {
71
+ pageBuilder.setDouble(column, ((FloatValue) value).toDouble());
72
+ }
73
+ catch (MessageTypeException e) {
74
+ throw new JsonRecordValidateException(e);
75
+ }
76
+ }
77
+ }
78
+
79
+ @Override
80
+ public void stringColumn(Column column)
81
+ {
82
+ if (isNil(value)) {
83
+ pageBuilder.setNull(column);
84
+ }
85
+ else {
86
+ try {
87
+ pageBuilder.setString(column, value.toString());
88
+ }
89
+ catch (MessageTypeException e) {
90
+ throw new JsonRecordValidateException(e);
91
+ }
92
+ }
93
+ }
94
+
95
+ @Override
96
+ public void timestampColumn(Column column)
97
+ {
98
+ if (isNil(value)) {
99
+ pageBuilder.setNull(column);
100
+ }
101
+ else {
102
+ try {
103
+ pageBuilder.setTimestamp(column, timestampParsers[column.getIndex()].parse(value.toString()));
104
+ }
105
+ catch (MessageTypeException e) {
106
+ throw new JsonRecordValidateException(e);
107
+ }
108
+ }
109
+ }
110
+
111
+ @Override
112
+ public void jsonColumn(Column column)
113
+ {
114
+ if (isNil(value)) {
115
+ pageBuilder.setNull(column);
116
+ }
117
+ else {
118
+ try {
119
+ pageBuilder.setJson(column, value);
120
+ }
121
+ catch (MessageTypeException e) {
122
+ throw new JsonRecordValidateException(e);
123
+ }
124
+ }
125
+ }
126
+
127
+ protected boolean isNil(Value v)
128
+ {
129
+ return v == null || v.isNilValue();
130
+ }
131
+ }
@@ -0,0 +1,68 @@
1
+ package org.embulk.parser.jsonl.getter;
2
+
3
+ import org.embulk.parser.jsonl.JsonRecordValidateException;
4
+ import org.embulk.spi.Column;
5
+ import org.embulk.spi.PageBuilder;
6
+ import org.embulk.spi.time.TimestampParser;
7
+ import org.msgpack.core.MessageTypeException;
8
+
9
+ public class StringColumnGetter extends CommonColumnGetter {
10
+
11
+ public StringColumnGetter(PageBuilder pageBuilder, TimestampParser[] timestampParsers)
12
+ {
13
+ super(pageBuilder, timestampParsers);
14
+ }
15
+
16
+ private String getValueAsString()
17
+ {
18
+ return value.toString();
19
+ }
20
+
21
+ @Override
22
+ public void booleanColumn(Column column)
23
+ {
24
+ if (isNil(value)) {
25
+ pageBuilder.setNull(column);
26
+ }
27
+ else {
28
+ try {
29
+ pageBuilder.setBoolean(column, Boolean.valueOf(getValueAsString()));
30
+ }
31
+ catch (MessageTypeException e) {
32
+ throw new JsonRecordValidateException(e);
33
+ }
34
+ }
35
+ }
36
+
37
+ @Override
38
+ public void longColumn(Column column)
39
+ {
40
+ if (isNil(value)) {
41
+ pageBuilder.setNull(column);
42
+ }
43
+ else {
44
+ try {
45
+ pageBuilder.setLong(column, Long.valueOf(getValueAsString()));
46
+ }
47
+ catch (MessageTypeException | NumberFormatException e) {
48
+ throw new JsonRecordValidateException(e);
49
+ }
50
+ }
51
+ }
52
+
53
+ @Override
54
+ public void doubleColumn(Column column)
55
+ {
56
+ if (isNil(value)) {
57
+ pageBuilder.setNull(column);
58
+ }
59
+ else {
60
+ try {
61
+ pageBuilder.setDouble(column, Double.valueOf(getValueAsString()));
62
+ }
63
+ catch (MessageTypeException | NumberFormatException e) {
64
+ throw new JsonRecordValidateException(e);
65
+ }
66
+ }
67
+ }
68
+ }
@@ -3,10 +3,12 @@ package org.embulk.parser.jsonl;
3
3
  import com.google.common.collect.ImmutableList;
4
4
  import com.google.common.collect.Lists;
5
5
  import org.embulk.EmbulkTestRuntime;
6
+ import org.embulk.config.ConfigLoader;
6
7
  import org.embulk.config.ConfigSource;
7
8
  import org.embulk.config.TaskSource;
8
9
  import org.embulk.spi.ColumnConfig;
9
10
  import org.embulk.spi.DataException;
11
+ import org.embulk.spi.Exec;
10
12
  import org.embulk.spi.FileInput;
11
13
  import org.embulk.spi.ParserPlugin;
12
14
  import org.embulk.spi.Schema;
@@ -21,6 +23,7 @@ import org.junit.Rule;
21
23
  import org.junit.Test;
22
24
 
23
25
  import java.io.ByteArrayInputStream;
26
+ import java.io.File;
24
27
  import java.io.IOException;
25
28
  import java.io.InputStream;
26
29
  import java.util.List;
@@ -176,11 +179,57 @@ public class TestJsonlParserPlugin
176
179
  }
177
180
  }
178
181
 
182
+ @Test
183
+ public void useColumnOptions()
184
+ throws Exception
185
+ {
186
+
187
+ SchemaConfig schema = schema(
188
+ column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE));
189
+ File yamlFile = getResourceFile("use_column_options.yml");
190
+ ConfigSource config = getConfigFromYamlFile(yamlFile);
191
+
192
+ transaction(config, fileInput(
193
+ "{\"_c0\":\"true\",\"_c1\":\"10\",\"_c2\":\"0.1\"}",
194
+ "{\"_c0\":\"false\",\"_c1\":\"-10\",\"_c2\":\"1.0\"}"
195
+ ));
196
+
197
+ List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
198
+ assertEquals(2, records.size());
199
+
200
+ Object[] record;
201
+ {
202
+ record = records.get(0);
203
+ assertEquals(true, record[0]);
204
+ assertEquals(10L, record[1]);
205
+ assertEquals(0.1, (Double) record[2], 0.0001);
206
+ }
207
+ {
208
+ record = records.get(1);
209
+ assertEquals(false, record[0]);
210
+ assertEquals(-10L, record[1]);
211
+ assertEquals(1.0, (Double) record[2], 0.0001);
212
+ }
213
+ }
214
+
179
215
  private ConfigSource config()
180
216
  {
181
217
  return runtime.getExec().newConfigSource();
182
218
  }
183
219
 
220
+ private File getResourceFile(String resourceName)
221
+ throws IOException
222
+ {
223
+ return new File(this.getClass().getResource(resourceName).getFile());
224
+ }
225
+
226
+ private ConfigSource getConfigFromYamlFile(File yamlFile)
227
+ throws IOException
228
+ {
229
+ ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
230
+ return loader.fromYamlFile(yamlFile);
231
+ }
232
+
184
233
  private void transaction(ConfigSource config, final FileInput input)
185
234
  {
186
235
  plugin.transaction(config, new ParserPlugin.Control()
@@ -0,0 +1,9 @@
1
+ type: jsonl
2
+ columns:
3
+ - {name: "_c0", type: "string"}
4
+ - {name: "_c1", type: "string"}
5
+ - {name: "_c2", type: "string"}
6
+ column_options:
7
+ _c0: {type: "boolean"}
8
+ _c1: {type: "long"}
9
+ _c2: {type: "double"}
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-parser-jsonl
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Shunsuke Mikami
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-03-17 00:00:00.000000000 Z
11
+ date: 2016-03-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -53,6 +53,8 @@ files:
53
53
  - Rakefile
54
54
  - build.gradle
55
55
  - embulk-parser-jsonl.gemspec
56
+ - example/example.yml
57
+ - example/sample.json
56
58
  - gradle/wrapper/gradle-wrapper.jar
57
59
  - gradle/wrapper/gradle-wrapper.properties
58
60
  - gradlew
@@ -60,9 +62,15 @@ files:
60
62
  - lib/embulk/guess/jsonl.rb
61
63
  - lib/embulk/parser/jsonl.rb
62
64
  - settings.gradle
65
+ - src/main/java/org/embulk/parser/jsonl/JsonRecordValidateException.java
66
+ - src/main/java/org/embulk/parser/jsonl/JsonlColumnOption.java
63
67
  - src/main/java/org/embulk/parser/jsonl/JsonlParserPlugin.java
68
+ - src/main/java/org/embulk/parser/jsonl/getter/ColumnGetterFactory.java
69
+ - src/main/java/org/embulk/parser/jsonl/getter/CommonColumnGetter.java
70
+ - src/main/java/org/embulk/parser/jsonl/getter/StringColumnGetter.java
64
71
  - src/test/java/org/embulk/parser/jsonl/TestJsonlParserPlugin.java
65
- - classpath/embulk-parser-jsonl-0.1.1.jar
72
+ - src/test/resources/org/embulk/parser/jsonl/use_column_options.yml
73
+ - classpath/embulk-parser-jsonl-0.1.2.jar
66
74
  homepage: https://github.com/shun0102/embulk-parser-jsonl
67
75
  licenses:
68
76
  - MIT