embulk 0.8.5 → 0.8.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/build.gradle +2 -2
- data/embulk-core/build.gradle +1 -1
- data/embulk-core/src/test/java/org/embulk/spi/MockParserPlugin.java +7 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestFileInputRunner.java +6 -3
- data/embulk-core/src/test/java/org/embulk/spi/TestFileOutputRunner.java +18 -4
- data/embulk-core/src/test/java/org/embulk/spi/TestPageBuilderReader.java +34 -4
- data/embulk-docs/src/release.rst +1 -0
- data/embulk-docs/src/release/release-0.8.6.rst +14 -0
- data/embulk-standards/src/test/java/org/embulk/standards/TestJsonParserPlugin.java +1 -1
- data/lib/embulk/version.rb +1 -1
- metadata +6 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: eaa5606c99e0324b6e5057569535386d5b9e3988
|
4
|
+
data.tar.gz: a55d68015412d824e5c77e310da4564488898590
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 05ede9f4f6ec3578e198827ef7e26a1089c2271c3a00c5c506b9bf7b702ce1ee650c7bcf4a39f9c3c4ce877a84d8348426c728d6a34b50c5ca46a35660adb7ae
|
7
|
+
data.tar.gz: 8b580c8b98a182465a70a3088de6c283a6b3c80c334176f40e3b08af7b915caf0918484140532f599393e2ff302cced24e79d438b8bb7defe9c8067bfe2a807a
|
data/build.gradle
CHANGED
@@ -16,10 +16,10 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
|
|
16
16
|
|
17
17
|
allprojects {
|
18
18
|
group = 'org.embulk'
|
19
|
-
version = '0.8.
|
19
|
+
version = '0.8.6'
|
20
20
|
|
21
21
|
ext {
|
22
|
-
jrubyVersion = '9.0.
|
22
|
+
jrubyVersion = '9.0.5.0'
|
23
23
|
}
|
24
24
|
|
25
25
|
apply plugin: 'java'
|
data/embulk-core/build.gradle
CHANGED
@@ -38,7 +38,7 @@ dependencies {
|
|
38
38
|
compile 'joda-time:joda-time:2.8.1'
|
39
39
|
compile 'io.netty:netty-buffer:5.0.0.Alpha1'
|
40
40
|
compile 'org.fusesource.jansi:jansi:1.11'
|
41
|
-
compile 'org.msgpack:msgpack-core:0.8.
|
41
|
+
compile 'org.msgpack:msgpack-core:0.8.3'
|
42
42
|
|
43
43
|
// For embulk/guess/charset.rb. See also embulk.gemspec
|
44
44
|
compile 'com.ibm.icu:icu4j:54.1.1'
|
@@ -4,6 +4,7 @@ import org.embulk.config.Config;
|
|
4
4
|
import org.embulk.config.ConfigSource;
|
5
5
|
import org.embulk.config.Task;
|
6
6
|
import org.embulk.config.TaskSource;
|
7
|
+
import org.embulk.spi.json.JsonParser;
|
7
8
|
import org.embulk.spi.time.Timestamp;
|
8
9
|
import org.embulk.spi.type.Type;
|
9
10
|
import org.embulk.spi.Column;
|
@@ -56,6 +57,12 @@ public class MockParserPlugin implements ParserPlugin
|
|
56
57
|
pageBuilder.setTimestamp(column,
|
57
58
|
Timestamp.ofEpochMilli(678L));
|
58
59
|
break;
|
60
|
+
case "json":
|
61
|
+
pageBuilder.setJson(
|
62
|
+
column,
|
63
|
+
new JsonParser().parse("{\"_c1\":true,\"_c2\":10,\"_c3\":\"embulk\",\"_c4\":{\"k\":\"v\"}}")
|
64
|
+
);
|
65
|
+
break;
|
59
66
|
default:
|
60
67
|
throw new IllegalStateException("Unknown type: "
|
61
68
|
+ type.getName());
|
@@ -121,7 +121,8 @@ public class TestFileInputRunner
|
|
121
121
|
ImmutableMap.of("name", "col2", "type", "long", "option", ImmutableMap.of()),
|
122
122
|
ImmutableMap.of("name", "col3", "type", "double", "option", ImmutableMap.of()),
|
123
123
|
ImmutableMap.of("name", "col4", "type", "string", "option", ImmutableMap.of()),
|
124
|
-
ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of())
|
124
|
+
ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()),
|
125
|
+
ImmutableMap.of("name", "col6", "type", "json", "option", ImmutableMap.of()))));
|
125
126
|
|
126
127
|
final MockPageOutput output = new MockPageOutput();
|
127
128
|
runner.transaction(config, new InputPlugin.Control()
|
@@ -145,12 +146,13 @@ public class TestFileInputRunner
|
|
145
146
|
List<Object[]> records = Pages.toObjects(schema, output.pages);
|
146
147
|
assertEquals(2, records.size());
|
147
148
|
for (Object[] record : records) {
|
148
|
-
assertEquals(
|
149
|
+
assertEquals(6, record.length);
|
149
150
|
assertEquals(true, record[0]);
|
150
151
|
assertEquals(2L, record[1]);
|
151
152
|
assertEquals(3.0D, (Double) record[2], 0.01D);
|
152
153
|
assertEquals("45", record[3]);
|
153
154
|
assertEquals(678L, ((Timestamp) record[4]).toEpochMilli());
|
155
|
+
assertEquals("{\"_c2\":10,\"_c1\":true,\"_c4\":{\"k\":\"v\"},\"_c3\":\"embulk\"}", record[5].toString());
|
154
156
|
}
|
155
157
|
}
|
156
158
|
|
@@ -171,7 +173,8 @@ public class TestFileInputRunner
|
|
171
173
|
ImmutableMap.of("name", "col2", "type", "long", "option", ImmutableMap.of()),
|
172
174
|
ImmutableMap.of("name", "col3", "type", "double", "option", ImmutableMap.of()),
|
173
175
|
ImmutableMap.of("name", "col4", "type", "string", "option", ImmutableMap.of()),
|
174
|
-
ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of())
|
176
|
+
ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()),
|
177
|
+
ImmutableMap.of("name", "col6", "type", "json", "option", ImmutableMap.of()))));
|
175
178
|
|
176
179
|
final MockPageOutput output = new MockPageOutput();
|
177
180
|
|
@@ -18,6 +18,11 @@ import org.junit.Test;
|
|
18
18
|
|
19
19
|
import com.google.common.collect.ImmutableList;
|
20
20
|
import com.google.common.collect.ImmutableMap;
|
21
|
+
import org.msgpack.value.ImmutableMapValue;
|
22
|
+
import static org.msgpack.value.ValueFactory.newBoolean;
|
23
|
+
import static org.msgpack.value.ValueFactory.newInteger;
|
24
|
+
import static org.msgpack.value.ValueFactory.newMap;
|
25
|
+
import static org.msgpack.value.ValueFactory.newString;
|
21
26
|
|
22
27
|
public class TestFileOutputRunner
|
23
28
|
{
|
@@ -110,7 +115,8 @@ public class TestFileOutputRunner
|
|
110
115
|
ImmutableMap.<String,Object>of("name", "col2", "type", "long", "option", ImmutableMap.of()),
|
111
116
|
ImmutableMap.<String,Object>of("name", "col3", "type", "double", "option", ImmutableMap.of()),
|
112
117
|
ImmutableMap.<String,Object>of("name", "col4", "type", "string", "option", ImmutableMap.of()),
|
113
|
-
ImmutableMap.<String,Object>of("name", "col5", "type", "timestamp", "option", ImmutableMap.of())
|
118
|
+
ImmutableMap.<String,Object>of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()),
|
119
|
+
ImmutableMap.<String,Object>of("name", "col6", "type", "json", "option", ImmutableMap.of()));
|
114
120
|
ConfigSource config = Exec
|
115
121
|
.newConfigSource()
|
116
122
|
.set("type", "unused?")
|
@@ -128,10 +134,16 @@ public class TestFileOutputRunner
|
|
128
134
|
1);
|
129
135
|
boolean committed = false;
|
130
136
|
try {
|
137
|
+
ImmutableMapValue jsonValue = newMap(
|
138
|
+
newString("_c1"), newBoolean(true),
|
139
|
+
newString("_c2"), newInteger(10),
|
140
|
+
newString("_c3"), newString("embulk"),
|
141
|
+
newString("_c4"), newMap(newString("k"), newString("v"))
|
142
|
+
);
|
131
143
|
for (Page page : PageTestUtils.buildPage(
|
132
144
|
runtime.getBufferAllocator(), schema, true, 2L,
|
133
|
-
3.0D, "45", Timestamp.ofEpochMilli(678L), true, 2L,
|
134
|
-
3.0D, "45", Timestamp.ofEpochMilli(678L))) {
|
145
|
+
3.0D, "45", Timestamp.ofEpochMilli(678L), jsonValue, true, 2L,
|
146
|
+
3.0D, "45", Timestamp.ofEpochMilli(678L), jsonValue)) {
|
135
147
|
tran.add(page);
|
136
148
|
}
|
137
149
|
tran.commit();
|
@@ -154,6 +166,7 @@ public class TestFileOutputRunner
|
|
154
166
|
assertEquals(3.0D, (Double) record.get(2), 0.1D);
|
155
167
|
assertEquals("45", record.get(3));
|
156
168
|
assertEquals(678L, ((Timestamp) record.get(4)).toEpochMilli());
|
169
|
+
assertEquals("{\"_c1\":true,\"_c2\":10,\"_c3\":\"embulk\",\"_c4\":{\"k\":\"v\"}}", record.get(5).toString());
|
157
170
|
}
|
158
171
|
}
|
159
172
|
|
@@ -168,7 +181,8 @@ public class TestFileOutputRunner
|
|
168
181
|
ImmutableMap.<String,Object>of("name", "col2", "type", "long", "option", ImmutableMap.of()),
|
169
182
|
ImmutableMap.<String,Object>of("name", "col3", "type", "double", "option", ImmutableMap.of()),
|
170
183
|
ImmutableMap.<String,Object>of("name", "col4", "type", "string", "option", ImmutableMap.of()),
|
171
|
-
ImmutableMap.<String,Object>of("name", "col5", "type", "timestamp", "option", ImmutableMap.of())
|
184
|
+
ImmutableMap.<String,Object>of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()),
|
185
|
+
ImmutableMap.<String,Object>of("name", "col6", "type", "json", "option", ImmutableMap.of()));
|
172
186
|
ConfigSource config = Exec
|
173
187
|
.newConfigSource()
|
174
188
|
.set("type", "unused?")
|
@@ -2,17 +2,25 @@ package org.embulk.spi;
|
|
2
2
|
|
3
3
|
import static org.embulk.spi.type.Types.BOOLEAN;
|
4
4
|
import static org.embulk.spi.type.Types.DOUBLE;
|
5
|
+
import static org.embulk.spi.type.Types.JSON;
|
5
6
|
import static org.embulk.spi.type.Types.LONG;
|
6
7
|
import static org.embulk.spi.type.Types.STRING;
|
7
8
|
import static org.embulk.spi.type.Types.TIMESTAMP;
|
8
9
|
import static org.junit.Assert.assertEquals;
|
9
10
|
import static org.junit.Assert.assertFalse;
|
10
11
|
import static org.junit.Assert.assertTrue;
|
12
|
+
import org.msgpack.value.ImmutableMapValue;
|
13
|
+
import static org.msgpack.value.ValueFactory.newBoolean;
|
14
|
+
import static org.msgpack.value.ValueFactory.newInteger;
|
15
|
+
import static org.msgpack.value.ValueFactory.newMap;
|
16
|
+
import static org.msgpack.value.ValueFactory.newString;
|
11
17
|
import java.util.ArrayList;
|
12
18
|
import java.util.List;
|
13
19
|
import org.embulk.spi.time.Timestamp;
|
14
20
|
import org.embulk.spi.Schema;
|
15
21
|
import org.embulk.EmbulkTestRuntime;
|
22
|
+
import org.msgpack.value.ImmutableMapValue;
|
23
|
+
import org.msgpack.value.Value;
|
16
24
|
import org.junit.After;
|
17
25
|
import org.junit.Before;
|
18
26
|
import org.junit.Rule;
|
@@ -124,6 +132,12 @@ public class TestPageBuilderReader
|
|
124
132
|
Timestamp.ofEpochMilli(0), Timestamp.ofEpochMilli(10));
|
125
133
|
}
|
126
134
|
|
135
|
+
@Test
|
136
|
+
public void testJson()
|
137
|
+
{
|
138
|
+
check(Schema.builder().add("col1", JSON).build(), getJsonSampleData());
|
139
|
+
}
|
140
|
+
|
127
141
|
@Test
|
128
142
|
public void testNull()
|
129
143
|
{
|
@@ -133,9 +147,10 @@ public class TestPageBuilderReader
|
|
133
147
|
.add("col3", LONG)
|
134
148
|
.add("col3", BOOLEAN)
|
135
149
|
.add("col2", TIMESTAMP)
|
150
|
+
.add("col4", JSON)
|
136
151
|
.build(),
|
137
|
-
null, null, null, null, null,
|
138
|
-
null, null, null, null, null);
|
152
|
+
null, null, null, null, null, null,
|
153
|
+
null, null, null, null, null, null);
|
139
154
|
}
|
140
155
|
|
141
156
|
@Test
|
@@ -147,9 +162,10 @@ public class TestPageBuilderReader
|
|
147
162
|
.add("col3", LONG)
|
148
163
|
.add("col3", BOOLEAN)
|
149
164
|
.add("col2", TIMESTAMP)
|
165
|
+
.add("col4", JSON)
|
150
166
|
.build(),
|
151
|
-
8122.0, "val1", 3L, false, Timestamp.ofEpochMilli(0),
|
152
|
-
140.15, "val2", Long.MAX_VALUE, true, Timestamp.ofEpochMilli(10));
|
167
|
+
8122.0, "val1", 3L, false, Timestamp.ofEpochMilli(0), getJsonSampleData(),
|
168
|
+
140.15, "val2", Long.MAX_VALUE, true, Timestamp.ofEpochMilli(10), getJsonSampleData());
|
153
169
|
}
|
154
170
|
|
155
171
|
private void check(Schema schema, Object... objects)
|
@@ -185,6 +201,8 @@ public class TestPageBuilderReader
|
|
185
201
|
builder.setString(column, (String) value);
|
186
202
|
} else if (value instanceof Timestamp) {
|
187
203
|
builder.setTimestamp(column, (Timestamp) value);
|
204
|
+
} else if (value instanceof Value) {
|
205
|
+
builder.setJson(column, (Value) value);
|
188
206
|
} else {
|
189
207
|
throw new IllegalStateException(
|
190
208
|
"Unsupported type in test utils: "
|
@@ -218,6 +236,8 @@ public class TestPageBuilderReader
|
|
218
236
|
assertEquals(value, reader.getString(column));
|
219
237
|
} else if (value instanceof Timestamp) {
|
220
238
|
assertEquals(value, reader.getTimestamp(column));
|
239
|
+
} else if (value instanceof Value) {
|
240
|
+
assertEquals(value, reader.getJson(column));
|
221
241
|
} else {
|
222
242
|
throw new IllegalStateException(
|
223
243
|
"Unsupported type in test utils: "
|
@@ -227,6 +247,16 @@ public class TestPageBuilderReader
|
|
227
247
|
}
|
228
248
|
}
|
229
249
|
|
250
|
+
private ImmutableMapValue getJsonSampleData()
|
251
|
+
{
|
252
|
+
return newMap(
|
253
|
+
newString("_c1"), newBoolean(true),
|
254
|
+
newString("_c2"), newInteger(10),
|
255
|
+
newString("_c3"), newString("embulk"),
|
256
|
+
newString("_c4"), newMap(newString("k"), newString("v"))
|
257
|
+
);
|
258
|
+
}
|
259
|
+
|
230
260
|
@Test
|
231
261
|
public void testEmptySchema()
|
232
262
|
{
|
data/embulk-docs/src/release.rst
CHANGED
@@ -0,0 +1,14 @@
|
|
1
|
+
Release 0.8.6
|
2
|
+
==================================
|
3
|
+
|
4
|
+
General Changes
|
5
|
+
------------------
|
6
|
+
|
7
|
+
* Upgraded JRuby from 9.0.4.0 to 9.0.5.0. See `JRuby 9.0.5.0 release notes <http://jruby.org/2016/01/26/jruby-9-0-5-0.html>`_ for details.
|
8
|
+
|
9
|
+
* Upgraded msgpack-java from 0.8.1 to 0.8.3. See `MessagePack Java release notes <https://github.com/msgpack/msgpack-java/blob/develop/RELEASE_NOTES.md>`_ for details.
|
10
|
+
|
11
|
+
|
12
|
+
Release Date
|
13
|
+
------------------
|
14
|
+
2016-02-24
|
@@ -156,7 +156,7 @@ public class TestJsonParserPlugin
|
|
156
156
|
sb.append(line).append("\n");
|
157
157
|
}
|
158
158
|
|
159
|
-
ByteArrayInputStream in = new ByteArrayInputStream(sb.toString().getBytes());
|
159
|
+
ByteArrayInputStream in = new ByteArrayInputStream(sb.toString().getBytes("UTF-8"));
|
160
160
|
return new InputStreamFileInput(runtime.getBufferAllocator(), provider(in));
|
161
161
|
}
|
162
162
|
|
data/lib/embulk/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-02-
|
11
|
+
date: 2016-02-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: jruby-jars
|
@@ -108,8 +108,8 @@ files:
|
|
108
108
|
- classpath/commons-beanutils-core-1.8.3.jar
|
109
109
|
- classpath/commons-compress-1.10.jar
|
110
110
|
- classpath/commons-lang3-3.1.jar
|
111
|
-
- classpath/embulk-core-0.8.
|
112
|
-
- classpath/embulk-standards-0.8.
|
111
|
+
- classpath/embulk-core-0.8.6.jar
|
112
|
+
- classpath/embulk-standards-0.8.6.jar
|
113
113
|
- classpath/guava-18.0.jar
|
114
114
|
- classpath/guice-4.0.jar
|
115
115
|
- classpath/guice-bootstrap-0.1.1.jar
|
@@ -125,7 +125,7 @@ files:
|
|
125
125
|
- classpath/joda-time-2.8.1.jar
|
126
126
|
- classpath/logback-classic-1.1.3.jar
|
127
127
|
- classpath/logback-core-1.1.3.jar
|
128
|
-
- classpath/msgpack-core-0.8.
|
128
|
+
- classpath/msgpack-core-0.8.3.jar
|
129
129
|
- classpath/netty-buffer-5.0.0.Alpha1.jar
|
130
130
|
- classpath/netty-common-5.0.0.Alpha1.jar
|
131
131
|
- classpath/slf4j-api-1.7.12.jar
|
@@ -425,6 +425,7 @@ files:
|
|
425
425
|
- embulk-docs/src/release/release-0.8.3.rst
|
426
426
|
- embulk-docs/src/release/release-0.8.4.rst
|
427
427
|
- embulk-docs/src/release/release-0.8.5.rst
|
428
|
+
- embulk-docs/src/release/release-0.8.6.rst
|
428
429
|
- embulk-standards/build.gradle
|
429
430
|
- embulk-standards/src/main/java/org/embulk/standards/Bzip2FileDecoderPlugin.java
|
430
431
|
- embulk-standards/src/main/java/org/embulk/standards/Bzip2FileEncoderPlugin.java
|