embulk 0.8.5 → 0.8.6

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 99cb7ac6a07d7e8db18b43a95012b56e20ae35cc
4
- data.tar.gz: 664f254e002c1d086e73412e67a9efd32c4b4787
3
+ metadata.gz: eaa5606c99e0324b6e5057569535386d5b9e3988
4
+ data.tar.gz: a55d68015412d824e5c77e310da4564488898590
5
5
  SHA512:
6
- metadata.gz: 0076dcca57267d34e1474373a56addccbaec0a43b30dc8a23bd504ccca3725802f0e41d90caba94387e0db560090e076829f48bb34789cf8cebaeb4fe7bd93f4
7
- data.tar.gz: c0889fa0e4a95b9adb83e4460f4e4821c2c6948ac6583649286e67af305459bd5855df03fa3784da01a048d9390c7218e739e1247ff5d1b84f353047baddd611
6
+ metadata.gz: 05ede9f4f6ec3578e198827ef7e26a1089c2271c3a00c5c506b9bf7b702ce1ee650c7bcf4a39f9c3c4ce877a84d8348426c728d6a34b50c5ca46a35660adb7ae
7
+ data.tar.gz: 8b580c8b98a182465a70a3088de6c283a6b3c80c334176f40e3b08af7b915caf0918484140532f599393e2ff302cced24e79d438b8bb7defe9c8067bfe2a807a
@@ -16,10 +16,10 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
16
16
 
17
17
  allprojects {
18
18
  group = 'org.embulk'
19
- version = '0.8.5'
19
+ version = '0.8.6'
20
20
 
21
21
  ext {
22
- jrubyVersion = '9.0.4.0'
22
+ jrubyVersion = '9.0.5.0'
23
23
  }
24
24
 
25
25
  apply plugin: 'java'
@@ -38,7 +38,7 @@ dependencies {
38
38
  compile 'joda-time:joda-time:2.8.1'
39
39
  compile 'io.netty:netty-buffer:5.0.0.Alpha1'
40
40
  compile 'org.fusesource.jansi:jansi:1.11'
41
- compile 'org.msgpack:msgpack-core:0.8.1'
41
+ compile 'org.msgpack:msgpack-core:0.8.3'
42
42
 
43
43
  // For embulk/guess/charset.rb. See also embulk.gemspec
44
44
  compile 'com.ibm.icu:icu4j:54.1.1'
@@ -4,6 +4,7 @@ import org.embulk.config.Config;
4
4
  import org.embulk.config.ConfigSource;
5
5
  import org.embulk.config.Task;
6
6
  import org.embulk.config.TaskSource;
7
+ import org.embulk.spi.json.JsonParser;
7
8
  import org.embulk.spi.time.Timestamp;
8
9
  import org.embulk.spi.type.Type;
9
10
  import org.embulk.spi.Column;
@@ -56,6 +57,12 @@ public class MockParserPlugin implements ParserPlugin
56
57
  pageBuilder.setTimestamp(column,
57
58
  Timestamp.ofEpochMilli(678L));
58
59
  break;
60
+ case "json":
61
+ pageBuilder.setJson(
62
+ column,
63
+ new JsonParser().parse("{\"_c1\":true,\"_c2\":10,\"_c3\":\"embulk\",\"_c4\":{\"k\":\"v\"}}")
64
+ );
65
+ break;
59
66
  default:
60
67
  throw new IllegalStateException("Unknown type: "
61
68
  + type.getName());
@@ -121,7 +121,8 @@ public class TestFileInputRunner
121
121
  ImmutableMap.of("name", "col2", "type", "long", "option", ImmutableMap.of()),
122
122
  ImmutableMap.of("name", "col3", "type", "double", "option", ImmutableMap.of()),
123
123
  ImmutableMap.of("name", "col4", "type", "string", "option", ImmutableMap.of()),
124
- ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()))));
124
+ ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()),
125
+ ImmutableMap.of("name", "col6", "type", "json", "option", ImmutableMap.of()))));
125
126
 
126
127
  final MockPageOutput output = new MockPageOutput();
127
128
  runner.transaction(config, new InputPlugin.Control()
@@ -145,12 +146,13 @@ public class TestFileInputRunner
145
146
  List<Object[]> records = Pages.toObjects(schema, output.pages);
146
147
  assertEquals(2, records.size());
147
148
  for (Object[] record : records) {
148
- assertEquals(5, record.length);
149
+ assertEquals(6, record.length);
149
150
  assertEquals(true, record[0]);
150
151
  assertEquals(2L, record[1]);
151
152
  assertEquals(3.0D, (Double) record[2], 0.01D);
152
153
  assertEquals("45", record[3]);
153
154
  assertEquals(678L, ((Timestamp) record[4]).toEpochMilli());
155
+ assertEquals("{\"_c2\":10,\"_c1\":true,\"_c4\":{\"k\":\"v\"},\"_c3\":\"embulk\"}", record[5].toString());
154
156
  }
155
157
  }
156
158
 
@@ -171,7 +173,8 @@ public class TestFileInputRunner
171
173
  ImmutableMap.of("name", "col2", "type", "long", "option", ImmutableMap.of()),
172
174
  ImmutableMap.of("name", "col3", "type", "double", "option", ImmutableMap.of()),
173
175
  ImmutableMap.of("name", "col4", "type", "string", "option", ImmutableMap.of()),
174
- ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()))));
176
+ ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()),
177
+ ImmutableMap.of("name", "col6", "type", "json", "option", ImmutableMap.of()))));
175
178
 
176
179
  final MockPageOutput output = new MockPageOutput();
177
180
 
@@ -18,6 +18,11 @@ import org.junit.Test;
18
18
 
19
19
  import com.google.common.collect.ImmutableList;
20
20
  import com.google.common.collect.ImmutableMap;
21
+ import org.msgpack.value.ImmutableMapValue;
22
+ import static org.msgpack.value.ValueFactory.newBoolean;
23
+ import static org.msgpack.value.ValueFactory.newInteger;
24
+ import static org.msgpack.value.ValueFactory.newMap;
25
+ import static org.msgpack.value.ValueFactory.newString;
21
26
 
22
27
  public class TestFileOutputRunner
23
28
  {
@@ -110,7 +115,8 @@ public class TestFileOutputRunner
110
115
  ImmutableMap.<String,Object>of("name", "col2", "type", "long", "option", ImmutableMap.of()),
111
116
  ImmutableMap.<String,Object>of("name", "col3", "type", "double", "option", ImmutableMap.of()),
112
117
  ImmutableMap.<String,Object>of("name", "col4", "type", "string", "option", ImmutableMap.of()),
113
- ImmutableMap.<String,Object>of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()));
118
+ ImmutableMap.<String,Object>of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()),
119
+ ImmutableMap.<String,Object>of("name", "col6", "type", "json", "option", ImmutableMap.of()));
114
120
  ConfigSource config = Exec
115
121
  .newConfigSource()
116
122
  .set("type", "unused?")
@@ -128,10 +134,16 @@ public class TestFileOutputRunner
128
134
  1);
129
135
  boolean committed = false;
130
136
  try {
137
+ ImmutableMapValue jsonValue = newMap(
138
+ newString("_c1"), newBoolean(true),
139
+ newString("_c2"), newInteger(10),
140
+ newString("_c3"), newString("embulk"),
141
+ newString("_c4"), newMap(newString("k"), newString("v"))
142
+ );
131
143
  for (Page page : PageTestUtils.buildPage(
132
144
  runtime.getBufferAllocator(), schema, true, 2L,
133
- 3.0D, "45", Timestamp.ofEpochMilli(678L), true, 2L,
134
- 3.0D, "45", Timestamp.ofEpochMilli(678L))) {
145
+ 3.0D, "45", Timestamp.ofEpochMilli(678L), jsonValue, true, 2L,
146
+ 3.0D, "45", Timestamp.ofEpochMilli(678L), jsonValue)) {
135
147
  tran.add(page);
136
148
  }
137
149
  tran.commit();
@@ -154,6 +166,7 @@ public class TestFileOutputRunner
154
166
  assertEquals(3.0D, (Double) record.get(2), 0.1D);
155
167
  assertEquals("45", record.get(3));
156
168
  assertEquals(678L, ((Timestamp) record.get(4)).toEpochMilli());
169
+ assertEquals("{\"_c1\":true,\"_c2\":10,\"_c3\":\"embulk\",\"_c4\":{\"k\":\"v\"}}", record.get(5).toString());
157
170
  }
158
171
  }
159
172
 
@@ -168,7 +181,8 @@ public class TestFileOutputRunner
168
181
  ImmutableMap.<String,Object>of("name", "col2", "type", "long", "option", ImmutableMap.of()),
169
182
  ImmutableMap.<String,Object>of("name", "col3", "type", "double", "option", ImmutableMap.of()),
170
183
  ImmutableMap.<String,Object>of("name", "col4", "type", "string", "option", ImmutableMap.of()),
171
- ImmutableMap.<String,Object>of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()));
184
+ ImmutableMap.<String,Object>of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()),
185
+ ImmutableMap.<String,Object>of("name", "col6", "type", "json", "option", ImmutableMap.of()));
172
186
  ConfigSource config = Exec
173
187
  .newConfigSource()
174
188
  .set("type", "unused?")
@@ -2,17 +2,25 @@ package org.embulk.spi;
2
2
 
3
3
  import static org.embulk.spi.type.Types.BOOLEAN;
4
4
  import static org.embulk.spi.type.Types.DOUBLE;
5
+ import static org.embulk.spi.type.Types.JSON;
5
6
  import static org.embulk.spi.type.Types.LONG;
6
7
  import static org.embulk.spi.type.Types.STRING;
7
8
  import static org.embulk.spi.type.Types.TIMESTAMP;
8
9
  import static org.junit.Assert.assertEquals;
9
10
  import static org.junit.Assert.assertFalse;
10
11
  import static org.junit.Assert.assertTrue;
12
+ import org.msgpack.value.ImmutableMapValue;
13
+ import static org.msgpack.value.ValueFactory.newBoolean;
14
+ import static org.msgpack.value.ValueFactory.newInteger;
15
+ import static org.msgpack.value.ValueFactory.newMap;
16
+ import static org.msgpack.value.ValueFactory.newString;
11
17
  import java.util.ArrayList;
12
18
  import java.util.List;
13
19
  import org.embulk.spi.time.Timestamp;
14
20
  import org.embulk.spi.Schema;
15
21
  import org.embulk.EmbulkTestRuntime;
22
+ import org.msgpack.value.ImmutableMapValue;
23
+ import org.msgpack.value.Value;
16
24
  import org.junit.After;
17
25
  import org.junit.Before;
18
26
  import org.junit.Rule;
@@ -124,6 +132,12 @@ public class TestPageBuilderReader
124
132
  Timestamp.ofEpochMilli(0), Timestamp.ofEpochMilli(10));
125
133
  }
126
134
 
135
+ @Test
136
+ public void testJson()
137
+ {
138
+ check(Schema.builder().add("col1", JSON).build(), getJsonSampleData());
139
+ }
140
+
127
141
  @Test
128
142
  public void testNull()
129
143
  {
@@ -133,9 +147,10 @@ public class TestPageBuilderReader
133
147
  .add("col3", LONG)
134
148
  .add("col3", BOOLEAN)
135
149
  .add("col2", TIMESTAMP)
150
+ .add("col4", JSON)
136
151
  .build(),
137
- null, null, null, null, null,
138
- null, null, null, null, null);
152
+ null, null, null, null, null, null,
153
+ null, null, null, null, null, null);
139
154
  }
140
155
 
141
156
  @Test
@@ -147,9 +162,10 @@ public class TestPageBuilderReader
147
162
  .add("col3", LONG)
148
163
  .add("col3", BOOLEAN)
149
164
  .add("col2", TIMESTAMP)
165
+ .add("col4", JSON)
150
166
  .build(),
151
- 8122.0, "val1", 3L, false, Timestamp.ofEpochMilli(0),
152
- 140.15, "val2", Long.MAX_VALUE, true, Timestamp.ofEpochMilli(10));
167
+ 8122.0, "val1", 3L, false, Timestamp.ofEpochMilli(0), getJsonSampleData(),
168
+ 140.15, "val2", Long.MAX_VALUE, true, Timestamp.ofEpochMilli(10), getJsonSampleData());
153
169
  }
154
170
 
155
171
  private void check(Schema schema, Object... objects)
@@ -185,6 +201,8 @@ public class TestPageBuilderReader
185
201
  builder.setString(column, (String) value);
186
202
  } else if (value instanceof Timestamp) {
187
203
  builder.setTimestamp(column, (Timestamp) value);
204
+ } else if (value instanceof Value) {
205
+ builder.setJson(column, (Value) value);
188
206
  } else {
189
207
  throw new IllegalStateException(
190
208
  "Unsupported type in test utils: "
@@ -218,6 +236,8 @@ public class TestPageBuilderReader
218
236
  assertEquals(value, reader.getString(column));
219
237
  } else if (value instanceof Timestamp) {
220
238
  assertEquals(value, reader.getTimestamp(column));
239
+ } else if (value instanceof Value) {
240
+ assertEquals(value, reader.getJson(column));
221
241
  } else {
222
242
  throw new IllegalStateException(
223
243
  "Unsupported type in test utils: "
@@ -227,6 +247,16 @@ public class TestPageBuilderReader
227
247
  }
228
248
  }
229
249
 
250
+ private ImmutableMapValue getJsonSampleData()
251
+ {
252
+ return newMap(
253
+ newString("_c1"), newBoolean(true),
254
+ newString("_c2"), newInteger(10),
255
+ newString("_c3"), newString("embulk"),
256
+ newString("_c4"), newMap(newString("k"), newString("v"))
257
+ );
258
+ }
259
+
230
260
  @Test
231
261
  public void testEmptySchema()
232
262
  {
@@ -4,6 +4,7 @@ Release Notes
4
4
  .. toctree::
5
5
  :maxdepth: 1
6
6
 
7
+ release/release-0.8.6
7
8
  release/release-0.8.5
8
9
  release/release-0.8.4
9
10
  release/release-0.8.3
@@ -0,0 +1,14 @@
1
+ Release 0.8.6
2
+ ==================================
3
+
4
+ General Changes
5
+ ------------------
6
+
7
+ * Upgraded JRuby from 9.0.4.0 to 9.0.5.0. See `JRuby 9.0.5.0 release notes <http://jruby.org/2016/01/26/jruby-9-0-5-0.html>`_ for details.
8
+
9
+ * Upgraded msgpack-java from 0.8.1 to 0.8.3. See `MessagePack Java release notes <https://github.com/msgpack/msgpack-java/blob/develop/RELEASE_NOTES.md>`_ for details.
10
+
11
+
12
+ Release Date
13
+ ------------------
14
+ 2016-02-24
@@ -156,7 +156,7 @@ public class TestJsonParserPlugin
156
156
  sb.append(line).append("\n");
157
157
  }
158
158
 
159
- ByteArrayInputStream in = new ByteArrayInputStream(sb.toString().getBytes());
159
+ ByteArrayInputStream in = new ByteArrayInputStream(sb.toString().getBytes("UTF-8"));
160
160
  return new InputStreamFileInput(runtime.getBufferAllocator(), provider(in));
161
161
  }
162
162
 
@@ -1,3 +1,3 @@
1
1
  module Embulk
2
- VERSION = '0.8.5'
2
+ VERSION = '0.8.6'
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.5
4
+ version: 0.8.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-02-18 00:00:00.000000000 Z
11
+ date: 2016-02-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: jruby-jars
@@ -108,8 +108,8 @@ files:
108
108
  - classpath/commons-beanutils-core-1.8.3.jar
109
109
  - classpath/commons-compress-1.10.jar
110
110
  - classpath/commons-lang3-3.1.jar
111
- - classpath/embulk-core-0.8.5.jar
112
- - classpath/embulk-standards-0.8.5.jar
111
+ - classpath/embulk-core-0.8.6.jar
112
+ - classpath/embulk-standards-0.8.6.jar
113
113
  - classpath/guava-18.0.jar
114
114
  - classpath/guice-4.0.jar
115
115
  - classpath/guice-bootstrap-0.1.1.jar
@@ -125,7 +125,7 @@ files:
125
125
  - classpath/joda-time-2.8.1.jar
126
126
  - classpath/logback-classic-1.1.3.jar
127
127
  - classpath/logback-core-1.1.3.jar
128
- - classpath/msgpack-core-0.8.1.jar
128
+ - classpath/msgpack-core-0.8.3.jar
129
129
  - classpath/netty-buffer-5.0.0.Alpha1.jar
130
130
  - classpath/netty-common-5.0.0.Alpha1.jar
131
131
  - classpath/slf4j-api-1.7.12.jar
@@ -425,6 +425,7 @@ files:
425
425
  - embulk-docs/src/release/release-0.8.3.rst
426
426
  - embulk-docs/src/release/release-0.8.4.rst
427
427
  - embulk-docs/src/release/release-0.8.5.rst
428
+ - embulk-docs/src/release/release-0.8.6.rst
428
429
  - embulk-standards/build.gradle
429
430
  - embulk-standards/src/main/java/org/embulk/standards/Bzip2FileDecoderPlugin.java
430
431
  - embulk-standards/src/main/java/org/embulk/standards/Bzip2FileEncoderPlugin.java