embulk 0.7.11-java → 0.8.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +3 -3
- data/README.md +1 -1
- data/build.gradle +2 -2
- data/embulk-core/build.gradle +2 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigLoader.java +11 -3
- data/embulk-core/src/main/java/org/embulk/config/YamlTagResolver.java +53 -0
- data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java +0 -1
- data/embulk-core/src/main/java/org/embulk/exec/LocalExecutorPlugin.java +479 -69
- data/embulk-core/src/main/java/org/embulk/spi/Column.java +3 -0
- data/embulk-core/src/main/java/org/embulk/spi/ColumnVisitor.java +2 -0
- data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java +12 -5
- data/embulk-core/src/main/java/org/embulk/spi/Page.java +19 -0
- data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java +26 -5
- data/embulk-core/src/main/java/org/embulk/spi/PageReader.java +13 -0
- data/embulk-core/src/main/java/org/embulk/spi/json/JsonParseException.java +17 -0
- data/embulk-core/src/main/java/org/embulk/spi/json/JsonParser.java +125 -0
- data/embulk-core/src/main/java/org/embulk/spi/json/RubyValueApi.java +55 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/JsonType.java +14 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/TypeDeserializer.java +1 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/Types.java +2 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetterFactory.java +6 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/PagePrinter.java +5 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Pages.java +10 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java +3 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/BooleanColumnSetter.java +7 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DefaultValueSetter.java +2 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DoubleColumnSetter.java +7 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/JsonColumnSetter.java +73 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/LongColumnSetter.java +11 -2
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/NullDefaultValueSetter.java +5 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/SkipColumnSetter.java +5 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/StringColumnSetter.java +7 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/TimestampColumnSetter.java +9 -1
- data/embulk-core/src/test/java/org/embulk/spi/MockFormatterPlugin.java +7 -0
- data/embulk-docs/src/built-in.rst +40 -3
- data/embulk-docs/src/conf.py +2 -2
- data/embulk-docs/src/release.rst +1 -1
- data/embulk-docs/src/release/release-0.8.0.rst +68 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java +12 -1
- data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +18 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java +1 -1
- data/embulk.gemspec +1 -1
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/gradlew +3 -7
- data/lib/embulk/column.rb +2 -0
- data/lib/embulk/command/embulk_migrate_plugin.rb +76 -10
- data/lib/embulk/command/embulk_new_plugin.rb +2 -0
- data/lib/embulk/command/embulk_run.rb +17 -10
- data/lib/embulk/data/bundle/.ruby-version +1 -1
- data/lib/embulk/data/new/java/build.gradle.erb +21 -0
- data/lib/embulk/data/new/java/config/checkstyle/checkstyle.xml +128 -0
- data/lib/embulk/data/new/java/config/checkstyle/default.xml +108 -0
- data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/lib/embulk/data/new/java/gradlew +3 -7
- data/lib/embulk/data/new/ruby/.ruby-version +1 -1
- data/lib/embulk/guess/csv.rb +1 -1
- data/lib/embulk/guess/schema_guess.rb +6 -0
- data/lib/embulk/guess_plugin.rb +1 -1
- data/lib/embulk/java/imports.rb +4 -0
- data/lib/embulk/plugin_registry.rb +8 -12
- data/lib/embulk/schema.rb +6 -0
- data/lib/embulk/version.rb +1 -1
- data/test/guess/test_csv_guess.rb +170 -0
- data/test/helper.rb +2 -0
- metadata +17 -15
- data/embulk-core/src/main/java/org/embulk/exec/LocalThreadExecutor.java +0 -34
- data/embulk-core/src/main/java/org/embulk/guice/Bootstrap.java +0 -157
- data/embulk-core/src/main/java/org/embulk/guice/CloseableInjector.java +0 -22
- data/embulk-core/src/main/java/org/embulk/guice/InjectorProxy.java +0 -145
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleInjector.java +0 -26
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleInjectorProxy.java +0 -61
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleManager.java +0 -187
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleMethods.java +0 -89
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleMethodsMap.java +0 -38
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleModule.java +0 -97
- data/embulk-docs/src/release/release-0.7.11.rst +0 -13
@@ -8,11 +8,13 @@ import org.embulk.spi.type.LongType;
|
|
8
8
|
import org.embulk.spi.type.DoubleType;
|
9
9
|
import org.embulk.spi.type.StringType;
|
10
10
|
import org.embulk.spi.type.TimestampType;
|
11
|
+
import org.embulk.spi.type.JsonType;
|
11
12
|
import org.embulk.spi.util.dynamic.BooleanColumnSetter;
|
12
13
|
import org.embulk.spi.util.dynamic.LongColumnSetter;
|
13
14
|
import org.embulk.spi.util.dynamic.DoubleColumnSetter;
|
14
15
|
import org.embulk.spi.util.dynamic.StringColumnSetter;
|
15
16
|
import org.embulk.spi.util.dynamic.TimestampColumnSetter;
|
17
|
+
import org.embulk.spi.util.dynamic.JsonColumnSetter;
|
16
18
|
import org.embulk.spi.util.dynamic.DefaultValueSetter;
|
17
19
|
import org.embulk.spi.util.dynamic.NullDefaultValueSetter;
|
18
20
|
import org.embulk.spi.time.TimestampFormatter;
|
@@ -58,6 +60,10 @@ public class DynamicColumnSetterFactory
|
|
58
60
|
TimestampParser parser = new TimestampParser(task.getJRuby(),
|
59
61
|
getTimestampFormat(column).getFormat(), getTimeZone(column));
|
60
62
|
return new TimestampColumnSetter(pageBuilder, column, defaultValue, parser);
|
63
|
+
} else if (type instanceof JsonType) {
|
64
|
+
TimestampFormatter formatter = new TimestampFormatter(task.getJRuby(),
|
65
|
+
getTimestampFormat(column).getFormat(), getTimeZone(column));
|
66
|
+
return new JsonColumnSetter(pageBuilder, column, defaultValue, formatter);
|
61
67
|
}
|
62
68
|
throw new ConfigException("Unknown column type: "+type);
|
63
69
|
}
|
@@ -107,6 +107,16 @@ public class Pages
|
|
107
107
|
visit(column, record.getTimestamp(column));
|
108
108
|
}
|
109
109
|
}
|
110
|
+
|
111
|
+
@Override
|
112
|
+
public void jsonColumn(Column column)
|
113
|
+
{
|
114
|
+
if (record.isNull(column)) {
|
115
|
+
visit(column, null);
|
116
|
+
} else {
|
117
|
+
visit(column, record.getJson(column));
|
118
|
+
}
|
119
|
+
}
|
110
120
|
}
|
111
121
|
|
112
122
|
public static Object getObject(PageReader record, Column column)
|
@@ -12,6 +12,7 @@ import org.embulk.spi.PageBuilder;
|
|
12
12
|
import org.embulk.spi.Column;
|
13
13
|
import org.embulk.spi.util.DynamicColumnSetter;
|
14
14
|
import org.embulk.spi.time.Timestamp;
|
15
|
+
import org.msgpack.value.Value;
|
15
16
|
|
16
17
|
public abstract class AbstractDynamicColumnSetter
|
17
18
|
implements DynamicColumnSetter
|
@@ -40,6 +41,8 @@ public abstract class AbstractDynamicColumnSetter
|
|
40
41
|
|
41
42
|
public abstract void set(Timestamp value);
|
42
43
|
|
44
|
+
public abstract void set(Value value);
|
45
|
+
|
43
46
|
public void setRubyObject(IRubyObject rubyObject)
|
44
47
|
{
|
45
48
|
if (rubyObject == null || rubyObject instanceof RubyNil) {
|
@@ -4,6 +4,7 @@ import com.google.common.collect.ImmutableSet;
|
|
4
4
|
import org.embulk.spi.Column;
|
5
5
|
import org.embulk.spi.PageBuilder;
|
6
6
|
import org.embulk.spi.time.Timestamp;
|
7
|
+
import org.msgpack.value.Value;
|
7
8
|
|
8
9
|
public class BooleanColumnSetter
|
9
10
|
extends AbstractDynamicColumnSetter
|
@@ -61,4 +62,10 @@ public class BooleanColumnSetter
|
|
61
62
|
{
|
62
63
|
defaultValue.setBoolean(pageBuilder, column);
|
63
64
|
}
|
65
|
+
|
66
|
+
@Override
|
67
|
+
public void set(Value v)
|
68
|
+
{
|
69
|
+
pageBuilder.setJson(column, v);
|
70
|
+
}
|
64
71
|
}
|
@@ -4,6 +4,7 @@ import java.math.RoundingMode;
|
|
4
4
|
import org.embulk.spi.Column;
|
5
5
|
import org.embulk.spi.PageBuilder;
|
6
6
|
import org.embulk.spi.time.Timestamp;
|
7
|
+
import org.msgpack.value.Value;
|
7
8
|
|
8
9
|
public class DoubleColumnSetter
|
9
10
|
extends AbstractDynamicColumnSetter
|
@@ -58,4 +59,10 @@ public class DoubleColumnSetter
|
|
58
59
|
double frac = v.getNano() / 1000000000.0;
|
59
60
|
pageBuilder.setDouble(column, sec + frac);
|
60
61
|
}
|
62
|
+
|
63
|
+
@Override
|
64
|
+
public void set(Value v)
|
65
|
+
{
|
66
|
+
pageBuilder.setJson(column, v);
|
67
|
+
}
|
61
68
|
}
|
@@ -0,0 +1,73 @@
|
|
1
|
+
package org.embulk.spi.util.dynamic;
|
2
|
+
|
3
|
+
import org.embulk.spi.Column;
|
4
|
+
import org.embulk.spi.PageBuilder;
|
5
|
+
import org.embulk.spi.time.Timestamp;
|
6
|
+
import org.embulk.spi.time.TimestampFormatter;
|
7
|
+
import org.embulk.spi.json.JsonParser;
|
8
|
+
import org.embulk.spi.json.JsonParseException;
|
9
|
+
import org.msgpack.value.Value;
|
10
|
+
import org.msgpack.value.ValueFactory;
|
11
|
+
|
12
|
+
public class JsonColumnSetter
|
13
|
+
extends AbstractDynamicColumnSetter
|
14
|
+
{
|
15
|
+
private final TimestampFormatter timestampFormatter;
|
16
|
+
private final JsonParser jsonParser;
|
17
|
+
|
18
|
+
public JsonColumnSetter(PageBuilder pageBuilder, Column column,
|
19
|
+
DefaultValueSetter defaultValue,
|
20
|
+
TimestampFormatter timestampFormatter)
|
21
|
+
{
|
22
|
+
super(pageBuilder, column, defaultValue);
|
23
|
+
this.timestampFormatter = timestampFormatter;
|
24
|
+
this.jsonParser = new JsonParser();
|
25
|
+
}
|
26
|
+
|
27
|
+
@Override
|
28
|
+
public void setNull()
|
29
|
+
{
|
30
|
+
pageBuilder.setNull(column);
|
31
|
+
}
|
32
|
+
|
33
|
+
@Override
|
34
|
+
public void set(boolean v)
|
35
|
+
{
|
36
|
+
pageBuilder.setJson(column, ValueFactory.newBoolean(v));
|
37
|
+
}
|
38
|
+
|
39
|
+
@Override
|
40
|
+
public void set(long v)
|
41
|
+
{
|
42
|
+
pageBuilder.setJson(column, ValueFactory.newInteger(v));
|
43
|
+
}
|
44
|
+
|
45
|
+
@Override
|
46
|
+
public void set(double v)
|
47
|
+
{
|
48
|
+
pageBuilder.setJson(column, ValueFactory.newFloat(v));
|
49
|
+
}
|
50
|
+
|
51
|
+
@Override
|
52
|
+
public void set(String v)
|
53
|
+
{
|
54
|
+
try {
|
55
|
+
pageBuilder.setJson(column, jsonParser.parse(v));
|
56
|
+
}
|
57
|
+
catch (JsonParseException ex) {
|
58
|
+
defaultValue.setTimestamp(pageBuilder, column);
|
59
|
+
}
|
60
|
+
}
|
61
|
+
|
62
|
+
@Override
|
63
|
+
public void set(Timestamp v)
|
64
|
+
{
|
65
|
+
pageBuilder.setJson(column, ValueFactory.newString(timestampFormatter.format(v)));
|
66
|
+
}
|
67
|
+
|
68
|
+
@Override
|
69
|
+
public void set(Value v)
|
70
|
+
{
|
71
|
+
pageBuilder.setJson(column, v);
|
72
|
+
}
|
73
|
+
}
|
@@ -5,6 +5,7 @@ import com.google.common.math.DoubleMath;
|
|
5
5
|
import org.embulk.spi.Column;
|
6
6
|
import org.embulk.spi.PageBuilder;
|
7
7
|
import org.embulk.spi.time.Timestamp;
|
8
|
+
import org.msgpack.value.Value;
|
8
9
|
|
9
10
|
public class LongColumnSetter
|
10
11
|
extends AbstractDynamicColumnSetter
|
@@ -40,7 +41,8 @@ public class LongColumnSetter
|
|
40
41
|
try {
|
41
42
|
// TODO configurable rounding mode
|
42
43
|
lv = DoubleMath.roundToLong(v, RoundingMode.HALF_UP);
|
43
|
-
}
|
44
|
+
}
|
45
|
+
catch (ArithmeticException ex) {
|
44
46
|
// NaN / Infinite / -Infinite
|
45
47
|
defaultValue.setLong(pageBuilder, column);
|
46
48
|
return;
|
@@ -54,7 +56,8 @@ public class LongColumnSetter
|
|
54
56
|
long lv;
|
55
57
|
try {
|
56
58
|
lv = Long.parseLong(v);
|
57
|
-
}
|
59
|
+
}
|
60
|
+
catch (NumberFormatException e) {
|
58
61
|
defaultValue.setLong(pageBuilder, column);
|
59
62
|
return;
|
60
63
|
}
|
@@ -66,4 +69,10 @@ public class LongColumnSetter
|
|
66
69
|
{
|
67
70
|
pageBuilder.setDouble(column, v.getEpochSecond());
|
68
71
|
}
|
72
|
+
|
73
|
+
@Override
|
74
|
+
public void set(Value v)
|
75
|
+
{
|
76
|
+
pageBuilder.setJson(column, v);
|
77
|
+
}
|
69
78
|
}
|
@@ -6,6 +6,7 @@ import org.embulk.spi.Column;
|
|
6
6
|
import org.embulk.spi.time.Timestamp;
|
7
7
|
import org.embulk.spi.time.TimestampParser;
|
8
8
|
import org.embulk.spi.time.TimestampParseException;
|
9
|
+
import org.msgpack.value.Value;
|
9
10
|
|
10
11
|
public class SkipColumnSetter
|
11
12
|
extends AbstractDynamicColumnSetter
|
@@ -46,6 +47,10 @@ public class SkipColumnSetter
|
|
46
47
|
public void set(Timestamp v)
|
47
48
|
{ }
|
48
49
|
|
50
|
+
@Override
|
51
|
+
public void set(Value v)
|
52
|
+
{ }
|
53
|
+
|
49
54
|
@Override
|
50
55
|
public void setRubyObject(IRubyObject rubyObject)
|
51
56
|
{ }
|
@@ -4,6 +4,7 @@ import org.embulk.spi.PageBuilder;
|
|
4
4
|
import org.embulk.spi.Column;
|
5
5
|
import org.embulk.spi.time.Timestamp;
|
6
6
|
import org.embulk.spi.time.TimestampFormatter;
|
7
|
+
import org.msgpack.value.Value;
|
7
8
|
|
8
9
|
public class StringColumnSetter
|
9
10
|
extends AbstractDynamicColumnSetter
|
@@ -53,4 +54,10 @@ public class StringColumnSetter
|
|
53
54
|
{
|
54
55
|
pageBuilder.setString(column, timestampFormatter.format(v));
|
55
56
|
}
|
57
|
+
|
58
|
+
@Override
|
59
|
+
public void set(Value v)
|
60
|
+
{
|
61
|
+
pageBuilder.setJson(column, v);
|
62
|
+
}
|
56
63
|
}
|
@@ -5,6 +5,7 @@ import org.embulk.spi.Column;
|
|
5
5
|
import org.embulk.spi.time.Timestamp;
|
6
6
|
import org.embulk.spi.time.TimestampParser;
|
7
7
|
import org.embulk.spi.time.TimestampParseException;
|
8
|
+
import org.msgpack.value.Value;
|
8
9
|
|
9
10
|
public class TimestampColumnSetter
|
10
11
|
extends AbstractDynamicColumnSetter
|
@@ -51,7 +52,8 @@ public class TimestampColumnSetter
|
|
51
52
|
{
|
52
53
|
try {
|
53
54
|
pageBuilder.setTimestamp(column, timestampParser.parse(v));
|
54
|
-
}
|
55
|
+
}
|
56
|
+
catch (TimestampParseException e) {
|
55
57
|
defaultValue.setTimestamp(pageBuilder, column);
|
56
58
|
}
|
57
59
|
}
|
@@ -61,4 +63,10 @@ public class TimestampColumnSetter
|
|
61
63
|
{
|
62
64
|
pageBuilder.setTimestamp(column, v);
|
63
65
|
}
|
66
|
+
|
67
|
+
@Override
|
68
|
+
public void set(Value v)
|
69
|
+
{
|
70
|
+
pageBuilder.setJson(column, v);
|
71
|
+
}
|
64
72
|
}
|
@@ -92,6 +92,13 @@ public class MockFormatterPlugin implements FormatterPlugin
|
|
92
92
|
record.add(pageReader.getTimestamp(column));
|
93
93
|
}
|
94
94
|
}
|
95
|
+
|
96
|
+
public void jsonColumn(Column column)
|
97
|
+
{
|
98
|
+
if (!pageReader.isNull(column)) {
|
99
|
+
record.add(pageReader.getJson(column));
|
100
|
+
}
|
101
|
+
}
|
95
102
|
});
|
96
103
|
records.add(record);
|
97
104
|
}
|
@@ -470,7 +470,44 @@ Example
|
|
470
470
|
filters:
|
471
471
|
...
|
472
472
|
- type: rename
|
473
|
-
|
474
|
-
|
475
|
-
|
473
|
+
columns:
|
474
|
+
my_existing_column1: new_column1
|
475
|
+
my_existing_column2: new_column2
|
476
|
+
|
477
|
+
Local executor plugin
|
478
|
+
------------------
|
479
|
+
|
480
|
+
The ``local`` executor plugin runs tasks using local threads. This is the only built-in executor plugin.
|
481
|
+
|
482
|
+
Options
|
483
|
+
~~~~~~~~~~~~~~~~~~
|
484
|
+
|
485
|
+
+------------------+----------+----------------------------------------------------------------------+--------------------------------------+
|
486
|
+
| name | type | description | required? |
|
487
|
+
+==================+==========+======================================================================+======================================+
|
488
|
+
| max_threads | integer | Maximum number of threads to run concurrently. | 2x of available CPU cores by default |
|
489
|
+
+------------------+----------+----------------------------------------------------------------------+--------------------------------------+
|
490
|
+
| min_output_tasks | integer | Mimimum number of output tasks to enable page scattering. | 1x of available CPU cores by default |
|
491
|
+
+------------------+----------+----------------------------------------------------------------------+--------------------------------------+
|
492
|
+
|
493
|
+
|
494
|
+
The ``max_threads`` option controls maximum concurrency. Setting smaller number here is useful if too many threads make the destination or source storage overloaded. Setting larger number here is useful if CPU utilization is too low due to high latency.
|
495
|
+
|
496
|
+
The ``min_output_tasks`` option enables "page scattering". The feature is enabled if number of input tasks is less than ``min_output_tasks``. It uses multiple filter & output threads for each input task so that one input task can use multiple threads. Setting larger number here is useful if embulk doesn't use multi-threading with enough concurrency due to too few number of input tasks. Setting 1 here disables page scattering completely.
|
497
|
+
|
498
|
+
Example
|
499
|
+
~~~~~~~~~~~~~~~~~~
|
500
|
+
|
501
|
+
.. code-block:: yaml
|
502
|
+
|
503
|
+
exec:
|
504
|
+
max_threads: 8 # run at most 8 tasks concurrently
|
505
|
+
min_output_tasks: 1 # disable page scattering
|
506
|
+
in:
|
507
|
+
type: ...
|
508
|
+
...
|
509
|
+
out:
|
510
|
+
type: ...
|
511
|
+
...
|
512
|
+
|
476
513
|
|
data/embulk-docs/src/conf.py
CHANGED
@@ -53,9 +53,9 @@ copyright = u'2015, Embulk Project'
|
|
53
53
|
# built documents.
|
54
54
|
#
|
55
55
|
# The short X.Y version.
|
56
|
-
version = '0.
|
56
|
+
version = '0.8'
|
57
57
|
# The full version, including alpha/beta/rc tags.
|
58
|
-
release = '0.
|
58
|
+
release = '0.8'
|
59
59
|
|
60
60
|
# The language for content autogenerated by Sphinx. Refer to documentation
|
61
61
|
# for a list of supported languages.
|
data/embulk-docs/src/release.rst
CHANGED
@@ -0,0 +1,68 @@
|
|
1
|
+
Release 0.8.0
|
2
|
+
==================================
|
3
|
+
|
4
|
+
General Changes
|
5
|
+
------------------
|
6
|
+
|
7
|
+
* Added JSON type support.
|
8
|
+
|
9
|
+
* A column with ``json`` type can represent nested values such as maps or arrays. This is useful when both input and output support dynamically-typed values.
|
10
|
+
|
11
|
+
* **IMPORTANT**: If input plugin uses JSON type but output plugin is compiled with an old embulk (< 0.8.0), a bulk load transaction fails with a confusing error message. To avoid this issue, please run ``embulk migrate /path/to/embulk-plugin-directory`` to upgrade plugin code, and use the latest plugin. This problem doesn't happen if input plugin doesn't use json type.
|
12
|
+
|
13
|
+
* Filter plugins to process JSON types are not ready yet. Expected plugins are for example, flatten a json column into statically-typed columns with guess plugin, extracting a value from a json column using an expression (such as JSONPath) and set it to another column, or building a json column by copying values from other columns.
|
14
|
+
|
15
|
+
* Local executor plugin (the default executor) runs multiple tasks even if there is only 1 input task. This improves performance a lot especially if input is a single huge file.
|
16
|
+
|
17
|
+
* Its mechanism is that the executor creates 2, 3, 4, or more number of output tasks for each input task. Page chunks from a input task is scattered to output tasks. All of the tasks run in parallel using threads. This feature is called "page scattering".
|
18
|
+
|
19
|
+
* Added ``min_output_tasks`` option at ``exec:`` section. Default is 1x of available CPU cores. Page scattering is enabled if number of input tasks is less than this ``min_output_tasks`` option. Setting larger number here is useful if embulk doesn't use multi-threading with enough concurrency due to too few number of input tasks.
|
20
|
+
|
21
|
+
* Added ``max_threads`` option at ``exec:`` section. Default is 2x of availalbe CPU cores. This option controls maximum concurrency. Setting smaller number here is useful if too many threads make the destination or source storage overloaded. Setting larger number here is useful if CPU utilization is too low due to high latency.
|
22
|
+
|
23
|
+
* The results of output transaction will be deterministic. There're no randomness that depends on timing. However, task assignment changes if ``min_output_tasks`` changes. If you need deterministic results regardless of machines that may have different number of CPU cores, please add ``min_output_tasks`` option to ``exec:`` section. Setting 1 there will disable page scattering completely.
|
24
|
+
|
25
|
+
* Upraded JRuby version to 9.0.4.0.
|
26
|
+
|
27
|
+
* YAML configuration parser uses stricter rules when it converts type of a non-quoted strings.
|
28
|
+
|
29
|
+
* Strings starting with 0 such as 019 or 0819 will be a string instead of float.
|
30
|
+
|
31
|
+
* Strings looks like a timestamp such as 2015-01-01 will be a string instead of UNIX timestamp.
|
32
|
+
|
33
|
+
* On, Off, Yes, and No (case-insensitive) will be a string instead of boolean. Only true, True, false, False are recognized as a boolean.
|
34
|
+
|
35
|
+
Java Plugin API
|
36
|
+
------------------
|
37
|
+
|
38
|
+
* Added ``org.embulk.spi.json.JsonParser`` class to parse a String into an internal representation of a JSON value (org.msgpack.value.Value). Usage of this class is similar to TimestampParser.
|
39
|
+
|
40
|
+
* Added ``jsonColumn`` method to ``org.embulk.spi.ColumnVisitor``. At an input plugin or parser plugin, please implement this method using above JsonParser class and ``PageBuilder#stJson`` method. At an output plugin or formatter plugin, you can use ``org.msgpack.value.Value#toJson`` method to convert it to a JSON string.
|
41
|
+
|
42
|
+
* Upgraded gradle version to 2.10. ``embulk migrate`` upgrades gradle version of your plugins.
|
43
|
+
|
44
|
+
* ``embulk new`` and ``embulk migrate`` adds checkstyle configuration for Java plugins. ``./gradlew checkstyle`` checks code styles.
|
45
|
+
|
46
|
+
|
47
|
+
Run Plugin API
|
48
|
+
------------------
|
49
|
+
|
50
|
+
* ``PageBuilder#add`` accepts a nested object (Hash or Array) when the column type is json.
|
51
|
+
|
52
|
+
* Internally, the value is serialized using MessagePack and passed to Java components through JRuby interface.
|
53
|
+
|
54
|
+
* ``page.each`` method will give an nested object (Hash or Array) if the column type is json.
|
55
|
+
|
56
|
+
|
57
|
+
Built-in plugins
|
58
|
+
------------------
|
59
|
+
|
60
|
+
* ``parser-cvs`` and ``guess-csv`` plugins support JSON type. JSON type will be automatically guessed.
|
61
|
+
|
62
|
+
* ``formatter-csv`` supports JSON type. JSON column is serialized into a string using JSON format (``Value#toJson()``).
|
63
|
+
|
64
|
+
* ``formatter-stdout`` supports JSON type. JSON column is serialized into a string using string representation of Value (``Value#toString()``). This is slightly different from JSON but can represent some superset of json such as non-string keys of objects.
|
65
|
+
|
66
|
+
Release Date
|
67
|
+
------------------
|
68
|
+
2016-01-13
|