embulk 0.6.19 → 0.6.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/build.gradle +1 -1
- data/embulk-core/src/main/java/org/embulk/command/Runner.java +12 -3
- data/embulk-core/src/main/java/org/embulk/config/ConfigLoader.java +15 -5
- data/embulk-core/src/main/java/org/embulk/exec/PooledBufferAllocator.java +10 -6
- data/embulk-core/src/main/java/org/embulk/spi/unit/ByteSize.java +8 -0
- data/embulk-docs/src/built-in.rst +1 -1
- data/embulk-docs/src/index.rst +2 -0
- data/embulk-docs/src/release.rst +1 -0
- data/embulk-docs/src/release/release-0.6.20.rst +19 -0
- data/lib/embulk/command/embulk_run.rb +21 -0
- data/lib/embulk/data/new/java/encoder.java.erb +3 -3
- data/lib/embulk/version.rb +1 -1
- metadata +5 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c20e3bb56bb0cfa868416afdc65bf0bc199600e1
|
4
|
+
data.tar.gz: 3821ab6ab119ac969f13b275a6d14f9b6dbc68c1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 98746e5f4bdd45986aadafcd3a7534cb2ae075c4eb69a7ec69634a52b34ed033da4599ab71ef1ca260d35b20e4133bd5c88769b024d2fef34f37b3d10cc416fe
|
7
|
+
data.tar.gz: 989a5fb745b0893d1c79dd799065fc35d866986e71a3998686c40bd2eeb97b44f3592900f40e04c835c4165c6615d249b7b6f3ba1e0379f187190f6d28055eb9
|
data/build.gradle
CHANGED
@@ -53,6 +53,9 @@ public class Runner
|
|
53
53
|
|
54
54
|
private boolean useGlobalRubyRuntime;
|
55
55
|
public boolean getUseGlobalRubyRuntime() { return useGlobalRubyRuntime; }
|
56
|
+
|
57
|
+
private Map<String, String> systemProperty;
|
58
|
+
public Map<String, String> getSystemProperty() { return systemProperty; }
|
56
59
|
}
|
57
60
|
|
58
61
|
private final Options options;
|
@@ -64,15 +67,21 @@ public class Runner
|
|
64
67
|
{
|
65
68
|
ModelManager bootstrapModelManager = new ModelManager(null, new ObjectMapper());
|
66
69
|
this.options = bootstrapModelManager.readObject(Options.class, optionJson);
|
67
|
-
|
68
|
-
|
70
|
+
|
71
|
+
ConfigLoader configLoader = new ConfigLoader(bootstrapModelManager);
|
72
|
+
ConfigSource systemConfig = configLoader.fromPropertiesYamlLiteral(System.getProperties(), "embulk.");
|
73
|
+
mergeOptionsToSystemConfig(options, configLoader, systemConfig);
|
74
|
+
|
75
|
+
this.systemConfig = systemConfig;
|
69
76
|
this.service = new EmbulkService(systemConfig);
|
70
77
|
this.injector = service.initialize();
|
71
78
|
}
|
72
79
|
|
73
80
|
@SuppressWarnings("unchecked")
|
74
|
-
private void mergeOptionsToSystemConfig(Options options, ConfigSource systemConfig)
|
81
|
+
private static void mergeOptionsToSystemConfig(Options options, ConfigLoader configLoader, ConfigSource systemConfig)
|
75
82
|
{
|
83
|
+
systemConfig.merge(configLoader.fromPropertiesYamlLiteral(options.getSystemProperty(), ""));
|
84
|
+
|
76
85
|
String logLevel = options.getLogLevel();
|
77
86
|
if (logLevel != null) {
|
78
87
|
// used by LoggerProvider
|
@@ -4,7 +4,9 @@ import java.io.File;
|
|
4
4
|
import java.io.FileInputStream;
|
5
5
|
import java.io.InputStream;
|
6
6
|
import java.io.IOException;
|
7
|
+
import java.util.Map;
|
7
8
|
import java.util.Properties;
|
9
|
+
import com.google.common.collect.ImmutableMap;
|
8
10
|
import com.google.inject.Inject;
|
9
11
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
10
12
|
import com.fasterxml.jackson.core.JsonParser;
|
@@ -67,17 +69,25 @@ public class ConfigLoader
|
|
67
69
|
}
|
68
70
|
|
69
71
|
public ConfigSource fromPropertiesYamlLiteral(Properties props, String keyPrefix)
|
72
|
+
{
|
73
|
+
ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
|
74
|
+
for (String propName : props.stringPropertyNames()) {
|
75
|
+
builder.put(propName, props.getProperty(propName));
|
76
|
+
}
|
77
|
+
return fromPropertiesYamlLiteral(builder.build(), keyPrefix);
|
78
|
+
}
|
79
|
+
|
80
|
+
public ConfigSource fromPropertiesYamlLiteral(Map<String, String> props, String keyPrefix)
|
70
81
|
{
|
71
82
|
ObjectNode source = new ObjectNode(JsonNodeFactory.instance);
|
72
83
|
DataSource ds = new DataSourceImpl(model, source);
|
73
84
|
Yaml yaml = new Yaml();
|
74
|
-
for (String
|
75
|
-
if (!
|
85
|
+
for (Map.Entry<String, String> pair : props.entrySet()) {
|
86
|
+
if (!pair.getKey().startsWith(keyPrefix)) {
|
76
87
|
continue;
|
77
88
|
}
|
78
|
-
String keyName =
|
79
|
-
|
80
|
-
Object parsedValue = yaml.load(yamlValue); // TODO exception handling
|
89
|
+
String keyName = pair.getKey().substring(keyPrefix.length());
|
90
|
+
Object parsedValue = yaml.load(pair.getValue()); // TODO exception handling
|
81
91
|
JsonNode node = objectToJson(parsedValue);
|
82
92
|
|
83
93
|
// handle "." as a map acccessor. for example:
|
@@ -5,29 +5,33 @@ import io.netty.buffer.ByteBuf;
|
|
5
5
|
import io.netty.util.ResourceLeakDetector;
|
6
6
|
import org.embulk.spi.Buffer;
|
7
7
|
import org.embulk.spi.BufferAllocator;
|
8
|
+
import com.google.inject.Inject;
|
9
|
+
import org.embulk.config.ConfigSource;
|
10
|
+
import org.embulk.spi.unit.ByteSize;
|
8
11
|
|
9
12
|
public class PooledBufferAllocator
|
10
13
|
implements BufferAllocator
|
11
14
|
{
|
12
|
-
private static final int
|
13
|
-
private static final int MINIMUM_BUFFER_SIZE = 8*1024;
|
15
|
+
private static final int DEFAULT_PAGE_SIZE = 32*1024;
|
14
16
|
|
15
17
|
private final PooledByteBufAllocator nettyBuffer;
|
18
|
+
private final int pageSize;
|
16
19
|
|
17
|
-
|
20
|
+
@Inject
|
21
|
+
public PooledBufferAllocator(@ForSystemConfig ConfigSource systemConfig)
|
18
22
|
{
|
19
|
-
// TODO configure parameters
|
20
23
|
this.nettyBuffer = new PooledByteBufAllocator(false);
|
24
|
+
this.pageSize = systemConfig.get(ByteSize.class, "page_size", new ByteSize(DEFAULT_PAGE_SIZE)).getBytesInt();
|
21
25
|
}
|
22
26
|
|
23
27
|
public Buffer allocate()
|
24
28
|
{
|
25
|
-
return allocate(
|
29
|
+
return allocate(pageSize);
|
26
30
|
}
|
27
31
|
|
28
32
|
public Buffer allocate(int minimumCapacity)
|
29
33
|
{
|
30
|
-
int size =
|
34
|
+
int size = this.pageSize;
|
31
35
|
while (size < minimumCapacity) {
|
32
36
|
size *= 2;
|
33
37
|
}
|
@@ -40,6 +40,14 @@ public class ByteSize
|
|
40
40
|
return bytes;
|
41
41
|
}
|
42
42
|
|
43
|
+
public int getBytesInt()
|
44
|
+
{
|
45
|
+
if (bytes > Integer.MAX_VALUE) {
|
46
|
+
throw new RuntimeException("Byte size is too large (must be smaller than 2GB)");
|
47
|
+
}
|
48
|
+
return (int) bytes;
|
49
|
+
}
|
50
|
+
|
43
51
|
public long roundTo(Unit unit)
|
44
52
|
{
|
45
53
|
return (long) Math.floor(getValue(unit) + 0.5);
|
data/embulk-docs/src/index.rst
CHANGED
data/embulk-docs/src/release.rst
CHANGED
@@ -0,0 +1,19 @@
|
|
1
|
+
Release 0.6.20
|
2
|
+
==================================
|
3
|
+
|
4
|
+
Command line interface
|
5
|
+
------------------
|
6
|
+
|
7
|
+
* Added ``-X key=value`` argument to set system config. This argument is intended to overwrite performance parameters such as number of threads (``max_threads``) or page buffer size (``page_size``).
|
8
|
+
|
9
|
+
|
10
|
+
General Changes
|
11
|
+
------------------
|
12
|
+
|
13
|
+
* Change default size of page buffer from 8KB to 32KB.
|
14
|
+
* Size of a page buffer is configurable by system config (@sonots++).
|
15
|
+
|
16
|
+
|
17
|
+
Release Date
|
18
|
+
------------------
|
19
|
+
2015-08-03
|
@@ -49,6 +49,7 @@ module Embulk
|
|
49
49
|
# use the global ruby runtime (the jruby Runtime running this embulk_run.rb script) for all
|
50
50
|
# ScriptingContainer injected by the org.embulk.command.Runner.
|
51
51
|
useGlobalRubyRuntime: true,
|
52
|
+
systemProperty: {},
|
52
53
|
}
|
53
54
|
|
54
55
|
op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory') do |path|
|
@@ -86,6 +87,11 @@ module Embulk
|
|
86
87
|
op.on('-r', '--resume-state PATH', 'Path to a file to write or read resume state') do |path|
|
87
88
|
options[:resumeStatePath] = path
|
88
89
|
end
|
90
|
+
op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
|
91
|
+
k, v = kv.split('=', 2)
|
92
|
+
v ||= "true"
|
93
|
+
options[:systemProperty][k] = v
|
94
|
+
end
|
89
95
|
args = 1..1
|
90
96
|
|
91
97
|
when :cleanup
|
@@ -105,6 +111,11 @@ module Embulk
|
|
105
111
|
op.on('-r', '--resume-state PATH', 'Path to a file to write or read resume state') do |path|
|
106
112
|
options[:resumeStatePath] = path
|
107
113
|
end
|
114
|
+
op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
|
115
|
+
k, v = kv.split('=', 2)
|
116
|
+
v ||= "true"
|
117
|
+
options[:systemProperty][k] = v
|
118
|
+
end
|
108
119
|
args = 1..1
|
109
120
|
|
110
121
|
when :preview
|
@@ -124,6 +135,11 @@ module Embulk
|
|
124
135
|
op.on('-G', '--vertical', "Use vertical output format", TrueClass) do |b|
|
125
136
|
options[:previewOutputFormat] = "vertical"
|
126
137
|
end
|
138
|
+
op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
|
139
|
+
k, v = kv.split('=', 2)
|
140
|
+
v ||= "true"
|
141
|
+
options[:systemProperty][k] = v
|
142
|
+
end
|
127
143
|
args = 1..1
|
128
144
|
|
129
145
|
when :guess
|
@@ -146,6 +162,11 @@ module Embulk
|
|
146
162
|
op.on('-g', '--guess NAMES', "Comma-separated list of guess plugin names") do |names|
|
147
163
|
(options[:guessPlugins] ||= []).concat names.split(",")
|
148
164
|
end
|
165
|
+
op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
|
166
|
+
k, v = kv.split('=', 2)
|
167
|
+
v ||= "true"
|
168
|
+
options[:systemProperty][k] = v
|
169
|
+
end
|
149
170
|
args = 1..1
|
150
171
|
|
151
172
|
when :new
|
@@ -64,17 +64,17 @@ public class <%= java_class_name %>
|
|
64
64
|
// public OutputStream openNext() throws IOException
|
65
65
|
// {
|
66
66
|
// output.nextFile();
|
67
|
-
// return newEncoderOutputStream(output);
|
67
|
+
// return newEncoderOutputStream(task, output);
|
68
68
|
// }
|
69
69
|
//
|
70
70
|
// public void finish() throws IOException
|
71
71
|
// {
|
72
|
-
//
|
72
|
+
// output.finish();
|
73
73
|
// }
|
74
74
|
//
|
75
75
|
// public void close() throws IOException
|
76
76
|
// {
|
77
|
-
//
|
77
|
+
// output.close();
|
78
78
|
// }
|
79
79
|
//});
|
80
80
|
}
|
data/lib/embulk/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.20
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-08-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -339,6 +339,7 @@ files:
|
|
339
339
|
- embulk-docs/src/release/release-0.6.18.rst
|
340
340
|
- embulk-docs/src/release/release-0.6.19.rst
|
341
341
|
- embulk-docs/src/release/release-0.6.2.rst
|
342
|
+
- embulk-docs/src/release/release-0.6.20.rst
|
342
343
|
- embulk-docs/src/release/release-0.6.3.rst
|
343
344
|
- embulk-docs/src/release/release-0.6.4.rst
|
344
345
|
- embulk-docs/src/release/release-0.6.5.rst
|
@@ -454,8 +455,8 @@ files:
|
|
454
455
|
- classpath/bval-jsr303-0.5.jar
|
455
456
|
- classpath/commons-beanutils-core-1.8.3.jar
|
456
457
|
- classpath/commons-lang3-3.1.jar
|
457
|
-
- classpath/embulk-core-0.6.
|
458
|
-
- classpath/embulk-standards-0.6.
|
458
|
+
- classpath/embulk-core-0.6.20.jar
|
459
|
+
- classpath/embulk-standards-0.6.20.jar
|
459
460
|
- classpath/guava-18.0.jar
|
460
461
|
- classpath/guice-4.0.jar
|
461
462
|
- classpath/guice-multibindings-4.0.jar
|