embulk 0.6.19 → 0.6.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0525d84746ae3f0cbd2839f3b6447063fc8f72ed
4
- data.tar.gz: ea6a6a7c3a2a74b05466b5c1170239a6f32d0fa3
3
+ metadata.gz: c20e3bb56bb0cfa868416afdc65bf0bc199600e1
4
+ data.tar.gz: 3821ab6ab119ac969f13b275a6d14f9b6dbc68c1
5
5
  SHA512:
6
- metadata.gz: b5972147aab6f9e81cbdd13eb536ffbbdee7dfcff9ee32122b90a76e3e41756fd144f709f6bbbd22d603a57c613e86c671f135903528f0c94beccea70066d0a1
7
- data.tar.gz: 10383e3a6a6362decade2631ad9b365455be007e9600c97e4564f3a7df2a9556f7356985ea45975a37bbc2c612a3255c79a1ae11657084fa78a643b345625722
6
+ metadata.gz: 98746e5f4bdd45986aadafcd3a7534cb2ae075c4eb69a7ec69634a52b34ed033da4599ab71ef1ca260d35b20e4133bd5c88769b024d2fef34f37b3d10cc416fe
7
+ data.tar.gz: 989a5fb745b0893d1c79dd799065fc35d866986e71a3998686c40bd2eeb97b44f3592900f40e04c835c4165c6615d249b7b6f3ba1e0379f187190f6d28055eb9
@@ -11,7 +11,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
11
11
 
12
12
  allprojects {
13
13
  group = 'org.embulk'
14
- version = '0.6.19'
14
+ version = '0.6.20'
15
15
 
16
16
  ext {
17
17
  jrubyVersion = '1.7.19'
@@ -53,6 +53,9 @@ public class Runner
53
53
 
54
54
  private boolean useGlobalRubyRuntime;
55
55
  public boolean getUseGlobalRubyRuntime() { return useGlobalRubyRuntime; }
56
+
57
+ private Map<String, String> systemProperty;
58
+ public Map<String, String> getSystemProperty() { return systemProperty; }
56
59
  }
57
60
 
58
61
  private final Options options;
@@ -64,15 +67,21 @@ public class Runner
64
67
  {
65
68
  ModelManager bootstrapModelManager = new ModelManager(null, new ObjectMapper());
66
69
  this.options = bootstrapModelManager.readObject(Options.class, optionJson);
67
- this.systemConfig = new ConfigLoader(bootstrapModelManager).fromPropertiesYamlLiteral(System.getProperties(), "embulk.");
68
- mergeOptionsToSystemConfig(options, systemConfig);
70
+
71
+ ConfigLoader configLoader = new ConfigLoader(bootstrapModelManager);
72
+ ConfigSource systemConfig = configLoader.fromPropertiesYamlLiteral(System.getProperties(), "embulk.");
73
+ mergeOptionsToSystemConfig(options, configLoader, systemConfig);
74
+
75
+ this.systemConfig = systemConfig;
69
76
  this.service = new EmbulkService(systemConfig);
70
77
  this.injector = service.initialize();
71
78
  }
72
79
 
73
80
  @SuppressWarnings("unchecked")
74
- private void mergeOptionsToSystemConfig(Options options, ConfigSource systemConfig)
81
+ private static void mergeOptionsToSystemConfig(Options options, ConfigLoader configLoader, ConfigSource systemConfig)
75
82
  {
83
+ systemConfig.merge(configLoader.fromPropertiesYamlLiteral(options.getSystemProperty(), ""));
84
+
76
85
  String logLevel = options.getLogLevel();
77
86
  if (logLevel != null) {
78
87
  // used by LoggerProvider
@@ -4,7 +4,9 @@ import java.io.File;
4
4
  import java.io.FileInputStream;
5
5
  import java.io.InputStream;
6
6
  import java.io.IOException;
7
+ import java.util.Map;
7
8
  import java.util.Properties;
9
+ import com.google.common.collect.ImmutableMap;
8
10
  import com.google.inject.Inject;
9
11
  import com.fasterxml.jackson.databind.ObjectMapper;
10
12
  import com.fasterxml.jackson.core.JsonParser;
@@ -67,17 +69,25 @@ public class ConfigLoader
67
69
  }
68
70
 
69
71
  public ConfigSource fromPropertiesYamlLiteral(Properties props, String keyPrefix)
72
+ {
73
+ ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
74
+ for (String propName : props.stringPropertyNames()) {
75
+ builder.put(propName, props.getProperty(propName));
76
+ }
77
+ return fromPropertiesYamlLiteral(builder.build(), keyPrefix);
78
+ }
79
+
80
+ public ConfigSource fromPropertiesYamlLiteral(Map<String, String> props, String keyPrefix)
70
81
  {
71
82
  ObjectNode source = new ObjectNode(JsonNodeFactory.instance);
72
83
  DataSource ds = new DataSourceImpl(model, source);
73
84
  Yaml yaml = new Yaml();
74
- for (String propName : props.stringPropertyNames()) {
75
- if (!propName.startsWith(keyPrefix)) {
85
+ for (Map.Entry<String, String> pair : props.entrySet()) {
86
+ if (!pair.getKey().startsWith(keyPrefix)) {
76
87
  continue;
77
88
  }
78
- String keyName = propName.substring(keyPrefix.length());
79
- String yamlValue = props.getProperty(propName);
80
- Object parsedValue = yaml.load(yamlValue); // TODO exception handling
89
+ String keyName = pair.getKey().substring(keyPrefix.length());
90
+ Object parsedValue = yaml.load(pair.getValue()); // TODO exception handling
81
91
  JsonNode node = objectToJson(parsedValue);
82
92
 
83
93
  // handle "." as a map acccessor. for example:
@@ -5,29 +5,33 @@ import io.netty.buffer.ByteBuf;
5
5
  import io.netty.util.ResourceLeakDetector;
6
6
  import org.embulk.spi.Buffer;
7
7
  import org.embulk.spi.BufferAllocator;
8
+ import com.google.inject.Inject;
9
+ import org.embulk.config.ConfigSource;
10
+ import org.embulk.spi.unit.ByteSize;
8
11
 
9
12
  public class PooledBufferAllocator
10
13
  implements BufferAllocator
11
14
  {
12
- private static final int DEFAULT_BUFFER_SIZE = 32*1024;
13
- private static final int MINIMUM_BUFFER_SIZE = 8*1024;
15
+ private static final int DEFAULT_PAGE_SIZE = 32*1024;
14
16
 
15
17
  private final PooledByteBufAllocator nettyBuffer;
18
+ private final int pageSize;
16
19
 
17
- public PooledBufferAllocator()
20
+ @Inject
21
+ public PooledBufferAllocator(@ForSystemConfig ConfigSource systemConfig)
18
22
  {
19
- // TODO configure parameters
20
23
  this.nettyBuffer = new PooledByteBufAllocator(false);
24
+ this.pageSize = systemConfig.get(ByteSize.class, "page_size", new ByteSize(DEFAULT_PAGE_SIZE)).getBytesInt();
21
25
  }
22
26
 
23
27
  public Buffer allocate()
24
28
  {
25
- return allocate(DEFAULT_BUFFER_SIZE);
29
+ return allocate(pageSize);
26
30
  }
27
31
 
28
32
  public Buffer allocate(int minimumCapacity)
29
33
  {
30
- int size = MINIMUM_BUFFER_SIZE;
34
+ int size = this.pageSize;
31
35
  while (size < minimumCapacity) {
32
36
  size *= 2;
33
37
  }
@@ -40,6 +40,14 @@ public class ByteSize
40
40
  return bytes;
41
41
  }
42
42
 
43
+ public int getBytesInt()
44
+ {
45
+ if (bytes > Integer.MAX_VALUE) {
46
+ throw new RuntimeException("Byte size is too large (must be smaller than 2GB)");
47
+ }
48
+ return (int) bytes;
49
+ }
50
+
43
51
  public long roundTo(Unit unit)
44
52
  {
45
53
  return (long) Math.floor(getValue(unit) + 0.5);
@@ -344,7 +344,7 @@ Example
344
344
  out:
345
345
  ...
346
346
  formatter:
347
- - type: csv
347
+ type: csv
348
348
  delimiter: '\t'
349
349
  newline: CRLF
350
350
  newline_in_field: LF
@@ -59,6 +59,8 @@ Documents
59
59
 
60
60
  * `Filter plugins <http://www.embulk.org/plugins/#filter>`_
61
61
 
62
+ * `Executor plugins <http://www.embulk.org/plugins/#executor>`_
63
+
62
64
  .. toctree::
63
65
  :maxdepth: 2
64
66
 
@@ -4,6 +4,7 @@ Release Notes
4
4
  .. toctree::
5
5
  :maxdepth: 1
6
6
 
7
+ release/release-0.6.20
7
8
  release/release-0.6.19
8
9
  release/release-0.6.18
9
10
  release/release-0.6.17
@@ -0,0 +1,19 @@
1
+ Release 0.6.20
2
+ ==================================
3
+
4
+ Command line interface
5
+ ------------------
6
+
7
+ * Added ``-X key=value`` argument to set system config. This argument is intended to overwrite performance parameters such as number of threads (``max_threads``) or page buffer size (``page_size``).
8
+
9
+
10
+ General Changes
11
+ ------------------
12
+
13
+ * Change default size of page buffer from 8KB to 32KB.
14
+ * Size of a page buffer is configurable by system config (@sonots++).
15
+
16
+
17
+ Release Date
18
+ ------------------
19
+ 2015-08-03
@@ -49,6 +49,7 @@ module Embulk
49
49
  # use the global ruby runtime (the jruby Runtime running this embulk_run.rb script) for all
50
50
  # ScriptingContainer injected by the org.embulk.command.Runner.
51
51
  useGlobalRubyRuntime: true,
52
+ systemProperty: {},
52
53
  }
53
54
 
54
55
  op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory') do |path|
@@ -86,6 +87,11 @@ module Embulk
86
87
  op.on('-r', '--resume-state PATH', 'Path to a file to write or read resume state') do |path|
87
88
  options[:resumeStatePath] = path
88
89
  end
90
+ op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
91
+ k, v = kv.split('=', 2)
92
+ v ||= "true"
93
+ options[:systemProperty][k] = v
94
+ end
89
95
  args = 1..1
90
96
 
91
97
  when :cleanup
@@ -105,6 +111,11 @@ module Embulk
105
111
  op.on('-r', '--resume-state PATH', 'Path to a file to write or read resume state') do |path|
106
112
  options[:resumeStatePath] = path
107
113
  end
114
+ op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
115
+ k, v = kv.split('=', 2)
116
+ v ||= "true"
117
+ options[:systemProperty][k] = v
118
+ end
108
119
  args = 1..1
109
120
 
110
121
  when :preview
@@ -124,6 +135,11 @@ module Embulk
124
135
  op.on('-G', '--vertical', "Use vertical output format", TrueClass) do |b|
125
136
  options[:previewOutputFormat] = "vertical"
126
137
  end
138
+ op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
139
+ k, v = kv.split('=', 2)
140
+ v ||= "true"
141
+ options[:systemProperty][k] = v
142
+ end
127
143
  args = 1..1
128
144
 
129
145
  when :guess
@@ -146,6 +162,11 @@ module Embulk
146
162
  op.on('-g', '--guess NAMES', "Comma-separated list of guess plugin names") do |names|
147
163
  (options[:guessPlugins] ||= []).concat names.split(",")
148
164
  end
165
+ op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
166
+ k, v = kv.split('=', 2)
167
+ v ||= "true"
168
+ options[:systemProperty][k] = v
169
+ end
149
170
  args = 1..1
150
171
 
151
172
  when :new
@@ -64,17 +64,17 @@ public class <%= java_class_name %>
64
64
  // public OutputStream openNext() throws IOException
65
65
  // {
66
66
  // output.nextFile();
67
- // return newEncoderOutputStream(output);
67
+ // return newEncoderOutputStream(task, output);
68
68
  // }
69
69
  //
70
70
  // public void finish() throws IOException
71
71
  // {
72
- // fileOutput.finish();
72
+ // output.finish();
73
73
  // }
74
74
  //
75
75
  // public void close() throws IOException
76
76
  // {
77
- // fileOutput.close();
77
+ // output.close();
78
78
  // }
79
79
  //});
80
80
  }
@@ -1,3 +1,3 @@
1
1
  module Embulk
2
- VERSION = '0.6.19'
2
+ VERSION = '0.6.20'
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.19
4
+ version: 0.6.20
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-07-24 00:00:00.000000000 Z
11
+ date: 2015-08-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -339,6 +339,7 @@ files:
339
339
  - embulk-docs/src/release/release-0.6.18.rst
340
340
  - embulk-docs/src/release/release-0.6.19.rst
341
341
  - embulk-docs/src/release/release-0.6.2.rst
342
+ - embulk-docs/src/release/release-0.6.20.rst
342
343
  - embulk-docs/src/release/release-0.6.3.rst
343
344
  - embulk-docs/src/release/release-0.6.4.rst
344
345
  - embulk-docs/src/release/release-0.6.5.rst
@@ -454,8 +455,8 @@ files:
454
455
  - classpath/bval-jsr303-0.5.jar
455
456
  - classpath/commons-beanutils-core-1.8.3.jar
456
457
  - classpath/commons-lang3-3.1.jar
457
- - classpath/embulk-core-0.6.19.jar
458
- - classpath/embulk-standards-0.6.19.jar
458
+ - classpath/embulk-core-0.6.20.jar
459
+ - classpath/embulk-standards-0.6.20.jar
459
460
  - classpath/guava-18.0.jar
460
461
  - classpath/guice-4.0.jar
461
462
  - classpath/guice-multibindings-4.0.jar