embulk 0.6.19 → 0.6.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0525d84746ae3f0cbd2839f3b6447063fc8f72ed
-  data.tar.gz: ea6a6a7c3a2a74b05466b5c1170239a6f32d0fa3
+  metadata.gz: c20e3bb56bb0cfa868416afdc65bf0bc199600e1
+  data.tar.gz: 3821ab6ab119ac969f13b275a6d14f9b6dbc68c1
 SHA512:
-  metadata.gz: b5972147aab6f9e81cbdd13eb536ffbbdee7dfcff9ee32122b90a76e3e41756fd144f709f6bbbd22d603a57c613e86c671f135903528f0c94beccea70066d0a1
-  data.tar.gz: 10383e3a6a6362decade2631ad9b365455be007e9600c97e4564f3a7df2a9556f7356985ea45975a37bbc2c612a3255c79a1ae11657084fa78a643b345625722
+  metadata.gz: 98746e5f4bdd45986aadafcd3a7534cb2ae075c4eb69a7ec69634a52b34ed033da4599ab71ef1ca260d35b20e4133bd5c88769b024d2fef34f37b3d10cc416fe
+  data.tar.gz: 989a5fb745b0893d1c79dd799065fc35d866986e71a3998686c40bd2eeb97b44f3592900f40e04c835c4165c6615d249b7b6f3ba1e0379f187190f6d28055eb9

data/build.gradle

@@ -11,7 +11,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.6.19'
+    version = '0.6.20'
 
     ext {
         jrubyVersion = '1.7.19'

data/embulk-core/src/main/java/org/embulk/command/Runner.java

@@ -53,6 +53,9 @@ public class Runner
 
         private boolean useGlobalRubyRuntime;
         public boolean getUseGlobalRubyRuntime() { return useGlobalRubyRuntime; }
+
+        private Map<String, String> systemProperty;
+        public Map<String, String> getSystemProperty() { return systemProperty; }
     }
 
     private final Options options;
@@ -64,15 +67,21 @@ public class Runner
     {
         ModelManager bootstrapModelManager = new ModelManager(null, new ObjectMapper());
         this.options = bootstrapModelManager.readObject(Options.class, optionJson);
-        this.systemConfig = new ConfigLoader(bootstrapModelManager).fromPropertiesYamlLiteral(System.getProperties(), "embulk.");
-        mergeOptionsToSystemConfig(options, systemConfig);
+
+        ConfigLoader configLoader = new ConfigLoader(bootstrapModelManager);
+        ConfigSource systemConfig = configLoader.fromPropertiesYamlLiteral(System.getProperties(), "embulk.");
+        mergeOptionsToSystemConfig(options, configLoader, systemConfig);
+
+        this.systemConfig = systemConfig;
         this.service = new EmbulkService(systemConfig);
         this.injector = service.initialize();
     }
 
     @SuppressWarnings("unchecked")
-    private void mergeOptionsToSystemConfig(Options options, ConfigSource systemConfig)
+    private static void mergeOptionsToSystemConfig(Options options, ConfigLoader configLoader, ConfigSource systemConfig)
     {
+        systemConfig.merge(configLoader.fromPropertiesYamlLiteral(options.getSystemProperty(), ""));
+
         String logLevel = options.getLogLevel();
         if (logLevel != null) {
             // used by LoggerProvider

data/embulk-core/src/main/java/org/embulk/config/ConfigLoader.java

@@ -4,7 +4,9 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStream;
 import java.io.IOException;
+import java.util.Map;
 import java.util.Properties;
+import com.google.common.collect.ImmutableMap;
 import com.google.inject.Inject;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.core.JsonParser;
@@ -67,17 +69,25 @@ public class ConfigLoader
     }
 
     public ConfigSource fromPropertiesYamlLiteral(Properties props, String keyPrefix)
+    {
+        ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+        for (String propName : props.stringPropertyNames()) {
+            builder.put(propName, props.getProperty(propName));
+        }
+        return fromPropertiesYamlLiteral(builder.build(), keyPrefix);
+    }
+
+    public ConfigSource fromPropertiesYamlLiteral(Map<String, String> props, String keyPrefix)
     {
         ObjectNode source = new ObjectNode(JsonNodeFactory.instance);
         DataSource ds = new DataSourceImpl(model, source);
         Yaml yaml = new Yaml();
-        for (String propName : props.stringPropertyNames()) {
-            if (!propName.startsWith(keyPrefix)) {
+        for (Map.Entry<String, String> pair : props.entrySet()) {
+            if (!pair.getKey().startsWith(keyPrefix)) {
                 continue;
             }
-            String keyName = propName.substring(keyPrefix.length());
-            String yamlValue = props.getProperty(propName);
-            Object parsedValue = yaml.load(yamlValue);  // TODO exception handling
+            String keyName = pair.getKey().substring(keyPrefix.length());
+            Object parsedValue = yaml.load(pair.getValue());  // TODO exception handling
             JsonNode node = objectToJson(parsedValue);
 
             // handle "." as a map acccessor. for example:

data/embulk-core/src/main/java/org/embulk/exec/PooledBufferAllocator.java

@@ -5,29 +5,33 @@ import io.netty.buffer.ByteBuf;
 import io.netty.util.ResourceLeakDetector;
 import org.embulk.spi.Buffer;
 import org.embulk.spi.BufferAllocator;
+import com.google.inject.Inject;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.unit.ByteSize;
 
 public class PooledBufferAllocator
         implements BufferAllocator
 {
-    private static final int DEFAULT_BUFFER_SIZE = 32*1024;
-    private static final int MINIMUM_BUFFER_SIZE = 8*1024;
+    private static final int DEFAULT_PAGE_SIZE = 32*1024;
 
     private final PooledByteBufAllocator nettyBuffer;
+    private final int pageSize;
 
-    public PooledBufferAllocator()
+    @Inject
+    public PooledBufferAllocator(@ForSystemConfig ConfigSource systemConfig)
     {
-        // TODO configure parameters
         this.nettyBuffer = new PooledByteBufAllocator(false);
+        this.pageSize = systemConfig.get(ByteSize.class, "page_size", new ByteSize(DEFAULT_PAGE_SIZE)).getBytesInt();
     }
 
     public Buffer allocate()
     {
-        return allocate(DEFAULT_BUFFER_SIZE);
+        return allocate(pageSize);
     }
 
     public Buffer allocate(int minimumCapacity)
     {
-        int size = MINIMUM_BUFFER_SIZE;
+        int size = this.pageSize;
         while (size < minimumCapacity) {
             size *= 2;
         }

data/embulk-core/src/main/java/org/embulk/spi/unit/ByteSize.java

@@ -40,6 +40,14 @@ public class ByteSize
         return bytes;
     }
 
+    public int getBytesInt()
+    {
+        if (bytes > Integer.MAX_VALUE) {
+            throw new RuntimeException("Byte size is too large (must be smaller than 2GB)");
+        }
+        return (int) bytes;
+    }
+
     public long roundTo(Unit unit)
     {
         return (long) Math.floor(getValue(unit) + 0.5);

data/embulk-docs/src/built-in.rst

@@ -344,7 +344,7 @@ Example
     out:
       ...
       formatter:
-      - type: csv
+        type: csv
         delimiter: '\t'
         newline: CRLF
         newline_in_field: LF

data/embulk-docs/src/index.rst

@@ -59,6 +59,8 @@ Documents
 
   * `Filter plugins <http://www.embulk.org/plugins/#filter>`_
 
+  * `Executor plugins <http://www.embulk.org/plugins/#executor>`_
+
 .. toctree::
    :maxdepth: 2
 

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.6.20
     release/release-0.6.19
     release/release-0.6.18
     release/release-0.6.17

data/embulk-docs/src/release/release-0.6.20.rst

@@ -0,0 +1,19 @@
+Release 0.6.20
+==================================
+
+Command line interface
+------------------
+
+* Added ``-X key=value`` argument to set system config. This argument is intended to overwrite performance parameters such as number of threads (``max_threads``) or page buffer size (``page_size``).
+
+
+General Changes
+------------------
+
+* Change default size of page buffer from 8KB to 32KB.
+* Size of a page buffer is configurable by system config (@sonots++).
+
+
+Release Date
+------------------
+2015-08-03

data/lib/embulk/command/embulk_run.rb

@@ -49,6 +49,7 @@ module Embulk
       # use the global ruby runtime (the jruby Runtime running this embulk_run.rb script) for all
       # ScriptingContainer injected by the org.embulk.command.Runner.
       useGlobalRubyRuntime: true,
+      systemProperty: {},
     }
 
     op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory') do |path|
@@ -86,6 +87,11 @@ module Embulk
       op.on('-r', '--resume-state PATH', 'Path to a file to write or read resume state') do |path|
         options[:resumeStatePath] = path
       end
+      op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
+        k, v = kv.split('=', 2)
+        v ||= "true"
+        options[:systemProperty][k] = v
+      end
       args = 1..1
 
     when :cleanup
@@ -105,6 +111,11 @@ module Embulk
       op.on('-r', '--resume-state PATH', 'Path to a file to write or read resume state') do |path|
         options[:resumeStatePath] = path
       end
+      op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
+        k, v = kv.split('=', 2)
+        v ||= "true"
+        options[:systemProperty][k] = v
+      end
       args = 1..1
 
     when :preview
@@ -124,6 +135,11 @@ module Embulk
       op.on('-G', '--vertical', "Use vertical output format", TrueClass) do |b|
         options[:previewOutputFormat] = "vertical"
       end
+      op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
+        k, v = kv.split('=', 2)
+        v ||= "true"
+        options[:systemProperty][k] = v
+      end
       args = 1..1
 
     when :guess
@@ -146,6 +162,11 @@ module Embulk
       op.on('-g', '--guess NAMES', "Comma-separated list of guess plugin names") do |names|
         (options[:guessPlugins] ||= []).concat names.split(",")
       end
+      op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
+        k, v = kv.split('=', 2)
+        v ||= "true"
+        options[:systemProperty][k] = v
+      end
       args = 1..1
 
     when :new

data/lib/embulk/data/new/java/encoder.java.erb

@@ -64,17 +64,17 @@ public class <%= java_class_name %>
         //    public OutputStream openNext() throws IOException
         //    {
         //        output.nextFile();
-        //        return newEncoderOutputStream(output);
+        //        return newEncoderOutputStream(task, output);
         //    }
         //
         //    public void finish() throws IOException
         //    {
-        //        fileOutput.finish();
+        //        output.finish();
         //    }
         //
         //    public void close() throws IOException
         //    {
-        //        fileOutput.close();
+        //        output.close();
         //    }
         //});
     }

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.6.19'
+  VERSION = '0.6.20'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.6.19
+  version: 0.6.20
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-24 00:00:00.000000000 Z
+date: 2015-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -339,6 +339,7 @@ files:
 - embulk-docs/src/release/release-0.6.18.rst
 - embulk-docs/src/release/release-0.6.19.rst
 - embulk-docs/src/release/release-0.6.2.rst
+- embulk-docs/src/release/release-0.6.20.rst
 - embulk-docs/src/release/release-0.6.3.rst
 - embulk-docs/src/release/release-0.6.4.rst
 - embulk-docs/src/release/release-0.6.5.rst
@@ -454,8 +455,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.6.19.jar
-- classpath/embulk-standards-0.6.19.jar
+- classpath/embulk-core-0.6.20.jar
+- classpath/embulk-standards-0.6.20.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar