embulk 0.6.2 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 148c75f1b85e1d85859d0be3293d85e5fc3f1bab
-  data.tar.gz: c07d2801fc7cc4b0caa60d2ff692553b29827cfe
+  metadata.gz: b37c9dd1d3a700e0057040d27f0a29e917e21b13
+  data.tar.gz: 3372de3c3f382dc6f83bea6936d84af3b815194c
 SHA512:
-  metadata.gz: 6f499389219a19663e1e3789b68bf1eea7f650df7921347ec5d0641395a1a1b11a5087f571a9e0ac3536cc1156592cb15b1f5b4283e069f68d456449c90d4c69
-  data.tar.gz: 78eeb275c30689f63947961fd1d2f0b01d96abfef0c16b1e68370fdca0570c809f1e4cb38924e67c10f11e86759c88e706dcb32c45f55eaf10876055bdedf8ea
+  metadata.gz: 7d0c416b6c063508b95d23941c5cf97149bcf6fd11134392f79d0d7540f106df019e502fc7296ba47a94a945a64c8b34a4d4aaf682227d390300592d598c0f1a
+  data.tar.gz: b370deb8322442973cc3109ca2a0b8eb3931d11840f55e7d230f56c10de45c93a71d39cb892dad491ef67870c9e77932fabee4d82cf99ea601ba3417ea3e7cc8

data/.travis.yml

@@ -8,7 +8,7 @@ sudo: false
 install:
   - pip install --user sphinx
 script:
-  - ./gradlew check
+  - ./gradlew check rubyTest
 after_success:
   - PATH="$HOME/.local/bin:$PATH" ./embulk-docs/push-gh-pages.sh
 env:

data/bin/embulk

@@ -69,9 +69,9 @@ unless jruby_complete
 end
 
 if overwrite_optimize == true || (default_optimize == true && overwrite_optimize != false)
-    java_args = %w[-XX:+AggressiveOpts -XX:+UseConcMarkSweepGC] + java_args
+  java_args = %w[-XX:+AggressiveOpts -XX:+UseConcMarkSweepGC] + java_args
 else
-    java_args = %w[-XX:+AggressiveOpts -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none] + java_args
+  java_args = %w[-XX:+AggressiveOpts -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none] + java_args
 end
 
 # java ... -jar ruby-complete.jar $EMBULK_HOME/lib/embulk/command/embulk.rb "$@"

data/build.gradle

@@ -5,14 +5,17 @@ plugins {
     id 'com.github.jruby-gradle.base' version '0.1.5'
     id 'com.github.johnrengelman.shadow' version '1.2.0'
 }
-import com.github.jrubygradle.JRubyExec
 
 def java_projects = [project(":embulk-core"), project(":embulk-standards"), project(":embulk-cli")]
 def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.6.2'
+    version = '0.6.3'
+
+    ext {
+        jrubyVersion = '1.7.19'
+    }
 
     apply plugin: 'java'
     apply plugin: 'maven-publish'
@@ -137,6 +140,11 @@ subprojects {
     }
 }
 
+import com.github.jrubygradle.JRubyExec
+jruby {
+    execVersion = project.jrubyVersion
+}
+
 dependencies {
     jrubyExec 'rubygems:test-unit:3.0.+'
 }
@@ -187,6 +195,7 @@ task rubyTest(type: JRubyExec) {
     jrubyArgs '-Ilib', '-Itest', '-rtest/unit', '-eTest::Unit::AutoRunner.run(true, *ARGV)'
     script 'test'
 }
+rubyTest.dependsOn('classpath')
 
 //
 // gem task

data/embulk-cli/src/main/java/org/embulk/cli/Main.java

@@ -4,8 +4,6 @@ public class Main
 {
     public static void main(String[] args)
     {
-        // TODO set GEM_HOME to point the internal gem repository created by gem-maven-plugin?
-
         // $ java -jar jruby-complete.jar classpath:embulk/command/embulk.rb "$@"
         String[] jrubyArgs = new String[args.length + 1];
         jrubyArgs[0] = "classpath:embulk/command/embulk.rb";

data/embulk-core/build.gradle

@@ -27,7 +27,7 @@ dependencies {
     compile 'log4j:log4j:1.2.17'
     compile 'org.slf4j:slf4j-api:1.7.10'
     compile 'org.slf4j:slf4j-log4j12:1.7.10'
-    compile 'org.jruby:jruby-complete:1.7.19'
+    compile 'org.jruby:jruby-complete:' + project.jrubyVersion
     compile 'com.google.code.findbugs:annotations:3.0.0'
     compile 'org.yaml:snakeyaml:1.14'
     compile 'javax.validation:validation-api:1.1.0.Final'

data/embulk-core/src/main/java/org/embulk/command/Runner.java

@@ -50,6 +50,9 @@ public class Runner
 
         private List<PluginType> guessPlugins;
         public List<PluginType> getGuessPlugins() { return guessPlugins; }
+
+        private boolean useGlobalRubyRuntime;
+        public boolean getUseGlobalRubyRuntime() { return useGlobalRubyRuntime; }
     }
 
     private final Options options;
@@ -72,16 +75,23 @@ public class Runner
     {
         String logLevel = options.getLogLevel();
         if (logLevel != null) {
+            // used by LoggerProvider
             systemConfig.set("log_level", logLevel);
         }
 
         List<PluginType> guessPlugins = options.getGuessPlugins();
         if (guessPlugins != null && !guessPlugins.isEmpty()) {
+            // used by GuessExecutor
             List<PluginType> list = new ArrayList<PluginType>() { };
             list = systemConfig.get((Class<List<PluginType>>) list.getClass(), "guess_plugins", list);
             list.addAll(guessPlugins);
             systemConfig.set("guess_plugins", list);
         }
+
+        if (options.getUseGlobalRubyRuntime()) {
+            // used by JRubyScriptingModule
+            systemConfig.set("use_global_ruby_runtime", true);
+        }
     }
 
     public void main(String command, String[] args)

data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java

@@ -3,6 +3,7 @@ package org.embulk.exec;
 import java.util.List;
 import java.util.Set;
 import java.util.ArrayList;
+import java.util.concurrent.ExecutionException;
 import com.google.common.collect.ImmutableList;
 import com.google.common.base.Throwables;
 import com.google.inject.Inject;
@@ -87,8 +88,8 @@ public class GuessExecutor
                     }
                 }
             });
-        } catch (Exception ex) {
-            throw Throwables.propagate(ex);
+        } catch (ExecutionException ex) {
+            throw Throwables.propagate(ex.getCause());
         }
     }
 

data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java

@@ -69,7 +69,7 @@ public class PreviewExecutor
                 }
             });
         } catch (Exception ex) {
-            throw Throwables.propagate(ex);
+            throw Throwables.propagate(ex.getCause());
         }
     }
 

data/embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.java

@@ -16,10 +16,12 @@ import com.google.inject.Key;
 import com.google.inject.spi.Dependency;
 import com.google.inject.spi.ProviderWithDependencies;
 import org.jruby.CompatVersion;
+import org.jruby.embed.LocalContextScope;
 import org.jruby.embed.ScriptingContainer;
 import org.embulk.plugin.PluginSource;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ModelManager;
+import org.embulk.exec.ForSystemConfig;
 import org.embulk.spi.BufferAllocator;
 
 public class JRubyScriptingModule
@@ -27,15 +29,12 @@ public class JRubyScriptingModule
 {
     public JRubyScriptingModule(ConfigSource systemConfig)
     {
-        // TODO get jruby-home from systemConfig to call jruby.container.setHomeDirectory
-        // TODO get jruby-load-paths from systemConfig to call jruby.container.setLoadPaths
     }
 
     @Override
     public void configure(Binder binder)
     {
         binder.bind(ScriptingContainer.class).toProvider(ScriptingContainerProvider.class).in(Scopes.SINGLETON);
-        //binder.bind(JRubyModule.class).in(Scopes.SINGLETON);
 
         Multibinder<PluginSource> multibinder = Multibinder.newSetBinder(binder, PluginSource.class);
         multibinder.addBinding().to(JRubyPluginSource.class);
@@ -45,16 +44,25 @@ public class JRubyScriptingModule
             implements ProviderWithDependencies<ScriptingContainer>
     {
         private final Injector injector;
+        private final boolean useGlobalRubyRuntime;
 
         @Inject
-        public ScriptingContainerProvider(Injector injector)
+        public ScriptingContainerProvider(Injector injector, @ForSystemConfig ConfigSource systemConfig)
         {
             this.injector = injector;
+
+            // use_global_ruby_runtime is valid only when it's guaranteed that just one Injector is
+            // instantiated in this JVM.
+            this.useGlobalRubyRuntime = systemConfig.get(boolean.class, "use_global_ruby_runtime", false);
+
+            // TODO get jruby-home from systemConfig to call jruby.container.setHomeDirectory
+            // TODO get jruby-load-paths from systemConfig to call jruby.container.setLoadPaths
         }
 
         public ScriptingContainer get()
         {
-            ScriptingContainer jruby = new ScriptingContainer();
+            LocalContextScope scope = (useGlobalRubyRuntime ? LocalContextScope.SINGLETON : LocalContextScope.SINGLETHREAD);
+            ScriptingContainer jruby = new ScriptingContainer(scope);
             jruby.setCompatVersion(CompatVersion.RUBY1_9);
 
             // Search embulk/java/bootstrap.rb from a $LOAD_PATH.
@@ -72,9 +80,13 @@ public class JRubyScriptingModule
             // jruby searches embulk/java/bootstrap.rb from the beginning of $LOAD_PATH.
             jruby.runScriptlet("require 'embulk/java/bootstrap'");
 
+            // TODO validate Embulk::Java::Injected::Injector doesn't exist? If it already exists,
+            //      Injector is created more than once in this JVM although use_global_ruby_runtime
+            //      is set to true.
+
             // set some constants
             jruby.callMethod(
-                    jruby.runScriptlet("Embulk::Java"),
+                    jruby.runScriptlet("Embulk::Java::Injected"),
                     "const_set", "Injector", injector);
             jruby.callMethod(
                     jruby.runScriptlet("Embulk::Java::Injected"),

data/embulk-core/src/main/java/org/embulk/plugin/PluginManager.java

@@ -13,7 +13,7 @@ public class PluginManager
     private final List<PluginSource> sources;
     private final Injector injector;
 
-    // Set<PluginSource> is injected BuiltinPluginSourceModule or extensions
+    // Set<PluginSource> is injected by BuiltinPluginSourceModule or extensions
     // using Multibinder<PluginSource>.
     @Inject
     public PluginManager(Set<PluginSource> pluginSources, Injector injector)
@@ -32,31 +32,34 @@ public class PluginManager
             throw new ConfigException(String.format("%s type is not set (if you intend to use NullOutputPlugin, you should enclose null in quotes such as {type: \"null\"}.", iface.getSimpleName()));
         }
 
-        List<Throwable> causes = new ArrayList<Throwable>();
+        List<PluginSourceNotMatchException> exceptions = new ArrayList<>();
         for (PluginSource source : sources) {
             try {
                 return source.newPlugin(iface, type);
             } catch (PluginSourceNotMatchException e) {
-                if (e.getCause() != null) {
-                    causes.add(e.getCause());
-                } else {
-                    causes.add(e);
-                }
+                exceptions.add(e);
             }
         }
 
+        throw buildPluginNotFoundException(iface, type, exceptions);
+    }
+
+    private static ConfigException buildPluginNotFoundException(Class<?> iface, PluginType type,
+            List<PluginSourceNotMatchException> exceptions)
+    {
         StringBuilder message = new StringBuilder();
         message.append(String.format("%s '%s' is not found.", iface.getSimpleName(), type.getName()));
-        for (Throwable cause : causes) {
+        for (PluginSourceNotMatchException exception : exceptions) {
+            Throwable cause = (exception.getCause() == null ? exception : exception.getCause());
             if (cause.getMessage() != null) {
                 message.append(String.format("%n"));
                 message.append(cause.getMessage());
             }
         }
         ConfigException e = new ConfigException(message.toString());
-        for (Throwable cause : causes) {
-            e.addSuppressed(cause);
+        for (PluginSourceNotMatchException exception : exceptions) {
+            e.addSuppressed(exception);
         }
-        throw e;
+        return e;
     }
 }

data/embulk-core/src/main/java/org/embulk/spi/Column.java

@@ -1,6 +1,6 @@
 package org.embulk.spi;
 
-import com.google.common.base.Objects;
+import java.util.Objects;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.embulk.spi.type.Type;
@@ -72,15 +72,15 @@ public class Column
             return false;
         }
         Column other = (Column) obj;
-        return Objects.equal(index, other.index) &&
-            Objects.equal(name, other.name) &&
-            Objects.equal(type, other.type);
+        return Objects.equals(index, other.index) &&
+            Objects.equals(name, other.name) &&
+            Objects.equals(type, other.type);
     }
 
     @Override
     public int hashCode()
     {
-        return Objects.hashCode(index, name, type);
+        return Objects.hash(index, name, type);
     }
 
     @Override

data/embulk-core/src/main/java/org/embulk/spi/ColumnConfig.java

@@ -1,6 +1,6 @@
 package org.embulk.spi;
 
-import com.google.common.base.Objects;
+import java.util.Objects;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.embulk.spi.type.Type;
@@ -60,14 +60,14 @@ public class ColumnConfig
             return false;
         }
         ColumnConfig other = (ColumnConfig) obj;
-        return Objects.equal(this.name, other.name) &&
-            Objects.equal(type, other.type);
+        return Objects.equals(this.name, other.name) &&
+            Objects.equals(type, other.type);
     }
 
     @Override
     public int hashCode()
     {
-        return Objects.hashCode(name, type);
+        return Objects.hash(name, type);
     }
 
     @Override

data/embulk-core/src/main/java/org/embulk/spi/Exec.java

@@ -2,6 +2,7 @@ package org.embulk.spi;
 
 import java.util.concurrent.ExecutionException;
 import org.slf4j.Logger;
+import com.google.inject.Injector;
 import org.embulk.config.Task;
 import org.embulk.config.ModelManager;
 import org.embulk.config.CommitReport;
@@ -38,6 +39,11 @@ public class Exec
         return session;
     }
 
+    public static Injector getInjector()
+    {
+        return session().getInjector();
+    }
+
     public static Logger getLogger(String name)
     {
         return session().getLogger(name);

data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java

@@ -80,7 +80,7 @@ public class FileInputRunner
             throw new NoSampleException("Can't get sample data because the first input file is empty");
         }
 
-        GuessExecutor guessExecutor = Exec.session().getInjector().getInstance(GuessExecutor.class);
+        GuessExecutor guessExecutor = Exec.getInjector().getInstance(GuessExecutor.class);
         return guessExecutor.guessParserConfig(sample, config, Exec.session().getExecConfig());
     }
 

data/embulk-core/src/main/java/org/embulk/spi/Schema.java

@@ -1,7 +1,7 @@
 package org.embulk.spi;
 
 import java.util.List;
-import com.google.common.base.Objects;
+import java.util.Objects;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;
 import org.embulk.spi.type.Type;
@@ -78,7 +78,7 @@ public class Schema
             return false;
         }
         Schema other = (Schema) obj;
-        return Objects.equal(columns, other.columns);
+        return Objects.equals(columns, other.columns);
     }
 
     @Override

data/embulk-core/src/main/java/org/embulk/spi/SchemaConfig.java

@@ -1,7 +1,7 @@
 package org.embulk.spi;
 
 import java.util.List;
-import com.google.common.base.Objects;
+import java.util.Objects;
 import com.google.common.collect.ImmutableList;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;
@@ -41,7 +41,7 @@ public class SchemaConfig
             return false;
         }
         SchemaConfig other = (SchemaConfig) obj;
-        return Objects.equal(columns, other.columns);
+        return Objects.equals(columns, other.columns);
     }
 
     @Override

data/embulk-core/src/main/java/org/embulk/spi/unit/ByteSize.java

@@ -0,0 +1,148 @@
+package org.embulk.spi.unit;
+
+import java.util.Objects;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import com.google.common.base.Preconditions;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+public class ByteSize
+        implements Comparable<ByteSize>
+{
+    private static final Pattern PATTERN = Pattern.compile("\\A(\\d+(?:\\.\\d+)?)\\s?([a-zA-Z]*)\\z");
+
+    private final long bytes;
+    private final Unit displayUnit;
+
+    public ByteSize(double size, Unit unit)
+    {
+        Preconditions.checkArgument(!Double.isInfinite(size), "size is infinite");
+        Preconditions.checkArgument(!Double.isNaN(size), "size is not a number");
+        Preconditions.checkArgument(size >= 0, "size is negative");
+        Preconditions.checkNotNull(unit, "unit is null");
+        Preconditions.checkArgument(size * unit.getFactor() <= (double) Long.MAX_VALUE, "size is large than (2^63)-1 in bytes");
+        this.bytes = (long) (size * unit.getFactor());
+        this.displayUnit = unit;
+    }
+
+    @JsonCreator
+    public ByteSize(long bytes)
+    {
+        Preconditions.checkArgument(bytes >= 0, "size is negative");
+        this.bytes = bytes;
+        this.displayUnit = Unit.BYTES;
+    }
+
+    public long getBytes()
+    {
+        return bytes;
+    }
+
+    public long roundTo(Unit unit)
+    {
+        return (long) Math.floor(getValue(unit) + 0.5);
+    }
+
+    public double getValue(Unit unit)
+    {
+        return bytes / (double) unit.getFactor();
+    }
+
+    @JsonCreator
+    public static ByteSize parseByteSize(String size)
+    {
+        Preconditions.checkNotNull(size, "size is null");
+        Preconditions.checkArgument(!size.isEmpty(), "size is empty");
+
+        Matcher matcher = PATTERN.matcher(size);
+        if (!matcher.matches()) {
+            throw new IllegalArgumentException("Invalid byte size string '" + size + "'");
+        }
+
+        double value = Double.parseDouble(matcher.group(1));  // NumberFormatException extends IllegalArgumentException.
+
+        String unitString = matcher.group(2);
+        if (unitString.isEmpty()) {
+            return new ByteSize(value, Unit.BYTES);
+        } else {
+            String upperUnitString = unitString.toUpperCase(Locale.ENGLISH);
+            for (Unit unit : Unit.values()) {
+                if (unit.getUnitString().toUpperCase(Locale.ENGLISH).equals(upperUnitString)) {
+                    return new ByteSize(value, unit);
+                }
+            }
+        }
+
+        throw new IllegalArgumentException("Unknown unit '" + unitString + "'");
+    }
+
+    @JsonValue
+    @Override
+    public String toString()
+    {
+        double value = getValue(displayUnit);
+        String integer = String.format(Locale.ENGLISH, "%d", (long) value);
+        String decimal = String.format(Locale.ENGLISH, "%.2f", value);
+        if (decimal.equals(integer + ".00")) {
+            return integer + displayUnit.getUnitString();
+        } else {
+            return decimal + displayUnit.getUnitString();
+        }
+    }
+
+    @Override
+    public int compareTo(ByteSize o)
+    {
+        return Long.compare(bytes, o.bytes);
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof ByteSize)) {
+            return false;
+        }
+        ByteSize o = (ByteSize) obj;
+        return this.bytes == o.bytes;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hashCode(bytes);
+    }
+
+    public enum Unit
+    {
+        BYTES(1L, "B"),
+        KB(1L << 10, "KB"),
+        MB(1L << 20, "MB"),
+        GB(1L << 30, "GB"),
+        TB(1L << 40, "TB"),
+        PT(1L << 50, "PB");
+
+        private final long factor;
+        private final String unitString;
+
+        Unit(long factor, String unitString)
+        {
+            this.factor = factor;
+            this.unitString = unitString;
+        }
+
+        long getFactor()
+        {
+            return factor;
+        }
+
+        String getUnitString()
+        {
+            return unitString;
+        }
+    }
+}