embulk 0.4.10 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1aea38d2d64e785a477bb3b7aa81a0801b78bbf9
-  data.tar.gz: b1c68158658a6c8274cb4375be5c03e6c58410e1
+  metadata.gz: 5579fdedc41918d4b640030c9f2c8b088e3687ce
+  data.tar.gz: 0c01cabaca3edfee8c64a4add44a824b000d7a10
 SHA512:
-  metadata.gz: 38bed0a199023c3989997a48a466c70e8e6a778714951c0cbef215dbb351c4a6eced66722f41ed7dfb8984976f2d2c44352a955c7146d871de0c1c8e9e8d47d3
-  data.tar.gz: 68fa1091e21c306cc7ed12e87f68e387e3a98e4319b5806b2eaf09eb4991fe1a0bc09fd86490155a8be0fab2000c37c6781fdfcfd0c6072eac61913f79e09f82
+  metadata.gz: d4c53e8d4452494e615d6cf0e8600a9874512fa2ca7ed7e024621438fb8a88a358fe91d800c4c8fe6516cc6cf5cd0bb29f3499fe431007deda40e0a9b1fadece
+  data.tar.gz: eea74616fed84cbbefd494a457748c533f9f8551ae240417b7741cba10497cd7a7e1c319410b7738e3f790785527c025c627390593cfcdeb132506359fa9eea9

data/README.md

@@ -28,7 +28,7 @@ The single-file package is the simplest way to try Embulk. You can download the
 Following 4 commands install embulk to your home directory:
 
 ```
-curl --create-dirs -o ~/.embulk/bin/embulk -L https://bintray.com/artifact/download/embulk/maven/embulk-0.4.10.jar
+curl --create-dirs -o ~/.embulk/bin/embulk -L https://bintray.com/artifact/download/embulk/maven/embulk-0.5.0.jar
 chmod +x ~/.embulk/bin/embulk
 echo 'export PATH="$HOME/.embulk/bin:$PATH"' >> ~/.bashrc
 source ~/.bashrc
@@ -39,7 +39,7 @@ source ~/.bashrc
 You can assume the jar file is a .bat file.
 
 ```
-curl -o embulk.bat -L https://bintray.com/artifact/download/embulk/maven/embulk-0.4.10.jar
+curl -o embulk.bat -L https://bintray.com/artifact/download/embulk/maven/embulk-0.5.0.jar
 ```
 
 ### Trying examples

data/build.gradle

@@ -12,7 +12,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.4.10'
+    version = '0.5.0'
 
     apply plugin: 'java'
     apply plugin: 'maven-publish'
@@ -171,9 +171,10 @@ project(':embulk-cli') {
     }
 
     task classpath(type: Copy) {
-        doFirst { file("${rootProject.projectDir}/classpath").mkdirs() }
+        File dest = file("${rootProject.projectDir}/classpath")
+        doFirst { dest.deleteDir() }
         from configurations.runtime
-        into "${rootProject.projectDir}/classpath"
+        into dest
     }
 }
 

data/embulk-core/src/main/java/org/embulk/command/Runner.java

@@ -1,6 +1,7 @@
 package org.embulk.command;
 
 import java.util.List;
+import java.util.ArrayList;
 import java.util.Map;
 import java.util.HashMap;
 import java.io.File;
@@ -19,6 +20,7 @@ import org.embulk.config.ConfigLoader;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ModelManager;
 import org.embulk.config.ConfigException;
+import org.embulk.plugin.PluginType;
 import org.embulk.exec.LocalExecutor;
 import org.embulk.exec.ExecutionResult;
 import org.embulk.exec.GuessExecutor;
@@ -45,6 +47,9 @@ public class Runner
 
         private String previewOutputFormat;
         public String getPreviewOutputFormat() { return previewOutputFormat; };
+
+        private List<PluginType> guessPlugins;
+        public List<PluginType> getGuessPlugins() { return guessPlugins; }
     }
 
     private final Options options;
@@ -57,12 +62,26 @@ public class Runner
         ModelManager bootstrapModelManager = new ModelManager(null, new ObjectMapper());
         this.options = bootstrapModelManager.readObject(Options.class, optionJson);
         this.systemConfig = new ConfigLoader(bootstrapModelManager).fromPropertiesYamlLiteral(System.getProperties(), "embulk.");
+        mergeOptionsToSystemConfig(options, systemConfig);
+        this.service = new EmbulkService(systemConfig);
+        this.injector = service.getInjector();
+    }
+
+    @SuppressWarnings("unchecked")
+    private void mergeOptionsToSystemConfig(Options options, ConfigSource systemConfig)
+    {
         String logLevel = options.getLogLevel();
         if (logLevel != null) {
-            systemConfig.set("logLevel", logLevel);
+            systemConfig.set("log_level", logLevel);
+        }
+
+        List<PluginType> guessPlugins = options.getGuessPlugins();
+        if (guessPlugins != null && !guessPlugins.isEmpty()) {
+            List<PluginType> list = new ArrayList<PluginType>() { };
+            list = systemConfig.get((Class<List<PluginType>>) list.getClass(), "guess_plugins", list);
+            list.addAll(guessPlugins);
+            systemConfig.set("guess_plugins", list);
         }
-        this.service = new EmbulkService(systemConfig);
-        this.injector = service.getInjector();
     }
 
     public void main(String command, String[] args)

data/embulk-core/src/main/java/org/embulk/exec/ForGuess.java

@@ -0,0 +1,16 @@
+package org.embulk.exec;
+
+import javax.inject.Qualifier;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+@Retention(RUNTIME)
+@Target({FIELD, PARAMETER, METHOD})
+@Qualifier
+public @interface ForGuess
+{
+}

data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java

@@ -1,11 +1,15 @@
 package org.embulk.exec;
 
 import java.util.List;
+import java.util.Set;
 import java.util.ArrayList;
 import com.google.common.collect.ImmutableList;
+import com.google.common.base.Throwables;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
-import com.google.common.base.Throwables;
+import com.google.inject.Binder;
+import com.google.inject.multibindings.Multibinder;
+import org.embulk.plugin.PluginType;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
@@ -14,7 +18,6 @@ import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.CommitReport;
-import org.embulk.plugin.PluginType;
 import org.embulk.spi.Schema;
 import org.embulk.spi.Column;
 import org.embulk.spi.Page;
@@ -33,10 +36,16 @@ import org.embulk.spi.FileInputRunner;
 
 public class GuessExecutor
 {
-    private final Injector injector;
-    private final ConfigSource systemConfig;
     private final List<PluginType> defaultGuessPlugins;
 
+    private interface GuessExecutorSystemTask
+            extends Task
+    {
+        @Config("guess_plugins")
+        @ConfigDefault("[]")
+        public List<PluginType> getGuessPlugins();
+    }
+
     private interface GuessExecutorTask
             extends Task
     {
@@ -49,19 +58,22 @@ public class GuessExecutor
         public List<PluginType> getExcludeGuessPlugins();
     }
 
+    public static void registerDefaultGuessPluginTo(Binder binder, PluginType type)
+    {
+        Multibinder<PluginType> multibinder = Multibinder.newSetBinder(binder, PluginType.class, ForGuess.class);
+        multibinder.addBinding().toInstance(type);
+    }
+
     @Inject
-    public GuessExecutor(Injector injector,
-            @ForSystemConfig ConfigSource systemConfig)
+    public GuessExecutor(@ForSystemConfig ConfigSource systemConfig,
+            @ForGuess Set<PluginType> defaultGuessPlugins)
     {
-        this.injector = injector;
-        this.systemConfig = systemConfig;
-
-        // TODO get default guess plugins from injector using Multibinder
-        this.defaultGuessPlugins = ImmutableList.of(
-                new PluginType("gzip"),
-                new PluginType("charset"),
-                new PluginType("newline"),
-                new PluginType("csv"));
+        GuessExecutorSystemTask systemTask = systemConfig.loadConfig(GuessExecutorSystemTask.class);
+
+        ImmutableList.Builder<PluginType> list = ImmutableList.builder();
+        list.addAll(defaultGuessPlugins);
+        list.addAll(systemTask.getGuessPlugins());
+        this.defaultGuessPlugins = list.build();
     }
 
     public ConfigDiff guess(ExecSession exec, final ConfigSource config)
@@ -80,29 +92,50 @@ public class GuessExecutor
         }
     }
 
+    protected InputPlugin newInputPlugin(ConfigSource inputConfig)
+    {
+        return Exec.newPlugin(InputPlugin.class, inputConfig.get(PluginType.class, "type"));
+    }
+
     private ConfigDiff doGuess(ConfigSource config)
     {
-        Buffer sample = SamplingParserPlugin.runFileInputSampling(config);
-        if (sample.limit() == 0) {
-            throw new NoSampleException("Can't get sample data because the first input file is empty");
+        ConfigSource inputConfig = config.getNested("in");
+
+        InputPlugin input = newInputPlugin(inputConfig);
+
+        ConfigDiff inputGuessed;
+        try {
+            inputGuessed = input.guess(inputConfig);
+        } catch (AbstractMethodError ex) {
+            // for backward compatibility with embulk v0.4 interface
+            throw new UnsupportedOperationException(input.getClass().getSimpleName()+".guess(ConfigSource) is not implemented. This input plugin does not support guessing.");
         }
 
+        ConfigDiff wrapped = Exec.newConfigDiff();
+        wrapped.getNestedOrSetEmpty("in").merge(inputGuessed);
+        return wrapped;
+    }
+
+    // called by FileInputRunner
+    public ConfigDiff guessParserConfig(Buffer sample, ConfigSource inputConfig, ConfigSource execConfig)
+    {
         List<PluginType> guessPlugins = new ArrayList<PluginType>(defaultGuessPlugins);
-        GuessExecutorTask task = config.getNestedOrSetEmpty("exec").loadConfig(GuessExecutorTask.class);
+
+        GuessExecutorTask task = execConfig.loadConfig(GuessExecutorTask.class);
         guessPlugins.addAll(task.getGuessPlugins());
         guessPlugins.removeAll(task.getExcludeGuessPlugins());
 
-        return runGuessInput(sample, config, guessPlugins);
+        return guessParserConfig(sample, inputConfig, guessPlugins);
     }
 
-    private ConfigDiff runGuessInput(Buffer sample,
+    private ConfigDiff guessParserConfig(Buffer sample,
             ConfigSource config, List<PluginType> guessPlugins)
     {
         // repeat guessing upto 10 times
         ConfigDiff lastGuessed = Exec.newConfigDiff();
         for (int i=0; i < 10; i++) {
             // include last-guessed config to run guess input
-            ConfigSource originalConfig = config.getNested("in").deepCopy().merge(lastGuessed);
+            ConfigSource originalConfig = config.deepCopy().merge(lastGuessed);
             ConfigSource guessInputConfig = originalConfig.deepCopy();
             guessInputConfig.getNestedOrSetEmpty("parser")
                 .set("type", "system_guess")  // override in.parser.type so that FileInputPlugin creates GuessParserPlugin
@@ -142,19 +175,12 @@ public class GuessExecutor
             // merge to the last-guessed config
             if (lastGuessed.equals(guessed)) {
                 // not changed
-                return wrapInIn(lastGuessed);
+                return lastGuessed;
             }
             lastGuessed = guessed;
         }
 
-        return wrapInIn(lastGuessed);
-    }
-
-    private static ConfigDiff wrapInIn(ConfigDiff lastGuessed)
-    {
-        ConfigDiff wrapped = Exec.newConfigDiff();
-        wrapped.getNestedOrSetEmpty("in").merge(lastGuessed);
-        return wrapped;
+        return lastGuessed;
     }
 
     private static class BufferFileInputPlugin

data/embulk-core/src/main/java/org/embulk/exec/LoggerProvider.java

@@ -18,7 +18,7 @@ public class LoggerProvider
         Properties prop = new Properties();
 
         final String level;
-        String logLevel = systemConfig.get(String.class, "logLevel", "info");
+        String logLevel = systemConfig.get(String.class, "log_level", "info");  // here can't use loadConfig because ModelManager uses LoggerProvider
         switch (logLevel) {
         case "fatal": level = "FATAL"; break;
         case "error": level = "ERROR"; break;

data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java

@@ -13,13 +13,15 @@ import org.embulk.spi.Buffer;
 import org.embulk.spi.InputPlugin;
 import org.embulk.spi.ParserPlugin;
 import org.embulk.spi.FileInput;
+import org.embulk.spi.FileInputRunner;
 import org.embulk.spi.PageOutput;
+import org.embulk.exec.ForSystemConfig;
 import static org.embulk.spi.util.Inputs.each;
 
 /*
- * Used by GuessExecutor
+ * Used by FileInputRunner.guess
  */
-class SamplingParserPlugin
+public class SamplingParserPlugin
         implements ParserPlugin
 {
     private final int maxSampleSize;
@@ -44,13 +46,12 @@ class SamplingParserPlugin
         throw new SampledNoticeError(buffer);
     }
 
-    static Buffer runFileInputSampling(ConfigSource config)
+    public static Buffer runFileInputSampling(final FileInputRunner input, ConfigSource inputConfig)
     {
         // override in.parser.type so that FileInputRunner creates GuessParserPlugin
-        ConfigSource samplingInputConfig = config.getNested("in").deepCopy();
+        ConfigSource samplingInputConfig = inputConfig.deepCopy();
         samplingInputConfig.getNestedOrSetEmpty("parser").set("type", "system_sampling");
 
-        final InputPlugin input = Exec.newPlugin(InputPlugin.class, samplingInputConfig.get(PluginType.class, "type"));
         try {
             input.transaction(samplingInputConfig, new InputPlugin.Control() {
                 public List<CommitReport> run(TaskSource taskSource, Schema schema, int taskCount)

data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java

@@ -27,8 +27,11 @@ public class ExecSession
     private final ModelManager modelManager;
     private final PluginManager pluginManager;
     private final BufferAllocator bufferAllocator;
+
+    private final ConfigSource execConfig;
     private final Timestamp transactionTime;
     private final DateTimeZone transactionTimeZone;
+
     private final boolean preview;
 
     public interface SessionTask
@@ -44,16 +47,6 @@ public class ExecSession
     }
 
     public ExecSession(Injector injector, ConfigSource execConfig)
-    {
-        this(injector, execConfig.loadConfig(SessionTask.class));
-    }
-
-    public ExecSession(Injector injector, TaskSource taskSource)
-    {
-        this(injector, taskSource.loadTask(SessionTask.class));
-    }
-
-    public ExecSession(Injector injector, SessionTask task)
     {
         this.injector = injector;
         this.loggerFactory = injector.getInstance(ILoggerFactory.class);
@@ -61,8 +54,11 @@ public class ExecSession
         this.pluginManager = injector.getInstance(PluginManager.class);
         this.bufferAllocator = injector.getInstance(BufferAllocator.class);
 
+        this.execConfig = execConfig.deepCopy();
+        SessionTask task = execConfig.loadConfig(SessionTask.class);
         this.transactionTime = task.getTransactionTime().or(Timestamp.ofEpochMilli(System.currentTimeMillis()));  // TODO get nanoseconds for default
         this.transactionTimeZone = task.getTransactionTimeZone();
+
         this.preview = false;
     }
 
@@ -73,8 +69,11 @@ public class ExecSession
         this.modelManager = copy.modelManager;
         this.pluginManager = copy.pluginManager;
         this.bufferAllocator = copy.bufferAllocator;
+
+        this.execConfig = copy.execConfig;
         this.transactionTime = copy.transactionTime;
         this.transactionTimeZone = copy.transactionTimeZone;
+
         this.preview = preview;
     }
 
@@ -120,6 +119,11 @@ public class ExecSession
         return bufferAllocator;
     }
 
+    public ConfigSource getExecConfig()
+    {
+        return execConfig;
+    }
+
     public <T> T newPlugin(Class<T> iface, PluginType type)
     {
         return pluginManager.newPlugin(iface, type);

data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java

@@ -11,6 +11,9 @@ import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.plugin.PluginType;
 import org.embulk.spi.util.Decoders;
+import org.embulk.exec.GuessExecutor;
+import org.embulk.exec.SamplingParserPlugin;
+import org.embulk.exec.NoSampleException;
 
 public class FileInputRunner
         implements InputPlugin
@@ -60,6 +63,7 @@ public class FileInputRunner
         return fileInputPlugin.transaction(config, new RunnerControl(task, control));
     }
 
+    @Override
     public ConfigDiff resume(TaskSource taskSource,
             Schema schema, int taskCount,
             InputPlugin.Control control)
@@ -68,6 +72,18 @@ public class FileInputRunner
         return fileInputPlugin.resume(task.getFileInputTaskSource(), taskCount, new RunnerControl(task, control));
     }
 
+    @Override
+    public ConfigDiff guess(ConfigSource config)
+    {
+        Buffer sample = SamplingParserPlugin.runFileInputSampling(this, config);
+        if (sample.limit() == 0) {
+            throw new NoSampleException("Can't get sample data because the first input file is empty");
+        }
+
+        GuessExecutor guessExecutor = Exec.session().getInjector().getInstance(GuessExecutor.class);
+        return guessExecutor.guessParserConfig(sample, config, Exec.session().getExecConfig());
+    }
+
     private class RunnerControl
             implements FileInputPlugin.Control
     {

data/embulk-core/src/main/java/org/embulk/spi/InputPlugin.java

@@ -28,4 +28,6 @@ public interface InputPlugin
     public CommitReport run(TaskSource taskSource,
             Schema schema, int taskIndex,
             PageOutput output);
+
+    public ConfigDiff guess(ConfigSource config);
 }

data/embulk-core/src/main/java/org/embulk/spi/util/FileInputInputStream.java

@@ -80,7 +80,8 @@ public class FileInputInputStream
     @Override
     public long skip(long len)
     {
-        return read(null, 0, (int) Math.min(len, Integer.MAX_VALUE));
+        int skipped = read(null, 0, (int) Math.min(len, Integer.MAX_VALUE));
+        return skipped > 0 ? skipped : 0;
     }
 
     private boolean nextBuffer()

data/embulk-core/src/test/java/org/embulk/spi/TestFileInputInputStream.java

@@ -64,4 +64,26 @@ public class TestFileInputInputStream
         assertEquals(expected.length, pos);
         assertArrayEquals(expected, actual);
     }
+
+    @Test
+    public void testSkipReturnsZeroForNoData() {
+        FileInputInputStream in = new FileInputInputStream(new MockFileInput());
+        assertEquals("Verify skip() returns 0 when there is no data.", 0L, in.skip(1));
+    }
+
+    private static class MockFileInput implements FileInput {
+        @Override
+        public boolean nextFile() {
+            return false;
+        }
+
+        @Override
+        public Buffer poll() {
+            return null;
+        }
+
+        @Override
+        public void close() {
+        }
+    }
 }