embulk 0.7.0-java

data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java

@@ -0,0 +1,52 @@
+package org.embulk.exec;
+
+import org.slf4j.ILoggerFactory;
+import com.google.common.base.Preconditions;
+import com.google.inject.Module;
+import com.google.inject.Binder;
+import com.google.inject.Scopes;
+import com.fasterxml.jackson.module.guice.ObjectMapperModule;
+import com.fasterxml.jackson.datatype.guava.GuavaModule;
+import com.fasterxml.jackson.datatype.joda.JodaModule;
+import org.embulk.config.ModelManager;
+import org.embulk.spi.time.DateTimeZoneSerDe;
+import org.embulk.spi.time.TimestampSerDe;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.ExecutorPlugin;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.util.CharsetSerDe;
+import org.embulk.spi.unit.LocalFileSerDe;
+import static org.embulk.plugin.InjectedPluginSource.registerPluginTo;
+
+public class ExecModule
+        implements Module
+{
+    @Override
+    public void configure(Binder binder)
+    {
+        Preconditions.checkNotNull(binder, "binder is null.");
+
+        binder.bind(ILoggerFactory.class).toProvider(LoggerProvider.class).in(Scopes.SINGLETON);
+        binder.bind(ModelManager.class).in(Scopes.SINGLETON);
+        binder.bind(BufferAllocator.class).to(PooledBufferAllocator.class).in(Scopes.SINGLETON);
+        binder.bind(TempFileAllocator.class).in(Scopes.SINGLETON);
+
+        // GuessExecutor
+        registerPluginTo(binder, ParserPlugin.class, "system_guess", GuessExecutor.GuessParserPlugin.class);
+        registerPluginTo(binder, ParserPlugin.class, "system_sampling", SamplingParserPlugin.class);
+
+        // LocalExecutorPlugin
+        binder.bind(LocalThreadExecutor.class).in(Scopes.SINGLETON);
+        registerPluginTo(binder, ExecutorPlugin.class, "local", LocalExecutorPlugin.class);
+
+        // serde
+        ObjectMapperModule mapper = new ObjectMapperModule();
+        DateTimeZoneSerDe.configure(mapper);
+        TimestampSerDe.configure(mapper);
+        CharsetSerDe.configure(mapper);
+        LocalFileSerDe.configure(mapper);
+        mapper.registerModule(new GuavaModule());  // jackson-datatype-guava
+        mapper.registerModule(new JodaModule());  // jackson-datatype-joda
+        mapper.configure(binder);
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/ExecutionInterruptedException.java

@@ -0,0 +1,10 @@
+package org.embulk.exec;
+
+public class ExecutionInterruptedException
+        extends RuntimeException
+{
+    public ExecutionInterruptedException(Exception cause)
+    {
+        super(cause);
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/ExecutionResult.java

@@ -0,0 +1,26 @@
+package org.embulk.exec;
+
+import java.util.List;
+import org.embulk.config.ConfigDiff;
+
+public class ExecutionResult
+{
+    private final ConfigDiff configDiff;
+    private final List<Throwable> ignoredExceptions;
+
+    public ExecutionResult(ConfigDiff configDiff, List<Throwable> ignoredExceptions)
+    {
+        this.configDiff = configDiff;
+        this.ignoredExceptions = ignoredExceptions;
+    }
+
+    public ConfigDiff getConfigDiff()
+    {
+        return configDiff;
+    }
+
+    public List<Throwable> getIgnoredExceptions()
+    {
+        return ignoredExceptions;
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/ExtensionServiceLoaderModule.java

@@ -0,0 +1,43 @@
+package org.embulk.exec;
+
+import java.util.ServiceLoader;
+import com.google.inject.Module;
+import com.google.inject.Binder;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Extension;
+
+/**
+ * ExtensionServiceLoaderModule loads Extensions using java.util.ServiceLoader
+ * mechanism.
+ * Jar packages providing an extension need to include
+ * META-INF/services/org.embulk.exec.Extension file. Contents of the file is
+ * one-line text of the extension class name (e.g. com.example.MyPluginSourceExtension).
+ */
+public class ExtensionServiceLoaderModule
+        implements Module
+{
+    private final ClassLoader classLoader;
+    private final ConfigSource systemConfig;
+
+    public ExtensionServiceLoaderModule(ConfigSource systemConfig)
+    {
+        this(ExtensionServiceLoaderModule.class.getClassLoader(), systemConfig);
+    }
+
+    public ExtensionServiceLoaderModule(ClassLoader classLoader, ConfigSource systemConfig)
+    {
+        this.classLoader = classLoader;
+        this.systemConfig = systemConfig;
+    }
+
+    @Override
+    public void configure(Binder binder)
+    {
+        ServiceLoader<Extension> serviceLoader = ServiceLoader.load(Extension.class, classLoader);
+        for (Extension extension : serviceLoader) {
+            for (Module module : extension.getModules(systemConfig)) {
+                module.configure(binder);
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/ForGuess.java

@@ -0,0 +1,16 @@
+package org.embulk.exec;
+
+import javax.inject.Qualifier;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+@Retention(RUNTIME)
+@Target({FIELD, PARAMETER, METHOD})
+@Qualifier
+public @interface ForGuess
+{
+}

data/embulk-core/src/main/java/org/embulk/exec/ForSystemConfig.java

@@ -0,0 +1,16 @@
+package org.embulk.exec;
+
+import javax.inject.Qualifier;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+@Retention(RUNTIME)
+@Target({FIELD, PARAMETER, METHOD})
+@Qualifier
+public @interface ForSystemConfig
+{
+}

data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java

@@ -0,0 +1,373 @@
+package org.embulk.exec;
+
+import java.util.List;
+import java.util.Set;
+import java.util.ArrayList;
+import java.util.concurrent.ExecutionException;
+import com.google.common.collect.ImmutableList;
+import com.google.common.base.Throwables;
+import com.google.inject.Inject;
+import com.google.inject.Binder;
+import com.google.inject.multibindings.Multibinder;
+import org.embulk.plugin.PluginType;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.spi.Schema;
+import org.embulk.spi.Page;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.GuessPlugin;
+import org.embulk.spi.Exec;
+import org.embulk.spi.ExecAction;
+import org.embulk.spi.ExecSession;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.TransactionalFileInput;
+import org.embulk.spi.FileInputRunner;
+
+public class GuessExecutor
+{
+    private final List<PluginType> defaultGuessPlugins;
+
+    private interface GuessExecutorSystemTask
+            extends Task
+    {
+        @Config("guess_plugins")
+        @ConfigDefault("[]")
+        public List<PluginType> getGuessPlugins();
+    }
+
+    private interface GuessExecutorTask
+            extends Task
+    {
+        @Config("guess_plugins")
+        @ConfigDefault("[]")
+        public List<PluginType> getGuessPlugins();
+
+        @Config("exclude_guess_plugins")
+        @ConfigDefault("[]")
+        public List<PluginType> getExcludeGuessPlugins();
+    }
+
+    public static void registerDefaultGuessPluginTo(Binder binder, PluginType type)
+    {
+        Multibinder<PluginType> multibinder = Multibinder.newSetBinder(binder, PluginType.class, ForGuess.class);
+        multibinder.addBinding().toInstance(type);
+    }
+
+    @Inject
+    public GuessExecutor(@ForSystemConfig ConfigSource systemConfig,
+            @ForGuess Set<PluginType> defaultGuessPlugins)
+    {
+        GuessExecutorSystemTask systemTask = systemConfig.loadConfig(GuessExecutorSystemTask.class);
+
+        ImmutableList.Builder<PluginType> list = ImmutableList.builder();
+        list.addAll(defaultGuessPlugins);
+        list.addAll(systemTask.getGuessPlugins());
+        this.defaultGuessPlugins = list.build();
+    }
+
+    public ConfigDiff guess(ExecSession exec, final ConfigSource config)
+    {
+        try {
+            return Exec.doWith(exec, new ExecAction<ConfigDiff>() {
+                public ConfigDiff run()
+                {
+                    try (SetCurrentThreadName dontCare = new SetCurrentThreadName("guess")) {
+                        return doGuess(config);
+                    }
+                }
+            });
+        } catch (ExecutionException ex) {
+            throw Throwables.propagate(ex.getCause());
+        }
+    }
+
+    protected InputPlugin newInputPlugin(ConfigSource inputConfig)
+    {
+        return Exec.newPlugin(InputPlugin.class, inputConfig.get(PluginType.class, "type"));
+    }
+
+    private ConfigDiff doGuess(ConfigSource config)
+    {
+        ConfigSource inputConfig = config.getNested("in");
+
+        InputPlugin input = newInputPlugin(inputConfig);
+
+        ConfigDiff inputGuessed;
+        try {
+            inputGuessed = input.guess(inputConfig);
+        } catch (AbstractMethodError ex) {
+            // for backward compatibility with embulk v0.4 interface
+            throw new UnsupportedOperationException(input.getClass().getSimpleName()+".guess(ConfigSource) is not implemented. This input plugin does not support guessing.");
+        }
+
+        ConfigDiff wrapped = Exec.newConfigDiff();
+        wrapped.getNestedOrSetEmpty("in").merge(inputGuessed);
+        return wrapped;
+    }
+
+    // called by FileInputRunner
+    public ConfigDiff guessParserConfig(Buffer sample, ConfigSource inputConfig, ConfigSource execConfig)
+    {
+        List<PluginType> guessPlugins = new ArrayList<PluginType>(defaultGuessPlugins);
+
+        GuessExecutorTask task = execConfig.loadConfig(GuessExecutorTask.class);
+        guessPlugins.addAll(task.getGuessPlugins());
+        guessPlugins.removeAll(task.getExcludeGuessPlugins());
+
+        return guessParserConfig(sample, inputConfig, guessPlugins);
+    }
+
+    private ConfigDiff guessParserConfig(Buffer sample,
+            ConfigSource config, List<PluginType> guessPlugins)
+    {
+        // repeat guessing upto 10 times
+        ConfigDiff lastGuessed = Exec.newConfigDiff();
+        for (int i=0; i < 10; i++) {
+            // include last-guessed config to run guess input
+            ConfigSource originalConfig = config.deepCopy().merge(lastGuessed);
+            ConfigSource guessInputConfig = originalConfig.deepCopy();
+            guessInputConfig.getNestedOrSetEmpty("parser")
+                .set("type", "system_guess")  // override in.parser.type so that FileInputRunner.run uses GuessParserPlugin
+                .set("guess_plugins", guessPlugins)
+                .set("orig_config", originalConfig);
+
+            // run FileInputPlugin
+            final FileInputRunner input = new FileInputRunner(new BufferFileInputPlugin(sample));
+            ConfigDiff guessed;
+            try {
+                input.transaction(guessInputConfig, new InputPlugin.Control() {
+                    public List<TaskReport> run(TaskSource inputTaskSource, Schema schema, int taskCount)
+                    {
+                        if (taskCount == 0) {
+                            throw new NoSampleException("No input files to guess");
+                        }
+                        input.run(inputTaskSource, null, 0, new PageOutput() {
+                            @Override
+                            public void add(Page page)
+                            {
+                                throw new RuntimeException("Input plugin must be a FileInputPlugin to guess parser configuration");  // TODO exception class
+                            }
+
+                            @Override
+                            public void finish() { }
+
+                            @Override
+                            public void close() { }
+                        });
+                        throw new AssertionError("Guess executor must throw GuessedNoticeError");
+                    }
+                });
+
+                throw new AssertionError("Guess executor must throw GuessedNoticeError");
+
+            } catch (GuessedNoticeError error) {
+                guessed = lastGuessed.deepCopy().merge(error.getGuessedConfig());
+            }
+
+            // merge to the last-guessed config
+            if (lastGuessed.equals(guessed)) {
+                // not changed
+                return lastGuessed;
+            }
+            lastGuessed = guessed;
+        }
+
+        return lastGuessed;
+    }
+
+    private static class BufferFileInputPlugin
+            implements FileInputPlugin
+    {
+        private Buffer buffer;
+
+        public BufferFileInputPlugin(Buffer buffer)
+        {
+            this.buffer = buffer;
+        }
+
+        public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
+        {
+            control.run(Exec.newTaskSource(), 1);
+            return Exec.newConfigDiff();
+        }
+
+        public ConfigDiff resume(TaskSource taskSource,
+                int taskCount,
+                FileInputPlugin.Control control)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public void cleanup(TaskSource taskSource,
+                int taskCount,
+                List<TaskReport> successTaskReports)
+        {
+            if (buffer != null) {
+                buffer.release();
+                buffer = null;
+            }
+        }
+
+        public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
+        {
+            return new BufferTransactionalFileInput(buffer);
+        }
+    }
+
+    private static class BufferTransactionalFileInput
+            implements TransactionalFileInput
+    {
+        private Buffer buffer;
+
+        public BufferTransactionalFileInput(Buffer buffer)
+        {
+            this.buffer = buffer;
+        }
+
+        @Override
+        public Buffer poll()
+        {
+            Buffer b = buffer;
+            buffer = null;
+            return b;
+        }
+
+        @Override
+        public boolean nextFile()
+        {
+            return buffer != null;
+        }
+
+        @Override
+        public void close() { }
+
+        @Override
+        public void abort() { }
+
+        @Override
+        public TaskReport commit()
+        {
+            return null;
+        }
+    }
+
+    public static class GuessParserPlugin
+            implements ParserPlugin
+    {
+        private interface PluginTask
+                extends Task
+        {
+            @Config("guess_plugins")
+            public List<PluginType> getGuessPluginTypes();
+
+            @Config("orig_config")
+            public ConfigSource getOriginalConfig();
+        }
+
+        @Override
+        public void transaction(ConfigSource config, ParserPlugin.Control control)
+        {
+            PluginTask task = config.loadConfig(PluginTask.class);
+            control.run(task.dump(), null);
+        }
+
+        @Override
+        public void run(TaskSource taskSource, Schema schema,
+                FileInput input, PageOutput pageOutput)
+        {
+            PluginTask task = taskSource.loadTask(PluginTask.class);
+            final ConfigSource originalConfig = task.getOriginalConfig();
+
+            // get sample buffer
+            Buffer sample = getFirstBuffer(input);
+
+            // load guess plugins
+            ImmutableList.Builder<GuessPlugin> builder = ImmutableList.builder();
+            for (PluginType guessType : task.getGuessPluginTypes()) {
+                GuessPlugin guess = Exec.newPlugin(GuessPlugin.class, guessType);
+                builder.add(guess);
+            }
+            List<GuessPlugin> guesses = builder.build();
+
+            // run guess plugins
+            ConfigSource mergedConfig = originalConfig.deepCopy();
+            ConfigDiff mergedGuessed = Exec.newConfigDiff();
+            for (int i=0; i < guesses.size(); i++) {
+                ConfigDiff guessed = guesses.get(i).guess(originalConfig, sample);
+                guessed = addAssumedDecoderConfigs(originalConfig, guessed);
+                mergedGuessed.merge(guessed);
+                mergedConfig.merge(mergedGuessed);
+                if (!mergedConfig.equals(originalConfig)) {
+                    // config updated
+                    throw new GuessedNoticeError(mergedGuessed);
+                }
+            }
+            throw new GuessedNoticeError(mergedGuessed);
+        }
+
+        private static Buffer getFirstBuffer(FileInput input)
+        {
+            // The first buffer is created by SamplingParserPlugin. See FileInputRunner.guess.
+            RuntimeException decodeException = null;
+            try {
+                while (input.nextFile()) {
+                    Buffer sample = input.poll();
+                    if (sample != null) {
+                        return sample;
+                    }
+                }
+            } catch (RuntimeException ex) {
+                // ignores exceptions because FileDecoderPlugin can throw exceptions
+                // such as "Unexpected end of ZLIB input stream"
+                decodeException = ex;
+            }
+            if (decodeException != null) {
+                throw decodeException;
+            }
+            throw new NoSampleException("No input buffer to guess");
+        }
+
+        private static class ConfigSourceList extends ArrayList<ConfigSource> { };
+
+        private static ConfigDiff addAssumedDecoderConfigs(ConfigSource originalConfig, ConfigDiff guessed)
+        {
+            List<ConfigSource> guessedDecoders = guessed.get(ConfigSourceList.class, "decoders", null);
+            if (guessedDecoders == null) {
+                return guessed;
+            } else {
+                List<ConfigSource> assumedDecoders = originalConfig.get(ConfigSourceList.class, "decoders", new ConfigSourceList());
+                ImmutableList.Builder<ConfigSource> added = ImmutableList.builder();
+                for (ConfigSource assuemed : assumedDecoders) {
+                    added.add(Exec.newConfigSource());
+                }
+                added.addAll(guessedDecoders);
+                return guessed.set("decoders", added.build());
+            }
+        }
+    }
+
+    public static class GuessedNoticeError
+            extends Error
+    {
+        private final ConfigDiff guessedConfig;
+
+        public GuessedNoticeError(ConfigDiff guessedConfig)
+        {
+            this.guessedConfig = guessedConfig;
+        }
+
+        public ConfigDiff getGuessedConfig()
+        {
+            return guessedConfig;
+        }
+    }
+}