embulk 0.1.0

data/embulk-core/src/main/java/org/embulk/exec/ExecuteInterruptedException.java

@@ -0,0 +1,10 @@
+package org.embulk.exec;
+
+public class ExecuteInterruptedException
+        extends RuntimeException
+{
+    public ExecuteInterruptedException(Exception cause)
+    {
+        super(cause);
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/ExecuteResult.java

@@ -0,0 +1,19 @@
+package org.embulk.exec;
+
+import java.util.List;
+import org.embulk.config.NextConfig;
+
+public class ExecuteResult
+{
+    private final NextConfig nextConfig;
+
+    public ExecuteResult(NextConfig nextConfig)
+    {
+        this.nextConfig = nextConfig;
+    }
+
+    public NextConfig getNextConfig()
+    {
+        return nextConfig;
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/ExtensionServiceLoaderModule.java

@@ -0,0 +1,43 @@
+package org.embulk.exec;
+
+import java.util.ServiceLoader;
+import com.google.inject.Module;
+import com.google.inject.Binder;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Extension;
+
+/**
+ * ExtensionServiceLoaderModule loads Extensions using java.util.ServiceLoader
+ * mechanism.
+ * Jar packages providing an extension need to include
+ * META-INF/services/org.embulk.exec.Extension file. Contents of the file is
+ * one-line text of the extension class name (e.g. com.example.MyPluginSourceExtension).
+ */
+public class ExtensionServiceLoaderModule
+        implements Module
+{
+    private final ClassLoader classLoader;
+    private final ConfigSource systemConfig;
+
+    public ExtensionServiceLoaderModule(ConfigSource systemConfig)
+    {
+        this(ExtensionServiceLoaderModule.class.getClassLoader(), systemConfig);
+    }
+
+    public ExtensionServiceLoaderModule(ClassLoader classLoader, ConfigSource systemConfig)
+    {
+        this.classLoader = classLoader;
+        this.systemConfig = systemConfig;
+    }
+
+    @Override
+    public void configure(Binder binder)
+    {
+        ServiceLoader<Extension> serviceLoader = ServiceLoader.load(Extension.class, classLoader);
+        for (Extension extension : serviceLoader) {
+            for (Module module : extension.getModules(systemConfig)) {
+                module.configure(binder);
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/ForSystemConfig.java

@@ -0,0 +1,16 @@
+package org.embulk.exec;
+
+import javax.inject.Qualifier;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+@Retention(RUNTIME)
+@Target({FIELD, PARAMETER, METHOD})
+@Qualifier
+public @interface ForSystemConfig
+{
+}

data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java

@@ -0,0 +1,307 @@
+package org.embulk.exec;
+
+import java.util.List;
+import java.util.ArrayList;
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.common.base.Throwables;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.NextConfig;
+import org.embulk.config.DataSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.CommitReport;
+import org.embulk.plugin.PluginType;
+import org.embulk.spi.Schema;
+import org.embulk.spi.Column;
+import org.embulk.spi.Page;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.GuessPlugin;
+import org.embulk.spi.Exec;
+import org.embulk.spi.ExecAction;
+import org.embulk.spi.ExecSession;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.TransactionalFileInput;
+import org.embulk.spi.FileInputRunner;
+
+public class GuessExecutor
+{
+    private final Injector injector;
+    private final ConfigSource systemConfig;
+    private final List<PluginType> defaultGuessPlugins;
+
+    private interface GuessExecutorTask
+            extends Task
+    {
+        @Config("guess_plugins")
+        @ConfigDefault("[]")
+        public List<PluginType> getGuessPlugins();
+
+        @Config("exclude_guess_plugins")
+        @ConfigDefault("[]")
+        public List<PluginType> getExcludeGuessPlugins();
+    }
+
+    @Inject
+    public GuessExecutor(Injector injector,
+            @ForSystemConfig ConfigSource systemConfig)
+    {
+        this.injector = injector;
+        this.systemConfig = systemConfig;
+
+        // TODO get default guess plugins from injector using Multibinder
+        this.defaultGuessPlugins = ImmutableList.of(
+                new PluginType("gzip"),
+                new PluginType("charset"),
+                new PluginType("newline"),
+                new PluginType("csv"));
+    }
+
+    public NextConfig guess(ExecSession exec, final ConfigSource config)
+    {
+        try {
+            return Exec.doWith(exec, new ExecAction<NextConfig>() {
+                public NextConfig run()
+                {
+                    return doGuess(config);
+                }
+            });
+        } catch (Exception ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    private NextConfig doGuess(ConfigSource config)
+    {
+        Buffer sample = SamplingParserPlugin.runFileInputSampling(config);
+        if (sample.limit() == 0) {
+            throw new NoSampleException("Can't get sample data because the first input file is empty");
+        }
+
+        List<PluginType> guessPlugins = new ArrayList<PluginType>(defaultGuessPlugins);
+        GuessExecutorTask task = config.getNestedOrSetEmpty("exec").loadConfig(GuessExecutorTask.class);
+        guessPlugins.addAll(task.getGuessPlugins());
+        guessPlugins.removeAll(task.getExcludeGuessPlugins());
+
+        return runGuessInput(sample, config, guessPlugins);
+    }
+
+    private NextConfig runGuessInput(Buffer sample,
+            ConfigSource config, List<PluginType> guessPlugins)
+    {
+        // repeat guessing upto 10 times
+        NextConfig lastGuessed = Exec.newNextConfig();
+        for (int i=0; i < 10; i++) {
+            // include last-guessed config to run guess input
+            ConfigSource originalConfig = config.getNested("in").deepCopy().merge(lastGuessed);
+            ConfigSource guessInputConfig = originalConfig.deepCopy();
+            guessInputConfig.getNestedOrSetEmpty("parser")
+                .set("type", "system_guess")  // override in.parser.type so that FileInputPlugin creates GuessParserPlugin
+                .set("guess_plugins", guessPlugins)
+                .set("orig_config", originalConfig);
+
+            // run FileInputPlugin
+            final FileInputRunner input = new FileInputRunner(new BufferFileInputPlugin(sample));
+            NextConfig guessed;
+            try {
+                input.transaction(guessInputConfig, new InputPlugin.Control() {
+                    public List<CommitReport> run(TaskSource inputTaskSource, Schema schema, int processorCount)
+                    {
+                        // TODO repeat runwith processorIndex++ if NoSampleException happens
+                        input.run(inputTaskSource, null, 0, new PageOutput() {
+                            @Override
+                            public void add(Page page)
+                            {
+                                throw new RuntimeException("Input plugin must be a FileInputPlugin to guess parser configuration");  // TODO exception class
+                            }
+
+                            @Override
+                            public void finish() { }
+
+                            @Override
+                            public void close() { }
+                        });
+                        throw new AssertionError("Guess executor must throw GuessedNoticeError");
+                    }
+                });
+                throw new AssertionError("Guess executor must throw GuessedNoticeError");
+
+            } catch (GuessedNoticeError error) {
+                guessed = lastGuessed.deepCopy().merge(error.getGuessedConfig());
+            }
+
+            // merge to the last-guessed config
+            if (lastGuessed.equals(guessed)) {
+                // not changed
+                return wrapInIn(lastGuessed);
+            }
+            lastGuessed = guessed;
+        }
+
+        return wrapInIn(lastGuessed);
+    }
+
+    private static NextConfig wrapInIn(NextConfig lastGuessed)
+    {
+        NextConfig wrapped = Exec.newNextConfig();
+        wrapped.getNestedOrSetEmpty("in").merge(lastGuessed);
+        return wrapped;
+    }
+
+    private static class BufferFileInputPlugin
+            implements FileInputPlugin
+    {
+        private final Buffer buffer;
+
+        public BufferFileInputPlugin(Buffer buffer)
+        {
+            this.buffer = buffer;
+        }
+
+        public NextConfig transaction(ConfigSource config, FileInputPlugin.Control control)
+        {
+            control.run(Exec.newTaskSource(), 1);
+            return Exec.newNextConfig();
+        }
+
+        public TransactionalFileInput open(TaskSource taskSource, int processorIndex)
+        {
+            return new BufferTransactionalFileInput(buffer);
+        }
+    }
+
+    private static class BufferTransactionalFileInput
+            implements TransactionalFileInput
+    {
+        private Buffer buffer;
+
+        public BufferTransactionalFileInput(Buffer buffer)
+        {
+            this.buffer = buffer;
+        }
+
+        @Override
+        public Buffer poll()
+        {
+            Buffer b = buffer;
+            buffer = null;
+            return b;
+        }
+
+        @Override
+        public boolean nextFile()
+        {
+            return buffer != null;
+        }
+
+        @Override
+        public void close() { }
+
+        @Override
+        public void abort() { }
+
+        @Override
+        public CommitReport commit()
+        {
+            return null;
+        }
+    }
+
+    public static class GuessParserPlugin
+            implements ParserPlugin
+    {
+        private interface PluginTask
+                extends Task
+        {
+            @Config("guess_plugins")
+            public List<PluginType> getGuessPluginTypes();
+
+            @Config("orig_config")
+            public ConfigSource getOriginalConfig();
+        }
+
+        @Override
+        public void transaction(ConfigSource config, ParserPlugin.Control control)
+        {
+            PluginTask task = config.loadConfig(PluginTask.class);
+            control.run(task.dump(), null);
+        }
+
+        @Override
+        public void run(TaskSource taskSource, Schema schema,
+                FileInput input, PageOutput pageOutput)
+        {
+            PluginTask task = taskSource.loadTask(PluginTask.class);
+            final ConfigSource originalConfig = task.getOriginalConfig();
+
+            // get sample buffer
+            Buffer sample = getFirstBuffer(input);
+
+            // load guess plugins
+            ImmutableList.Builder<GuessPlugin> builder = ImmutableList.builder();
+            for (PluginType guessType : task.getGuessPluginTypes()) {
+                GuessPlugin guess = Exec.newPlugin(GuessPlugin.class, guessType);
+                builder.add(guess);
+            }
+            List<GuessPlugin> guesses = builder.build();
+
+            // run guess plugins
+            ConfigSource mergedConfig = originalConfig.deepCopy();
+            NextConfig mergedGuessed = Exec.newNextConfig();
+            for (int i=0; i < guesses.size(); i++) {
+                NextConfig guessed = guesses.get(i).guess(originalConfig, sample);
+                mergedGuessed.merge(guessed);
+                mergedConfig.merge(mergedGuessed);
+                if (!mergedConfig.equals(originalConfig)) {
+                    // config updated
+                    throw new GuessedNoticeError(mergedGuessed);
+                }
+            }
+            throw new GuessedNoticeError(mergedGuessed);
+        }
+
+        private static Buffer getFirstBuffer(FileInput input)
+        {
+            RuntimeException decodeException = null;
+            try {
+                while (input.nextFile()) {
+                    Buffer sample = input.poll();
+                    if (sample != null) {
+                        return sample;
+                    }
+                }
+            } catch (RuntimeException ex) {
+                // ignores exceptions because FileDecoderPlugin can throw exceptions
+                // such as "Unexpected end of ZLIB input stream"
+                decodeException = ex;
+            }
+            if (decodeException != null) {
+                throw decodeException;
+            }
+            throw new NoSampleException("No input buffer to guess");
+        }
+    }
+
+    public static class GuessedNoticeError
+            extends Error
+    {
+        private final NextConfig guessedConfig;
+
+        public GuessedNoticeError(NextConfig guessedConfig)
+        {
+            this.guessedConfig = guessedConfig;
+        }
+
+        public NextConfig getGuessedConfig()
+        {
+            return guessedConfig;
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/LocalExecutor.java

@@ -0,0 +1,274 @@
+package org.embulk.exec;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Future;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.common.base.Throwables;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.embulk.config.Task;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
+import org.embulk.config.NextConfig;
+import org.embulk.config.CommitReport;
+import org.embulk.plugin.PluginType;
+import org.embulk.spi.Schema;
+import org.embulk.spi.Exec;
+import org.embulk.spi.ExecSession;
+import org.embulk.spi.ExecAction;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.TransactionalPageOutput;
+import org.slf4j.Logger;
+
+public class LocalExecutor
+{
+    private final Injector injector;
+    private final ConfigSource systemConfig;
+    private final int maxThreads;
+    private final ExecutorService executor;
+
+    private Logger log;
+    private final AtomicInteger runningTaskCount;
+    private final AtomicInteger completedTaskCount;
+
+    public interface ExecutorTask
+            extends Task
+    {
+        @Config("in")
+        public ConfigSource getInputConfig();
+
+        @Config("out")
+        public ConfigSource getOutputConfig();
+
+        public TaskSource getInputTask();
+        public void setInputTask(TaskSource taskSource);
+
+        public TaskSource getOutputTask();
+        public void setOutputTask(TaskSource taskSource);
+    }
+
+    @Inject
+    public LocalExecutor(Injector injector,
+            @ForSystemConfig ConfigSource systemConfig)
+    {
+        this.injector = injector;
+        this.systemConfig = systemConfig;
+
+        int defaultMaxThreads = Runtime.getRuntime().availableProcessors() * 2;
+        this.maxThreads = systemConfig.get(Integer.class, "max_threads", defaultMaxThreads);
+        this.executor = Executors.newFixedThreadPool(maxThreads,
+                new ThreadFactoryBuilder()
+                        .setNameFormat("embulk-executor-%d")
+                        .setDaemon(true)
+                        .build());
+
+        this.runningTaskCount = new AtomicInteger(0);
+        this.completedTaskCount = new AtomicInteger(0);
+    }
+
+    private static class ExecuteResultBuilder
+    {
+        private NextConfig inputNextConfig;
+        private NextConfig outputNextConfig;
+
+        public void setInputNextConfig(NextConfig inputNextConfig)
+        {
+            this.inputNextConfig = inputNextConfig;
+        }
+
+        public void setOutputNextConfig(NextConfig outputNextConfig)
+        {
+            this.outputNextConfig = outputNextConfig;
+        }
+
+        public NextConfig getInputNextConfig()
+        {
+            return inputNextConfig;
+        }
+
+        public NextConfig getOutputNextConfig()
+        {
+            return outputNextConfig;
+        }
+
+        public ExecuteResult build()
+        {
+            if (inputNextConfig == null) {
+                inputNextConfig = Exec.newNextConfig();
+            }
+            if (outputNextConfig == null) {
+                outputNextConfig = Exec.newNextConfig();
+            }
+            NextConfig nextConfig = inputNextConfig.deepCopy().merge(outputNextConfig);
+            return new ExecuteResult(nextConfig);
+        }
+    }
+
+    private static class ProcessResult
+    {
+        private final CommitReport inputCommitReport;
+        private final CommitReport outputCommitReport;
+
+        public ProcessResult(CommitReport inputCommitReport, CommitReport outputCommitReport)
+        {
+            this.inputCommitReport = inputCommitReport;
+            this.outputCommitReport = outputCommitReport;
+        }
+
+        public CommitReport getInputCommitReport()
+        {
+            return inputCommitReport;
+        }
+
+        public CommitReport getOutputCommitReport()
+        {
+            return outputCommitReport;
+        }
+    }
+
+    protected InputPlugin newInputPlugin(ExecutorTask task)
+    {
+        return Exec.newPlugin(InputPlugin.class, task.getInputConfig().get(PluginType.class, "type"));
+    }
+
+    protected OutputPlugin newOutputPlugin(ExecutorTask task)
+    {
+        return Exec.newPlugin(OutputPlugin.class, task.getOutputConfig().get(PluginType.class, "type"));
+    }
+
+    public ExecuteResult run(ExecSession exec, final ConfigSource config)
+    {
+        log = exec.getLogger(LocalExecutor.class);
+        try {
+            return Exec.doWith(exec, new ExecAction<ExecuteResult>() {
+                public ExecuteResult run()
+                {
+                    return doRun(config);
+                }
+            });
+        } catch (Exception ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    private ExecuteResult doRun(ConfigSource config)
+    {
+        final ExecutorTask task = config.loadConfig(ExecutorTask.class);
+
+        final InputPlugin in = newInputPlugin(task);
+        final OutputPlugin out = newOutputPlugin(task);
+
+        final ExecuteResultBuilder execResult = new ExecuteResultBuilder();
+
+        NextConfig inputNextConfig = in.transaction(task.getInputConfig(), new InputPlugin.Control() {
+            public List<CommitReport> run(final TaskSource inputTask, final Schema schema, final int processorCount)
+            {
+                final ImmutableList.Builder<CommitReport> inputCommitReports = ImmutableList.builder();
+                NextConfig outputNextConfig = out.transaction(task.getOutputConfig(), schema, processorCount, new OutputPlugin.Control() {
+                    public List<CommitReport> run(final TaskSource outputTask)
+                    {
+                        final ImmutableList.Builder<CommitReport> outputCommitReports = ImmutableList.builder();
+                        task.setInputTask(inputTask);
+                        task.setOutputTask(outputTask);
+
+                        //log.debug("input: %s", task.getInputTask());
+                        //log.debug("output: %s", task.getOutputTask());
+
+                        List<ProcessResult> results = process(task.dump(), schema, processorCount);
+                        for (ProcessResult result : results) {
+                            inputCommitReports.add(result.getInputCommitReport());
+                            outputCommitReports.add(result.getOutputCommitReport());
+                        }
+
+                        return outputCommitReports.build();
+                    }
+                });
+                execResult.setOutputNextConfig(outputNextConfig);
+                return inputCommitReports.build();
+            }
+        });
+        execResult.setInputNextConfig(inputNextConfig);
+
+        return execResult.build();
+    }
+
+    private List<ProcessResult> process(TaskSource taskSource, Schema schema, int processorCount)
+    {
+        List<Future<ProcessResult>> futures = new ArrayList<>();
+        List<ProcessResult> joined = new ArrayList<>();
+        try {
+            log.info("Running {} tasks using {} local threads", processorCount, maxThreads);
+            showProgress(processorCount);
+            for (int i=0; i < processorCount; i++) {
+                futures.add(startProcessor(taskSource, schema, i));
+            }
+
+            for (int i=0; i < processorCount; i++) {
+                try {
+                    joined.add(futures.get(i).get());
+                    showProgress(processorCount);
+
+                } catch (ExecutionException ex) {
+                    throw Throwables.propagate(ex.getCause());
+                } catch (InterruptedException ex) {
+                    throw new ExecuteInterruptedException(ex);
+                }
+            }
+            return joined;
+        } finally {
+            for (int i=joined.size(); i < futures.size(); i++) {
+                futures.get(i).cancel(true);
+                // TODO join?
+            }
+        }
+    }
+
+    private void showProgress(int total)
+    {
+        int running = runningTaskCount.get();
+        int done = completedTaskCount.get();
+        log.info(String.format("{done:%3d / %d, running: %d}", done, total, running));
+    }
+
+    private Future<ProcessResult> startProcessor(final TaskSource taskSource, final Schema schema, final int index)
+    {
+        return executor.submit(new Callable<ProcessResult>() {
+            public ProcessResult call()
+            {
+                try {
+                    runningTaskCount.getAndIncrement();
+                    final ExecutorTask task = taskSource.loadTask(ExecutorTask.class);
+                    final InputPlugin in = newInputPlugin(task);
+                    final OutputPlugin out = newOutputPlugin(task);
+
+                    TransactionalPageOutput tran = out.open(task.getOutputTask(), schema, index);
+                    boolean committed = false;
+                    try {
+                        CommitReport inReport = in.run(task.getInputTask(), schema, index, tran);
+                        CommitReport outReport = tran.commit();  // TODO check output.finish() is called. wrap or abstract
+                        committed = true;
+                        return new ProcessResult(inReport, outReport);
+                    } finally {
+                        if (!committed) {
+                            tran.abort();
+                        }
+                        tran.close();
+                    }
+                } finally {
+                    runningTaskCount.getAndDecrement();
+                    completedTaskCount.getAndIncrement();
+                }
+            }
+        });
+    }
+}