embulk 0.7.0-java

data/embulk-core/src/main/java/org/embulk/exec/LocalExecutorPlugin.java

@@ -0,0 +1,129 @@
+package org.embulk.exec;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Future;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ExecutionException;
+import org.slf4j.Logger;
+import com.google.inject.Inject;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.spi.Exec;
+import org.embulk.spi.ExecutorPlugin;
+import org.embulk.spi.ProcessTask;
+import org.embulk.spi.ProcessState;
+import org.embulk.spi.Schema;
+import org.embulk.spi.util.Executors;
+import org.embulk.spi.util.Executors.ProcessStateCallback;
+
+public class LocalExecutorPlugin
+        implements ExecutorPlugin
+{
+    private final ExecutorService executor;
+
+    @Inject
+    public LocalExecutorPlugin(LocalThreadExecutor executor)
+    {
+        this.executor = executor.getExecutorService();
+    }
+
+    @Override
+    public void transaction(ConfigSource config, Schema outputSchema, final int inputTaskCount,
+            ExecutorPlugin.Control control)
+    {
+        control.transaction(outputSchema, inputTaskCount, new Executor() {
+            public void execute(ProcessTask task, ProcessState state)
+            {
+                localExecute(task, inputTaskCount, state);
+            }
+        });
+    }
+
+    private void localExecute(ProcessTask task, int taskCount, ProcessState state)
+    {
+        Logger log = Exec.getLogger(LocalExecutorPlugin.class);
+
+        state.initialize(taskCount, taskCount);
+
+        List<Future<Throwable>> futures = new ArrayList<>(taskCount);
+        try {
+            for (int i=0; i < taskCount; i++) {
+                if (state.getOutputTaskState(i).isCommitted()) {
+                    log.warn("Skipped resumed task {}", i);
+                    futures.add(null);  // resumed
+                } else {
+                    futures.add(startProcessor(task, i, state));
+                }
+            }
+            showProgress(log, state, taskCount);
+
+            for (int i=0; i < taskCount; i++) {
+                if (futures.get(i) == null) {
+                    continue;
+                }
+                try {
+                    state.getInputTaskState(i).setException(futures.get(i).get());
+                } catch (ExecutionException ex) {
+                    state.getInputTaskState(i).setException(ex.getCause());
+                    //Throwables.propagate(ex.getCause());
+                } catch (InterruptedException ex) {
+                    state.getInputTaskState(i).setException(new ExecutionInterruptedException(ex));
+                }
+                showProgress(log, state, taskCount);
+            }
+        } finally {
+            for (Future<Throwable> future : futures) {
+                if (future != null && !future.isDone()) {
+                    future.cancel(true);
+                    // TODO join?
+                }
+            }
+        }
+    }
+
+    private void showProgress(Logger log, ProcessState state, int taskCount)
+    {
+        int started = 0;
+        int finished = 0;
+        for (int i=0; i < taskCount; i++) {
+            if (state.getInputTaskState(i).isStarted()) { started++; }
+            if (state.getOutputTaskState(i).isFinished()) { finished++; }
+        }
+
+        log.info(String.format("{done:%3d / %d, running: %d}", finished, taskCount, started - finished));
+    }
+
+    private Future<Throwable> startProcessor(final ProcessTask task, final int taskIndex, final ProcessState state)
+    {
+        return executor.submit(new Callable<Throwable>() {
+            public Throwable call()
+            {
+                try (SetCurrentThreadName dontCare = new SetCurrentThreadName(String.format("task-%04d", taskIndex))) {
+                    Executors.process(Exec.session(), task, taskIndex, new ProcessStateCallback() {
+                        public void started()
+                        {
+                            state.getInputTaskState(taskIndex).start();
+                            state.getOutputTaskState(taskIndex).start();
+                        }
+
+                        public void inputCommitted(TaskReport report)
+                        {
+                            state.getInputTaskState(taskIndex).setTaskReport(report);
+                        }
+
+                        public void outputCommitted(TaskReport report)
+                        {
+                            state.getOutputTaskState(taskIndex).setTaskReport(report);
+                        }
+                    });
+                    return null;
+                } finally {
+                    state.getInputTaskState(taskIndex).finish();
+                    state.getOutputTaskState(taskIndex).finish();
+                }
+            }
+        });
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/LocalThreadExecutor.java

@@ -0,0 +1,34 @@
+package org.embulk.exec;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.inject.Inject;
+import org.embulk.config.ConfigSource;
+
+/*
+ * Injected in SINGLETON scope at ExecModule
+ */
+public class LocalThreadExecutor
+{
+    private final ExecutorService executor;
+
+    @Inject
+    public LocalThreadExecutor(@ForSystemConfig ConfigSource systemConfig)
+    {
+        int defaultMaxThreads = Runtime.getRuntime().availableProcessors() * 2;
+        int maxThreads = systemConfig.get(Integer.class, "max_threads", defaultMaxThreads);
+        this.executor = Executors.newFixedThreadPool(maxThreads,
+                new ThreadFactoryBuilder()
+                        .setNameFormat("embulk-executor-%d")
+                        .setDaemon(true)
+                        .build());
+    }
+
+    public ExecutorService getExecutorService()
+    {
+        return executor;
+    }
+
+    // TODO shutdown
+}

data/embulk-core/src/main/java/org/embulk/exec/LoggerProvider.java

@@ -0,0 +1,60 @@
+package org.embulk.exec;
+
+import org.slf4j.ILoggerFactory;
+import org.slf4j.LoggerFactory;
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.joran.JoranConfigurator;
+import ch.qos.logback.core.joran.spi.JoranException;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import org.embulk.config.ConfigSource;
+
+public class LoggerProvider
+        implements Provider<ILoggerFactory>
+{
+    @Inject
+    public LoggerProvider(@ForSystemConfig ConfigSource systemConfig)
+    {
+        final String level;
+        String logLevel = systemConfig.get(String.class, "log_level", "info");  // here can't use loadConfig because ModelManager uses LoggerProvider
+        switch (logLevel) {
+        case "error": level = "ERROR"; break;
+        case "warn":  level = "WARN";  break;
+        case "info":  level = "INFO";  break;
+        case "debug": level = "DEBUG"; break;
+        case "trace": level = "TRACE"; break;
+        default:
+            throw new IllegalArgumentException(String.format(
+                        "System property embulk.logLevel=%s is invalid. Available levels are error, warn, info, debug and trace.", logLevel));
+        }
+
+        LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
+        JoranConfigurator configurator = new JoranConfigurator();
+        configurator.setContext(context);
+        context.reset();
+
+        String name;
+        if (System.console() != null) {
+            name = "/embulk/logback-color.xml";
+        } else {
+            name = "/embulk/logback-console.xml";
+        }
+        try {
+            configurator.doConfigure(getClass().getResource(name));
+        } catch (JoranException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        org.slf4j.Logger logger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
+        if (logger instanceof Logger) {
+            ((Logger) logger).setLevel(Level.toLevel(level.toUpperCase(), Level.DEBUG));
+        }
+    }
+
+    public ILoggerFactory get()
+    {
+        return LoggerFactory.getILoggerFactory();
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/NoSampleException.java

@@ -0,0 +1,10 @@
+package org.embulk.exec;
+
+public class NoSampleException
+        extends RuntimeException
+{
+    public NoSampleException(String message)
+    {
+        super(message);
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/PartialExecutionException.java

@@ -0,0 +1,18 @@
+package org.embulk.exec;
+
+public class PartialExecutionException
+        extends RuntimeException
+{
+    private final ResumeState resumeState;
+
+    public PartialExecutionException(Throwable cause, ResumeState resumeState)
+    {
+        super(cause);
+        this.resumeState = resumeState;
+    }
+
+    public ResumeState getResumeState()
+    {
+        return resumeState;
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/PooledBufferAllocator.java

@@ -0,0 +1,77 @@
+package org.embulk.exec;
+
+import io.netty.buffer.PooledByteBufAllocator;
+import io.netty.buffer.ByteBuf;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.BufferAllocator;
+import com.google.inject.Inject;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.unit.ByteSize;
+
+public class PooledBufferAllocator
+        implements BufferAllocator
+{
+    private static final int DEFAULT_PAGE_SIZE = 32*1024;
+
+    private final PooledByteBufAllocator nettyBuffer;
+    private final int pageSize;
+
+    @Inject
+    public PooledBufferAllocator(@ForSystemConfig ConfigSource systemConfig, org.slf4j.ILoggerFactory factory)
+    {
+        this.nettyBuffer = new PooledByteBufAllocator(false);
+        this.pageSize = systemConfig.get(ByteSize.class, "page_size", new ByteSize(DEFAULT_PAGE_SIZE)).getBytesInt();
+    }
+
+    public Buffer allocate()
+    {
+        return allocate(pageSize);
+    }
+
+    public Buffer allocate(int minimumCapacity)
+    {
+        int size = this.pageSize;
+        while (size < minimumCapacity) {
+            size *= 2;
+        }
+        return new NettyByteBufBuffer(nettyBuffer.buffer(size));
+    }
+
+    private static class NettyByteBufBuffer
+            extends Buffer
+    {
+        private ByteBuf buf;
+        private BufferReleasedBeforeAt doubleFreeCheck;
+
+        public NettyByteBufBuffer(ByteBuf buf)
+        {
+            super(buf.array(), buf.arrayOffset(), buf.capacity());
+            this.buf = buf;
+        }
+
+        public void release()
+        {
+            if (doubleFreeCheck != null) {
+                new BufferDoubleReleasedException(doubleFreeCheck).printStackTrace();
+            }
+            if (buf != null) {
+                buf.release();
+                buf = null;
+                doubleFreeCheck = new BufferReleasedBeforeAt();
+            }
+        }
+    }
+
+    static class BufferReleasedBeforeAt
+            extends Throwable
+    { }
+
+    static class BufferDoubleReleasedException
+            extends IllegalStateException
+    {
+        public BufferDoubleReleasedException(BufferReleasedBeforeAt releasedAt)
+        {
+            super("Detected double release() call of a buffer", releasedAt);
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java

@@ -0,0 +1,183 @@
+package org.embulk.exec;
+
+import java.util.List;
+import java.util.ArrayList;
+import javax.validation.constraints.NotNull;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.common.base.Throwables;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.plugin.PluginType;
+import org.embulk.spi.Schema;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.FilterPlugin;
+import org.embulk.spi.Exec;
+import org.embulk.spi.ExecSession;
+import org.embulk.spi.ExecAction;
+import org.embulk.spi.util.Filters;
+
+public class PreviewExecutor
+{
+    private final Injector injector;
+    private final ConfigSource systemConfig;
+
+    public interface PreviewTask
+            extends Task
+    {
+        @Config("in")
+        @NotNull
+        public ConfigSource getInputConfig();
+
+        @Config("filters")
+        @ConfigDefault("[]")
+        public List<ConfigSource> getFilterConfigs();
+
+        // TODO take preview_sample_rows from exec: config
+        @Config("preview_sample_rows")
+        @ConfigDefault("15")
+        public int getSampleRows();
+
+        public TaskSource getInputTask();
+        public void setInputTask(TaskSource taskSource);
+    }
+
+    @Inject
+    public PreviewExecutor(Injector injector,
+            @ForSystemConfig ConfigSource systemConfig)
+    {
+        this.injector = injector;
+        this.systemConfig = systemConfig;
+    }
+
+    public PreviewResult preview(ExecSession exec, final ConfigSource config)
+    {
+        try {
+            return Exec.doWith(exec.forPreview(), new ExecAction<PreviewResult>() {
+                public PreviewResult run()
+                {
+                    try (SetCurrentThreadName dontCare = new SetCurrentThreadName("preview")) {
+                        return doPreview(config);
+                    }
+                }
+            });
+        } catch (Exception ex) {
+            throw Throwables.propagate(ex.getCause());
+        }
+    }
+
+    protected InputPlugin newInputPlugin(PreviewTask task)
+    {
+        return Exec.newPlugin(InputPlugin.class, task.getInputConfig().get(PluginType.class, "type"));
+    }
+
+    protected List<FilterPlugin> newFilterPlugins(PreviewTask task)
+    {
+        return Filters.newFilterPluginsFromConfigSources(Exec.session(), task.getFilterConfigs());
+    }
+
+    private PreviewResult doPreview(ConfigSource config)
+    {
+        final PreviewTask task = config.loadConfig(PreviewTask.class);
+        final InputPlugin input = newInputPlugin(task);
+        final List<FilterPlugin> filterPlugins = newFilterPlugins(task);
+
+        try {
+            input.transaction(task.getInputConfig(), new InputPlugin.Control() {
+                public List<TaskReport> run(final TaskSource inputTask, Schema inputSchema, final int taskCount)
+                {
+                    Filters.transaction(filterPlugins, task.getFilterConfigs(), inputSchema, new Filters.Control() {
+                        public void run(final List<TaskSource> filterTasks, final List<Schema> filterSchemas)
+                        {
+                            InputPlugin input = newInputPlugin(task);
+                            List<FilterPlugin> filterPlugins = newFilterPlugins(task);
+                            Schema inputSchema = filterSchemas.get(0);
+                            Schema outputSchema = filterSchemas.get(filterSchemas.size() - 1);
+
+                            PageOutput out = new SamplingPageOutput(task.getSampleRows(), outputSchema);
+                            try {
+                                for (int taskIndex=0; taskIndex < taskCount; taskIndex++) {
+                                    try {
+                                        out = Filters.open(filterPlugins, filterTasks, filterSchemas, out);
+                                        input.run(inputTask, inputSchema, taskIndex, out);
+                                    } catch (NoSampleException ex) {
+                                        if (taskIndex == taskCount - 1) {
+                                            throw ex;
+                                        }
+                                    }
+                                }
+                            } finally {
+                                out.close();
+                            }
+                        }
+                    });
+                    // program never reaches here because SamplingPageOutput.finish throws an error.
+                    throw new NoSampleException("No input records to preview");
+                }
+            });
+            throw new AssertionError("PreviewExecutor executor must throw PreviewedNoticeError");
+        } catch (PreviewedNoticeError previewed) {
+            return previewed.getPreviewResult();
+        }
+    }
+
+    private static class SamplingPageOutput
+            implements PageOutput
+    {
+        private final int sampleRows;
+        private final Schema schema;
+        private List<Page> pages;
+        private int recordCount;
+
+        public SamplingPageOutput(int sampleRows, Schema schema)
+        {
+            this.sampleRows = sampleRows;
+            this.schema = schema;
+            this.pages = new ArrayList<Page>();
+        }
+
+        public int getRecordCount()
+        {
+            return recordCount;
+        }
+
+        @Override
+        public void add(Page page)
+        {
+            pages.add(page);
+            recordCount += PageReader.getRecordCount(page);
+            if (recordCount >= sampleRows) {
+                finish();
+            }
+        }
+
+        @Override
+        public void finish()
+        {
+            if (recordCount == 0) {
+                throw new NoSampleException("No input records to preview");
+            }
+            PreviewResult res = new PreviewResult(schema, pages);
+            pages = null;
+            throw new PreviewedNoticeError(res);
+        }
+
+        @Override
+        public void close()
+        {
+            if (pages != null) {
+                for (Page page : pages) {
+                    page.release();
+                }
+                pages = null;
+            }
+        }
+    }
+}