embulk 0.5.5 → 0.6.0

data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java

@@ -13,6 +13,7 @@ import org.embulk.config.ModelManager;
 import org.embulk.spi.time.DateTimeZoneSerDe;
 import org.embulk.spi.time.TimestampSerDe;
 import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.ExecutorPlugin;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.util.CharsetSerDe;
 import static org.embulk.plugin.InjectedPluginSource.registerPluginTo;
@@ -33,6 +34,10 @@ public class ExecModule
         registerPluginTo(binder, ParserPlugin.class, "system_guess", GuessExecutor.GuessParserPlugin.class);
         registerPluginTo(binder, ParserPlugin.class, "system_sampling", SamplingParserPlugin.class);
 
+        // LocalExecutorPlugin
+        binder.bind(LocalThreadExecutor.class).in(Scopes.SINGLETON);
+        registerPluginTo(binder, ExecutorPlugin.class, "local", LocalExecutorPlugin.class);
+
         // serde
         ObjectMapperModule mapper = new ObjectMapperModule();
         DateTimeZoneSerDe.configure(mapper);

data/embulk-core/src/main/java/org/embulk/exec/LocalExecutorPlugin.java

@@ -0,0 +1,130 @@
+package org.embulk.exec;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Future;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ExecutionException;
+import org.slf4j.Logger;
+import com.google.inject.Inject;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.CommitReport;
+import org.embulk.spi.Exec;
+import org.embulk.spi.ExecutorPlugin;
+import org.embulk.spi.ProcessTask;
+import org.embulk.spi.ProcessState;
+import org.embulk.spi.TaskState;
+import org.embulk.spi.Schema;
+import org.embulk.spi.util.Executors;
+import org.embulk.spi.util.Executors.ProcessStateCallback;
+
+public class LocalExecutorPlugin
+        implements ExecutorPlugin
+{
+    private final ExecutorService executor;
+
+    @Inject
+    public LocalExecutorPlugin(LocalThreadExecutor executor)
+    {
+        this.executor = executor.getExecutorService();
+    }
+
+    @Override
+    public void transaction(ConfigSource config, Schema outputSchema, final int inputTaskCount,
+            ExecutorPlugin.Control control)
+    {
+        control.transaction(outputSchema, inputTaskCount, new Executor() {
+            public void execute(ProcessTask task, ProcessState state)
+            {
+                localExecute(task, inputTaskCount, state);
+            }
+        });
+    }
+
+    private void localExecute(ProcessTask task, int taskCount, ProcessState state)
+    {
+        Logger log = Exec.getLogger(LocalExecutorPlugin.class);
+
+        state.initialize(taskCount, taskCount);
+
+        List<Future<Throwable>> futures = new ArrayList<>(taskCount);
+        try {
+            for (int i=0; i < taskCount; i++) {
+                if (state.getOutputTaskState(i).isCommitted()) {
+                    log.warn("Skipped resumed task {}", i);
+                    futures.add(null);  // resumed
+                } else {
+                    futures.add(startProcessor(task, i, state));
+                }
+            }
+            showProgress(log, state, taskCount);
+
+            for (int i=0; i < taskCount; i++) {
+                if (futures.get(i) == null) {
+                    continue;
+                }
+                try {
+                    state.getInputTaskState(i).setException(futures.get(i).get());
+                } catch (ExecutionException ex) {
+                    state.getInputTaskState(i).setException(ex.getCause());
+                    //Throwables.propagate(ex.getCause());
+                } catch (InterruptedException ex) {
+                    state.getInputTaskState(i).setException(new ExecutionInterruptedException(ex));
+                }
+                showProgress(log, state, taskCount);
+            }
+        } finally {
+            for (Future<Throwable> future : futures) {
+                if (future != null && !future.isDone()) {
+                    future.cancel(true);
+                    // TODO join?
+                }
+            }
+        }
+    }
+
+    private void showProgress(Logger log, ProcessState state, int taskCount)
+    {
+        int started = 0;
+        int finished = 0;
+        for (int i=0; i < taskCount; i++) {
+            if (state.getInputTaskState(i).isStarted()) { started++; }
+            if (state.getOutputTaskState(i).isFinished()) { finished++; }
+        }
+
+        log.info(String.format("{done:%3d / %d, running: %d}", finished, taskCount, started - finished));
+    }
+
+    private Future<Throwable> startProcessor(final ProcessTask task, final int taskIndex, final ProcessState state)
+    {
+        return executor.submit(new Callable<Throwable>() {
+            public Throwable call()
+            {
+                try (SetCurrentThreadName dontCare = new SetCurrentThreadName(String.format("task-%04d", taskIndex))) {
+                    Executors.process(Exec.session(), task, taskIndex, new ProcessStateCallback() {
+                        public void started()
+                        {
+                            state.getInputTaskState(taskIndex).start();
+                            state.getOutputTaskState(taskIndex).start();
+                        }
+
+                        public void inputCommitted(CommitReport report)
+                        {
+                            state.getInputTaskState(taskIndex).setCommitReport(report);
+                        }
+
+                        public void outputCommitted(CommitReport report)
+                        {
+                            state.getOutputTaskState(taskIndex).setCommitReport(report);
+                        }
+                    });
+                    return null;
+                } finally {
+                    state.getInputTaskState(taskIndex).finish();
+                    state.getOutputTaskState(taskIndex).finish();
+                }
+            }
+        });
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/LocalThreadExecutor.java

@@ -0,0 +1,34 @@
+package org.embulk.exec;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.inject.Inject;
+import org.embulk.config.ConfigSource;
+
+/*
+ * Injected in SINGLETON scope at ExecModule
+ */
+public class LocalThreadExecutor
+{
+    private final ExecutorService executor;
+
+    @Inject
+    public LocalThreadExecutor(@ForSystemConfig ConfigSource systemConfig)
+    {
+        int defaultMaxThreads = Runtime.getRuntime().availableProcessors() * 2;
+        int maxThreads = systemConfig.get(Integer.class, "max_threads", defaultMaxThreads);
+        this.executor = Executors.newFixedThreadPool(maxThreads,
+                new ThreadFactoryBuilder()
+                        .setNameFormat("embulk-executor-%d")
+                        .setDaemon(true)
+                        .build());
+    }
+
+    public ExecutorService getExecutorService()
+    {
+        return executor;
+    }
+
+    // TODO shutdown
+}

data/embulk-core/src/main/java/org/embulk/exec/PooledBufferAllocator.java

@@ -9,10 +9,10 @@ import org.embulk.spi.BufferAllocator;
 public class PooledBufferAllocator
         implements BufferAllocator
 {
-    private PooledByteBufAllocator nettyBuffer;
+    private static final int DEFAULT_BUFFER_SIZE = 32*1024;
+    private static final int MINIMUM_BUFFER_SIZE = 8*1024;
 
-    private int DEFAULT_BUFFER_SIZE = 32*1024;
-    private int MINIMUM_BUFFER_SIZE = 8*1024;
+    private final PooledByteBufAllocator nettyBuffer;
 
     public PooledBufferAllocator()
     {

data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java

@@ -80,7 +80,7 @@ public class PreviewExecutor
 
     protected List<FilterPlugin> newFilterPlugins(PreviewTask task)
     {
-        return Filters.newFilterPlugins(Exec.session(), task.getFilterConfigs());
+        return Filters.newFilterPluginsFromConfigSources(Exec.session(), task.getFilterConfigs());
     }
 
     private PreviewResult doPreview(ConfigSource config)

data/embulk-core/src/main/java/org/embulk/exec/ResumeState.java

@@ -1,6 +1,7 @@
 package org.embulk.exec;
 
 import java.util.List;
+import com.google.common.base.Optional;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.embulk.config.TaskSource;
@@ -16,8 +17,8 @@ public class ResumeState
     private final TaskSource outputTaskSource;
     private final Schema inputSchema;
     private final Schema outputSchema;
-    private final List<CommitReport> inputCommitReports;
-    private final List<CommitReport> outputCommitReports;
+    private final List<Optional<CommitReport>> inputCommitReports;
+    private final List<Optional<CommitReport>> outputCommitReports;
 
     @JsonCreator
     public ResumeState(
@@ -26,8 +27,8 @@ public class ResumeState
             @JsonProperty("out_task") TaskSource outputTaskSource,
             @JsonProperty("in_schema") Schema inputSchema,
             @JsonProperty("out_schema") Schema outputSchema,
-            @JsonProperty("in_reports") List<CommitReport> inputCommitReports,
-            @JsonProperty("out_reports") List<CommitReport> outputCommitReports)
+            @JsonProperty("in_reports") List<Optional<CommitReport>> inputCommitReports,
+            @JsonProperty("out_reports") List<Optional<CommitReport>> outputCommitReports)
     {
         this.execSessionConfigSource = execSessionConfigSource;
         this.inputTaskSource = inputTaskSource;
@@ -69,13 +70,13 @@ public class ResumeState
     }
 
     @JsonProperty("in_reports")
-    public List<CommitReport> getInputCommitReports()
+    public List<Optional<CommitReport>> getInputCommitReports()
     {
         return inputCommitReports;
     }
 
     @JsonProperty("out_reports")
-    public List<CommitReport> getOutputCommitReports()
+    public List<Optional<CommitReport>> getOutputCommitReports()
     {
         return outputCommitReports;
     }

data/embulk-core/src/main/java/org/embulk/jruby/JRubyPluginSource.java

@@ -14,6 +14,7 @@ import org.embulk.spi.DecoderPlugin;
 import org.embulk.spi.EncoderPlugin;
 import org.embulk.spi.FilterPlugin;
 import org.embulk.spi.GuessPlugin;
+import org.embulk.spi.ExecutorPlugin;
 
 public class JRubyPluginSource
         implements PluginSource
@@ -54,6 +55,8 @@ public class JRubyPluginSource
             category = "filter";
         } else if (GuessPlugin.class.isAssignableFrom(iface)) {
             category = "guess";
+        } else if (ExecutorPlugin.class.isAssignableFrom(iface)) {
+            category = "executor";
         } else {
             // unsupported plugin category
             throw new PluginSourceNotMatchException("Plugin interface "+iface+" is not supported in JRuby");

data/embulk-core/src/main/java/org/embulk/spi/Buffer.java

@@ -8,7 +8,7 @@ public class Buffer
 {
     public static final Buffer EMPTY = Buffer.allocate(0);
 
-    private byte[] array;
+    private final byte[] array;
     private int offset;
     private int filled;
     private final int capacity;
@@ -111,6 +111,38 @@ public class Buffer
     {
     }
 
-    // TODO equals
-    // TODO hashCode
+    @Override
+    public boolean equals(Object other)
+    {
+        if (!(other instanceof Buffer)) {
+            return false;
+        }
+        Buffer o = (Buffer) other;
+
+        // TODO optimize
+        if (limit() != o.limit()) {
+            return false;
+        }
+        int i = offset;
+        int io = o.offset;
+        while (i < filled) {
+            if (array[i] != o.array[io]) {
+                return false;
+            }
+            i++;
+            io++;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        // TODO optimize
+        int result = 1;
+        for (int i = offset; i < filled; i++) {
+            result = 31 * result + array[i];
+        }
+        return result;
+    }
 }

data/embulk-core/src/main/java/org/embulk/spi/Exec.java

@@ -1,5 +1,6 @@
 package org.embulk.spi;
 
+import java.util.concurrent.ExecutionException;
 import org.slf4j.Logger;
 import org.embulk.config.Task;
 import org.embulk.config.ModelManager;
@@ -15,11 +16,13 @@ public class Exec
 
     private Exec() { }
 
-    public static <T> T doWith(ExecSession session, ExecAction<T> action) throws Exception
+    public static <T> T doWith(ExecSession session, ExecAction<T> action) throws ExecutionException
     {
         Exec.session.set(session);
         try {
             return action.run();
+        } catch (Exception ex) {
+            throw new ExecutionException(ex);
         } finally {
             Exec.session.set(null);
         }

data/embulk-core/src/main/java/org/embulk/spi/ExecAction.java

@@ -2,5 +2,5 @@ package org.embulk.spi;
 
 public interface ExecAction <T>
 {
-    public T run();
+    public T run() throws Exception;
 }

data/embulk-core/src/main/java/org/embulk/spi/ExecutorPlugin.java

@@ -0,0 +1,19 @@
+package org.embulk.spi;
+
+import org.embulk.config.ConfigSource;
+
+public interface ExecutorPlugin
+{
+    public interface Executor
+    {
+        public void execute(ProcessTask task, ProcessState state);
+    }
+
+    public interface Control
+    {
+        public void transaction(Schema executorSchema, int outputTaskCount, Executor executor);
+    }
+
+    public void transaction(ConfigSource config, Schema outputSchema, int inputTaskCount,
+            ExecutorPlugin.Control control);
+}

data/embulk-core/src/main/java/org/embulk/spi/Page.java

@@ -28,6 +28,12 @@ public class Page
         return this;
     }
 
+    public List<String> getStringReferences()
+    {
+        // TODO used by mapreduce executor
+        return stringReferences;
+    }
+
     public String getStringReference(int index)
     {
         return stringReferences.get(index);

data/embulk-core/src/main/java/org/embulk/spi/PluginClassLoader.java

@@ -1,11 +1,16 @@
 package org.embulk.spi;
 
+import java.util.List;
+import java.util.Iterator;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.io.IOException;
 import java.nio.file.Path;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.net.MalformedURLException;
-import java.util.List;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterators;
 import org.jruby.Ruby;
 
 public class PluginClassLoader
@@ -17,6 +22,12 @@ public class PluginClassLoader
         "com.ibm.icu.",
     };
 
+    private static final String[] CHILD_FIRST_PATHS = new String[] {
+        "io/netty/",
+        "org/yaml/",
+        "com/ibm/icu/",
+    };
+
     public PluginClassLoader(Ruby pluginJRubyRuntime, List<URL> urls)
     {
         this(urls, pluginJRubyRuntime.getJRubyClassLoader());
@@ -80,6 +91,57 @@ public class PluginClassLoader
         return clazz;
     }
 
+    @Override
+    public URL getResource(String name)
+    {
+        boolean childFirst = isInChildFirstPath(name);
+
+        if (childFirst) {
+            URL childUrl = findResource(name);
+            if (childUrl != null) {
+                return childUrl;
+            }
+        }
+
+        URL parentUrl = getParent().getResource(name);
+        if (parentUrl != null) {
+            return parentUrl;
+        }
+
+        if (!childFirst) {
+            URL childUrl = findResource(name);
+            if (childUrl != null) {
+                return childUrl;
+            }
+        }
+
+        return null;
+    }
+
+    @Override
+    public Enumeration<URL> getResources(String name)
+            throws IOException
+    {
+        List<Iterator<URL>> resources = new ArrayList<>();
+
+        boolean childFirst = isInChildFirstPath(name);
+
+        if (childFirst) {
+            Iterator<URL> childResources = Iterators.forEnumeration(findResources(name));
+            resources.add(childResources);
+        }
+
+        Iterator<URL> parentResources = Iterators.forEnumeration(getParent().getResources(name));
+        resources.add(parentResources);
+
+        if (!childFirst) {
+            Iterator<URL> childResources = Iterators.forEnumeration(findResources(name));
+            resources.add(childResources);
+        }
+
+        return Iterators.asEnumeration(Iterators.concat(resources.iterator()));
+    }
+
     private boolean isInChildFirstPackage(String name)
     {
         for (String pkg : CHILD_FIRST_PACKAGES) {
@@ -89,4 +151,14 @@ public class PluginClassLoader
         }
         return false;
     }
+
+    private boolean isInChildFirstPath(String name)
+    {
+        for (String path : CHILD_FIRST_PATHS) {
+            if (name.startsWith(path)) {
+                return true;
+            }
+        }
+        return false;
+    }
 }