embulk 0.1.0

data/embulk-core/src/main/java/org/embulk/exec/LoggerProvider.java

@@ -0,0 +1,30 @@
+package org.embulk.exec;
+
+import java.util.Properties;
+import org.slf4j.ILoggerFactory;
+import org.slf4j.LoggerFactory;
+import org.apache.log4j.PropertyConfigurator;
+import com.google.inject.Provider;
+
+public class LoggerProvider
+        implements Provider<ILoggerFactory>
+{
+    public LoggerProvider()
+    {
+        // TODO system config
+        Properties prop = new Properties();
+
+        prop.setProperty("log4j.rootLogger", "INFO,root");
+        prop.setProperty("log4j.appender.root", "org.apache.log4j.ConsoleAppender");
+        prop.setProperty("log4j.appender.root.layout", "org.apache.log4j.PatternLayout");
+        prop.setProperty("log4j.appender.root.layout.ConversionPattern", "%d [%p]: %t:%c: %m%n");
+
+        // TODO
+        PropertyConfigurator.configure(prop);
+    }
+
+    public ILoggerFactory get()
+    {
+        return LoggerFactory.getILoggerFactory();
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/NoSampleException.java

@@ -0,0 +1,10 @@
+package org.embulk.exec;
+
+public class NoSampleException
+        extends RuntimeException
+{
+    public NoSampleException(String message)
+    {
+        super(message);
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/PooledBufferAllocator.java

@@ -0,0 +1,58 @@
+package org.embulk.exec;
+
+import io.netty.buffer.PooledByteBufAllocator;
+import io.netty.buffer.ByteBuf;
+import io.netty.util.ResourceLeakDetector;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.BufferAllocator;
+
+public class PooledBufferAllocator
+        implements BufferAllocator
+{
+    private PooledByteBufAllocator nettyBuffer;
+
+    public PooledBufferAllocator()
+    {
+        // TODO configure parameters
+        this.nettyBuffer = new PooledByteBufAllocator(false);
+    }
+
+    public Buffer allocate()
+    {
+        return new NettyByteBufBuffer(nettyBuffer.buffer());
+    }
+
+    public Buffer allocate(int minimumCapacity)
+    {
+        int size = 32*1024;
+        while (size < minimumCapacity) {
+            size *= 2;
+        }
+        return new NettyByteBufBuffer(nettyBuffer.buffer(size));
+    }
+
+    private static class NettyByteBufBuffer
+            extends Buffer
+    {
+        private ByteBuf buf;
+        private Exception doubleFreeCheck;
+
+        public NettyByteBufBuffer(ByteBuf buf)
+        {
+            super(buf.array(), buf.arrayOffset(), buf.capacity());
+            this.buf = buf;
+        }
+
+        public void release()
+        {
+            if (doubleFreeCheck != null) {
+                doubleFreeCheck.printStackTrace();
+            }
+            if (buf != null) {
+                buf.release();
+                buf = null;
+                doubleFreeCheck = new NullPointerException();
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java

@@ -0,0 +1,138 @@
+package org.embulk.exec;
+
+import java.util.List;
+import java.util.ArrayList;
+import javax.validation.constraints.NotNull;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.common.base.Throwables;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.CommitReport;
+import org.embulk.plugin.PluginType;
+import org.embulk.spi.Schema;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.Exec;
+import org.embulk.spi.ExecSession;
+import org.embulk.spi.ExecAction;
+
+public class PreviewExecutor
+{
+    private final Injector injector;
+    private final ConfigSource systemConfig;
+
+    public interface PreviewTask
+            extends Task
+    {
+        @Config("in")
+        @NotNull
+        public ConfigSource getInputConfig();
+
+        @Config("preview_sample_rows")
+        @ConfigDefault("15")
+        public int getSampleRows();
+
+        public TaskSource getInputTask();
+        public void setInputTask(TaskSource taskSource);
+    }
+
+    @Inject
+    public PreviewExecutor(Injector injector,
+            @ForSystemConfig ConfigSource systemConfig)
+    {
+        this.injector = injector;
+        this.systemConfig = systemConfig;
+    }
+
+    public PreviewResult preview(ExecSession exec, final ConfigSource config)
+    {
+        try {
+            return Exec.doWith(exec, new ExecAction<PreviewResult>() {
+                public PreviewResult run()
+                {
+                    return doPreview(config);
+                }
+            });
+        } catch (Exception ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    protected InputPlugin newInputPlugin(PreviewTask task)
+    {
+        return Exec.newPlugin(InputPlugin.class, task.getInputConfig().get(PluginType.class, "type"));
+    }
+
+    private PreviewResult doPreview(ConfigSource config)
+    {
+        final PreviewTask task = config.loadConfig(PreviewTask.class);
+        InputPlugin input = newInputPlugin(task);
+
+        try {
+            input.transaction(task.getInputConfig(), new InputPlugin.Control() {
+                public List<CommitReport> run(TaskSource taskSource, Schema schema, int processorCount)
+                {
+                    InputPlugin input = newInputPlugin(task);
+                    try (SamplingPageOutput out = new SamplingPageOutput(task.getSampleRows(), schema)) {
+                        input.run(taskSource, schema, 0, out);
+                    }
+                    throw new NoSampleException("No input records to preview");
+                }
+            });
+            throw new AssertionError("PreviewExecutor executor must throw PreviewedNoticeError");
+        } catch (PreviewedNoticeError previewed) {
+            return previewed.getPreviewResult();
+        }
+    }
+
+    private static class SamplingPageOutput
+            implements PageOutput
+    {
+        private final int sampleRows;
+        private final Schema schema;
+        private List<Page> pages;
+        private int recordCount;
+
+        public SamplingPageOutput(int sampleRows, Schema schema)
+        {
+            this.sampleRows = sampleRows;
+            this.schema = schema;
+            this.pages = new ArrayList<Page>();
+        }
+
+        @Override
+        public void add(Page page)
+        {
+            pages.add(page);
+            recordCount += PageReader.getRecordCount(page);
+            if (recordCount >= sampleRows) {
+                finish();
+            }
+        }
+
+        @Override
+        public void finish()
+        {
+            if (recordCount == 0) {
+                throw new NoSampleException("No input records to preview");
+            }
+            PreviewResult res = new PreviewResult(schema, pages);
+            pages = null;
+            throw new PreviewedNoticeError(res);
+        }
+
+        @Override
+        public void close()
+        {
+            for (Page page : pages) {
+                page.release();
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/PreviewResult.java

@@ -0,0 +1,27 @@
+package org.embulk.exec;
+
+import java.util.List;
+import org.embulk.spi.Schema;
+import org.embulk.spi.Page;
+
+public class PreviewResult
+{
+    private final Schema schema;
+    private final List<Page> pages;
+
+    public PreviewResult(Schema schema, List<Page> pages)
+    {
+        this.schema = schema;
+        this.pages = pages;
+    }
+
+    public Schema getSchema()
+    {
+        return schema;
+    }
+
+    public List<Page> getPages()
+    {
+        return pages;
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/PreviewedNoticeError.java

@@ -0,0 +1,17 @@
+package org.embulk.exec;
+
+public class PreviewedNoticeError
+        extends Error
+{
+    private final PreviewResult previewResult;
+
+    public PreviewedNoticeError(PreviewResult previewResult)
+    {
+        this.previewResult = previewResult;
+    }
+
+    public PreviewResult getPreviewResult()
+    {
+        return previewResult;
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java

@@ -0,0 +1,116 @@
+package org.embulk.exec;
+
+import java.util.List;
+import com.google.inject.Inject;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.CommitReport;
+import org.embulk.plugin.PluginType;
+import org.embulk.spi.Schema;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Page;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.PageOutput;
+import static org.embulk.spi.util.Inputs.each;
+
+/*
+ * Used by GuessExecutor
+ */
+class SamplingParserPlugin
+        implements ParserPlugin
+{
+    private final int maxSampleSize;
+
+    @Inject
+    public SamplingParserPlugin(@ForSystemConfig ConfigSource systemConfig)
+    {
+        this.maxSampleSize = 32*1024;  // TODO get sample syze from system config
+    }
+
+    @Override
+    public void transaction(ConfigSource config, ParserPlugin.Control control)
+    {
+        control.run(Exec.newTaskSource(), null);
+    }
+
+    @Override
+    public void run(TaskSource taskSource, Schema schema,
+            FileInput input, PageOutput output)
+    {
+        Buffer buffer = getSample(input, maxSampleSize);
+        throw new SampledNoticeError(buffer);
+    }
+
+    static Buffer runFileInputSampling(ConfigSource config)
+    {
+        // override in.parser.type so that FileInputRunner creates GuessParserPlugin
+        ConfigSource samplingInputConfig = config.getNested("in").deepCopy();
+        samplingInputConfig.getNestedOrSetEmpty("parser").set("type", "system_sampling");
+
+        final InputPlugin input = Exec.newPlugin(InputPlugin.class, samplingInputConfig.get(PluginType.class, "type"));
+        try {
+            input.transaction(samplingInputConfig, new InputPlugin.Control() {
+                public List<CommitReport> run(TaskSource taskSource, Schema schema, int processorCount)
+                {
+                    input.run(taskSource, schema, 0, new PageOutput() {
+                        @Override
+                        public void add(Page page)
+                        {
+                            throw new RuntimeException("Input plugin must be a FileInputPlugin to guess parser configuration");  // TODO exception class
+                        }
+
+                        public void finish() { }
+
+                        public void close() { }
+                    });
+                    throw new NoSampleException("No input files to guess parser configuration");
+                }
+            });
+            throw new AssertionError("SamplingParserPlugin must throw SampledNoticeError");
+        } catch (SampledNoticeError error) {
+            return error.getSample();
+        }
+    }
+
+    private static Buffer getSample(FileInput fileInput, int maxSampleSize)
+    {
+        Buffer sample = Buffer.allocate(maxSampleSize);
+        int sampleSize = 0;
+
+        while (fileInput.nextFile()) {
+            for (Buffer buffer : each(fileInput)) {
+                if (sampleSize >= maxSampleSize) {
+                    // skip remaining all buffers so that FileInputPlugin.runInput doesn't
+                    // throw exceptions at channel.join()
+                } else {
+                    int size = Math.min(buffer.limit(), sample.capacity() - sampleSize);
+                    sample.setBytes(sampleSize, buffer, 0, size);
+                    sampleSize += size;
+                }
+                buffer.release();
+            }
+        }
+
+        sample.limit(sampleSize);
+        return sample;
+    }
+
+    public static class SampledNoticeError
+            extends Error
+    {
+        private final Buffer sample;
+
+        public SampledNoticeError(Buffer sample)
+        {
+            this.sample = sample;
+        }
+
+        public Buffer getSample()
+        {
+            return sample;
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/SystemConfigModule.java

@@ -0,0 +1,24 @@
+package org.embulk.exec;
+
+import com.google.inject.Module;
+import com.google.inject.Binder;
+import org.embulk.config.ConfigSource;
+
+public class SystemConfigModule
+        implements Module
+{
+    private final ConfigSource systemConfig;
+
+    public SystemConfigModule(ConfigSource systemConfig)
+    {
+        this.systemConfig = systemConfig;
+    }
+
+    @Override
+    public void configure(Binder binder)
+    {
+        binder.bind(ConfigSource.class)
+            .annotatedWith(ForSystemConfig.class)
+            .toInstance(systemConfig);
+    }
+}

data/embulk-core/src/main/java/org/embulk/jruby/JRubyPluginSource.java

@@ -0,0 +1,69 @@
+package org.embulk.jruby;
+
+import com.google.inject.Inject;
+import org.jruby.embed.ScriptingContainer;
+import org.jruby.embed.InvokeFailedException;
+import org.embulk.plugin.PluginType;
+import org.embulk.plugin.PluginSource;
+import org.embulk.plugin.PluginSourceNotMatchException;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.FormatterPlugin;
+import org.embulk.spi.DecoderPlugin;
+import org.embulk.spi.EncoderPlugin;
+//import org.embulk.spi.LineFilterPlugin;
+import org.embulk.spi.GuessPlugin;
+
+public class JRubyPluginSource
+        implements PluginSource
+{
+    private final ScriptingContainer jruby;
+    private final Object rubyPluginManager;
+
+    @Inject
+    public JRubyPluginSource(ScriptingContainer jruby)
+    {
+        this.jruby = jruby;
+
+        // get Embulk::Plugin
+        //this.rubyPluginManager = ((RubyModule) jruby.get("Embulk")).const_get(
+        //        RubySymbol.newSymbol(
+        //            jruby.getProvider().getRuntime(), "Plugin"));
+        this.rubyPluginManager = jruby.runScriptlet("Embulk::Plugin");
+    }
+
+    public <T> T newPlugin(Class<T> iface, PluginType type) throws PluginSourceNotMatchException
+    {
+        String name = type.getName();
+
+        String category;
+        if (InputPlugin.class.isAssignableFrom(iface)) {
+            category = "input";
+        } else if (OutputPlugin.class.isAssignableFrom(iface)) {
+            category = "output";
+        } else if (ParserPlugin.class.isAssignableFrom(iface)) {
+            category = "parser";
+        } else if (FormatterPlugin.class.isAssignableFrom(iface)) {
+            category = "formatter";
+        } else if (DecoderPlugin.class.isAssignableFrom(iface)) {
+            category = "decoder";
+        } else if (EncoderPlugin.class.isAssignableFrom(iface)) {
+            category = "encoder";
+        //} else if (LineFilterPlugin.class.isAssignableFrom(iface)) {
+        //    category = "line_filter";
+        } else if (GuessPlugin.class.isAssignableFrom(iface)) {
+            category = "guess";
+        } else {
+            // unsupported plugin category
+            throw new PluginSourceNotMatchException("Plugin interface "+iface+" is not supported in JRuby");
+        }
+
+        String methodName = "new_java_" + category;
+        try {
+            return jruby.callMethod(rubyPluginManager, methodName, name, iface);
+        } catch (InvokeFailedException ex) {
+            throw new PluginSourceNotMatchException(ex.getCause());
+        }
+    }
+}