embulk 0.8.37-java → 0.10.26-java

data/embulk-core/src/main/java/org/embulk/exec/LoggerProvider.java

@@ -1,68 +0,0 @@
-package org.embulk.exec;
-
-import org.slf4j.ILoggerFactory;
-import org.slf4j.LoggerFactory;
-import ch.qos.logback.classic.Level;
-import ch.qos.logback.classic.Logger;
-import ch.qos.logback.classic.LoggerContext;
-import ch.qos.logback.classic.joran.JoranConfigurator;
-import ch.qos.logback.core.joran.spi.JoranException;
-import com.google.inject.Inject;
-import com.google.inject.Provider;
-import org.embulk.config.ConfigSource;
-
-public class LoggerProvider
-        implements Provider<ILoggerFactory>
-{
-    @Inject
-    public LoggerProvider(@ForSystemConfig ConfigSource systemConfig)
-    {
-        final String level;
-        String logLevel = systemConfig.get(String.class, "log_level", "info");  // here can't use loadConfig because ModelManager uses LoggerProvider
-        switch (logLevel) {
-        case "error": level = "ERROR"; break;
-        case "warn":  level = "WARN";  break;
-        case "info":  level = "INFO";  break;
-        case "debug": level = "DEBUG"; break;
-        case "trace": level = "TRACE"; break;
-        default:
-            throw new IllegalArgumentException(String.format(
-                        "System property embulk.logLevel=%s is invalid. Available levels are error, warn, info, debug and trace.", logLevel));
-        }
-
-        LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
-        JoranConfigurator configurator = new JoranConfigurator();
-        configurator.setContext(context);
-        context.reset();
-
-        String logPath = systemConfig.get(String.class, "log_path", "-");
-
-        String name;
-        if (logPath.equals("-")) {
-            if (System.console() != null) {
-                name = "/embulk/logback-color.xml";
-            } else {
-                name = "/embulk/logback-console.xml";
-            }
-        } else {
-            // logback uses system property to embed variables in XML file
-            System.setProperty("embulk.logPath", logPath);
-            name = "/embulk/logback-file.xml";
-        }
-        try {
-            configurator.doConfigure(getClass().getResource(name));
-        } catch (JoranException ex) {
-            throw new RuntimeException(ex);
-        }
-
-        org.slf4j.Logger logger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
-        if (logger instanceof Logger) {
-            ((Logger) logger).setLevel(Level.toLevel(level.toUpperCase(), Level.DEBUG));
-        }
-    }
-
-    public ILoggerFactory get()
-    {
-        return LoggerFactory.getILoggerFactory();
-    }
-}

data/embulk-core/src/main/java/org/embulk/exec/NoSampleException.java

@@ -1,10 +0,0 @@
-package org.embulk.exec;
-
-public class NoSampleException
-        extends RuntimeException
-{
-    public NoSampleException(String message)
-    {
-        super(message);
-    }
-}

data/embulk-core/src/main/java/org/embulk/exec/PartialExecutionException.java

@@ -1,26 +0,0 @@
-package org.embulk.exec;
-
-public class PartialExecutionException
-        extends RuntimeException
-{
-    private final ResumeState resumeState;
-    private final TransactionStage transactionStage;
-
-    public PartialExecutionException(Throwable cause, ResumeState resumeState,
-            TransactionStage transactionStage)
-    {
-        super(cause);
-        this.resumeState = resumeState;
-        this.transactionStage = transactionStage;
-    }
-
-    public ResumeState getResumeState()
-    {
-        return resumeState;
-    }
-
-    public TransactionStage getTransactionStage()
-    {
-        return transactionStage;
-    }
-}

data/embulk-core/src/main/java/org/embulk/exec/PooledBufferAllocator.java

@@ -1,77 +0,0 @@
-package org.embulk.exec;
-
-import io.netty.buffer.PooledByteBufAllocator;
-import io.netty.buffer.ByteBuf;
-import org.embulk.spi.Buffer;
-import org.embulk.spi.BufferAllocator;
-import com.google.inject.Inject;
-import org.embulk.config.ConfigSource;
-import org.embulk.spi.unit.ByteSize;
-
-public class PooledBufferAllocator
-        implements BufferAllocator
-{
-    private static final int DEFAULT_PAGE_SIZE = 32*1024;
-
-    private final PooledByteBufAllocator nettyBuffer;
-    private final int pageSize;
-
-    @Inject
-    public PooledBufferAllocator(@ForSystemConfig ConfigSource systemConfig, org.slf4j.ILoggerFactory factory)
-    {
-        this.nettyBuffer = new PooledByteBufAllocator(false);
-        this.pageSize = systemConfig.get(ByteSize.class, "page_size", new ByteSize(DEFAULT_PAGE_SIZE)).getBytesInt();
-    }
-
-    public Buffer allocate()
-    {
-        return allocate(pageSize);
-    }
-
-    public Buffer allocate(int minimumCapacity)
-    {
-        int size = this.pageSize;
-        while (size < minimumCapacity) {
-            size *= 2;
-        }
-        return new NettyByteBufBuffer(nettyBuffer.buffer(size));
-    }
-
-    private static class NettyByteBufBuffer
-            extends Buffer
-    {
-        private ByteBuf buf;
-        private BufferReleasedBeforeAt doubleFreeCheck;
-
-        public NettyByteBufBuffer(ByteBuf buf)
-        {
-            super(buf.array(), buf.arrayOffset(), buf.capacity());
-            this.buf = buf;
-        }
-
-        public void release()
-        {
-            if (doubleFreeCheck != null) {
-                new BufferDoubleReleasedException(doubleFreeCheck).printStackTrace();
-            }
-            if (buf != null) {
-                buf.release();
-                buf = null;
-                doubleFreeCheck = new BufferReleasedBeforeAt();
-            }
-        }
-    }
-
-    static class BufferReleasedBeforeAt
-            extends Throwable
-    { }
-
-    static class BufferDoubleReleasedException
-            extends IllegalStateException
-    {
-        public BufferDoubleReleasedException(BufferReleasedBeforeAt releasedAt)
-        {
-            super("Detected double release() call of a buffer", releasedAt);
-        }
-    }
-}

data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java

@@ -1,222 +0,0 @@
-package org.embulk.exec;
-
-import java.util.List;
-import java.util.ArrayList;
-import javax.validation.constraints.NotNull;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.common.base.Throwables;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.TaskReport;
-import org.embulk.exec.SamplingParserPlugin.SampleBufferTask;
-import org.embulk.plugin.PluginType;
-import org.embulk.spi.Buffer;
-import org.embulk.spi.FileInputRunner;
-import org.embulk.spi.Schema;
-import org.embulk.spi.Page;
-import org.embulk.spi.PageOutput;
-import org.embulk.spi.PageReader;
-import org.embulk.spi.InputPlugin;
-import org.embulk.spi.FilterPlugin;
-import org.embulk.spi.Exec;
-import org.embulk.spi.ExecSession;
-import org.embulk.spi.ExecAction;
-import org.embulk.spi.util.Filters;
-import org.slf4j.Logger;
-
-public class PreviewExecutor
-{
-    private final Injector injector;
-    private final ConfigSource systemConfig;
-
-    public interface PreviewTask
-            extends Task
-    {
-        @Config("exec")
-        @ConfigDefault("{}")
-        public ConfigSource getExecConfig();
-
-        @Config("in")
-        @NotNull
-        public ConfigSource getInputConfig();
-
-        @Config("filters")
-        @ConfigDefault("[]")
-        public List<ConfigSource> getFilterConfigs();
-
-        // TODO take preview_sample_rows from exec: config
-        @Config("preview_sample_rows")
-        @ConfigDefault("15")
-        public int getSampleRows();
-
-        public TaskSource getInputTask();
-        public void setInputTask(TaskSource taskSource);
-    }
-
-    public interface PreviewExecutorTask
-            extends Task
-    {
-        @Config("preview_sample_buffer_bytes")
-        @ConfigDefault("32768") // 32 * 1024
-        public int getSampleBufferBytes();
-    }
-
-    @Inject
-    public PreviewExecutor(Injector injector,
-            @ForSystemConfig ConfigSource systemConfig)
-    {
-        this.injector = injector;
-        this.systemConfig = systemConfig;
-    }
-
-    public PreviewResult preview(ExecSession exec, final ConfigSource config)
-    {
-        try {
-            return Exec.doWith(exec.forPreview(), new ExecAction<PreviewResult>() {
-                public PreviewResult run()
-                {
-                    try (SetCurrentThreadName dontCare = new SetCurrentThreadName("preview")) {
-                        return doPreview(config);
-                    }
-                }
-            });
-        } catch (Exception ex) {
-            throw Throwables.propagate(ex.getCause());
-        }
-    }
-
-    protected InputPlugin newInputPlugin(PreviewTask task)
-    {
-        return Exec.newPlugin(InputPlugin.class, task.getInputConfig().get(PluginType.class, "type"));
-    }
-
-    protected List<FilterPlugin> newFilterPlugins(PreviewTask task)
-    {
-        return Filters.newFilterPluginsFromConfigSources(Exec.session(), task.getFilterConfigs());
-    }
-
-    private PreviewResult doPreview(ConfigSource config)
-    {
-        PreviewTask task = config.loadConfig(PreviewTask.class);
-        InputPlugin inputPlugin = newInputPlugin(task);
-        List<FilterPlugin> filterPlugins = newFilterPlugins(task);
-
-        if (inputPlugin instanceof FileInputRunner) { // file input runner
-            Buffer sample = SamplingParserPlugin.runFileInputSampling((FileInputRunner)inputPlugin, config.getNested("in"), createSampleBufferConfigFromExecConfig(task.getExecConfig()));
-            FileInputRunner previewRunner = new FileInputRunner(new BufferFileInputPlugin(sample));
-            return doPreview(task, previewRunner, filterPlugins);
-        }
-        else {
-            return doPreview(task, inputPlugin, filterPlugins);
-        }
-    }
-
-    private static ConfigSource createSampleBufferConfigFromExecConfig(ConfigSource execConfig)
-    {
-        final PreviewExecutorTask execTask = execConfig.loadConfig(PreviewExecutorTask.class);
-        return Exec.newConfigSource().set("sample_buffer_bytes", execTask.getSampleBufferBytes());
-    }
-
-    private PreviewResult doPreview(final PreviewTask task, final InputPlugin input, final List<FilterPlugin> filterPlugins)
-    {
-        try {
-            input.transaction(task.getInputConfig(), new InputPlugin.Control() {
-                public List<TaskReport> run(final TaskSource inputTask, Schema inputSchema, final int taskCount)
-                {
-                    Filters.transaction(filterPlugins, task.getFilterConfigs(), inputSchema, new Filters.Control() {
-                        public void run(final List<TaskSource> filterTasks, final List<Schema> filterSchemas)
-                        {
-                            Schema inputSchema = filterSchemas.get(0);
-                            Schema outputSchema = filterSchemas.get(filterSchemas.size() - 1);
-
-                            PageOutput out = new SamplingPageOutput(task.getSampleRows(), outputSchema);
-                            try {
-                                for (int taskIndex=0; taskIndex < taskCount; taskIndex++) {
-                                    try {
-                                        out = Filters.open(filterPlugins, filterTasks, filterSchemas, out);
-                                        input.run(inputTask, inputSchema, taskIndex, out);
-                                    } catch (NoSampleException ex) {
-                                        if (taskIndex == taskCount - 1) {
-                                            throw ex;
-                                        }
-                                    }
-                                }
-                            } finally {
-                                out.close();
-                            }
-                        }
-                    });
-                    // program never reaches here because SamplingPageOutput.finish throws an error.
-                    throw new NoSampleException("No input records to preview");
-                }
-            });
-            throw new AssertionError("PreviewExecutor executor must throw PreviewedNoticeError");
-        } catch (PreviewedNoticeError previewed) {
-            return previewed.getPreviewResult();
-        }
-    }
-
-    private static class SamplingPageOutput
-            implements PageOutput
-    {
-        private final Logger log = Exec.getLogger(this.getClass());
-        private final int sampleRows;
-        private final Schema schema;
-        private List<Page> pages;
-        private int recordCount;
-        private PreviewResult res;
-
-        public SamplingPageOutput(int sampleRows, Schema schema)
-        {
-            this.sampleRows = sampleRows;
-            this.schema = schema;
-            this.pages = new ArrayList<Page>();
-            this.res = null;
-        }
-
-        public int getRecordCount()
-        {
-            return recordCount;
-        }
-
-        @Override
-        public void add(Page page)
-        {
-            pages.add(page);
-            recordCount += PageReader.getRecordCount(page);
-            if (recordCount >= sampleRows) {
-                finish();
-            }
-        }
-
-        @Override
-        public void finish()
-        {
-            if (res != null) {
-                log.error("PreviewResult recreation will cause a bug. The plugin must call PageOutput#finish() only once.");
-            }
-
-            if (recordCount == 0) {
-                throw new NoSampleException("No input records to preview");
-            }
-            res = new PreviewResult(schema, pages);
-            pages = null;
-            throw new PreviewedNoticeError(res);
-        }
-
-        @Override
-        public void close()
-        {
-            if (pages != null) {
-                for (Page page : pages) {
-                    page.release();
-                }
-                pages = null;
-            }
-        }
-    }
-}

data/embulk-core/src/main/java/org/embulk/exec/PreviewResult.java

@@ -1,27 +0,0 @@
-package org.embulk.exec;
-
-import java.util.List;
-import org.embulk.spi.Schema;
-import org.embulk.spi.Page;
-
-public class PreviewResult
-{
-    private final Schema schema;
-    private final List<Page> pages;
-
-    public PreviewResult(Schema schema, List<Page> pages)
-    {
-        this.schema = schema;
-        this.pages = pages;
-    }
-
-    public Schema getSchema()
-    {
-        return schema;
-    }
-
-    public List<Page> getPages()
-    {
-        return pages;
-    }
-}

data/embulk-core/src/main/java/org/embulk/exec/PreviewedNoticeError.java

@@ -1,17 +0,0 @@
-package org.embulk.exec;
-
-public class PreviewedNoticeError
-        extends Error
-{
-    private final PreviewResult previewResult;
-
-    public PreviewedNoticeError(PreviewResult previewResult)
-    {
-        this.previewResult = previewResult;
-    }
-
-    public PreviewResult getPreviewResult()
-    {
-        return previewResult;
-    }
-}