embulk 0.8.38-java → 0.10.27-java

data/embulk-core/src/main/java/org/embulk/exec/ConfigurableGuessInputPlugin.java

@@ -1,9 +0,0 @@
-package org.embulk.exec;
-
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigDiff;
-
-public interface ConfigurableGuessInputPlugin
-{
-    ConfigDiff guess(ConfigSource execConfig, ConfigSource config);
-}

data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java

@@ -1,53 +0,0 @@
-package org.embulk.exec;
-
-import org.slf4j.ILoggerFactory;
-import com.google.common.base.Preconditions;
-import com.google.inject.Module;
-import com.google.inject.Binder;
-import com.google.inject.Scopes;
-import com.fasterxml.jackson.module.guice.ObjectMapperModule;
-import com.fasterxml.jackson.datatype.guava.GuavaModule;
-import com.fasterxml.jackson.datatype.joda.JodaModule;
-import org.embulk.config.ModelManager;
-import org.embulk.spi.time.DateTimeZoneSerDe;
-import org.embulk.spi.time.TimestampSerDe;
-import org.embulk.spi.ParserPlugin;
-import org.embulk.spi.ExecutorPlugin;
-import org.embulk.spi.BufferAllocator;
-import org.embulk.spi.util.CharsetSerDe;
-import org.embulk.spi.unit.LocalFileSerDe;
-import static org.embulk.plugin.InjectedPluginSource.registerPluginTo;
-
-public class ExecModule
-        implements Module
-{
-    @Override
-    public void configure(Binder binder)
-    {
-        Preconditions.checkNotNull(binder, "binder is null.");
-
-        binder.bind(BulkLoader.class);
-
-        binder.bind(ILoggerFactory.class).toProvider(LoggerProvider.class).in(Scopes.SINGLETON);
-        binder.bind(ModelManager.class).in(Scopes.SINGLETON);
-        binder.bind(BufferAllocator.class).to(PooledBufferAllocator.class).in(Scopes.SINGLETON);
-        binder.bind(TempFileAllocator.class).in(Scopes.SINGLETON);
-
-        // GuessExecutor, PreviewExecutor
-        registerPluginTo(binder, ParserPlugin.class, "system_guess", GuessExecutor.GuessParserPlugin.class);
-        registerPluginTo(binder, ParserPlugin.class, "system_sampling", SamplingParserPlugin.class);
-
-        // LocalExecutorPlugin
-        registerPluginTo(binder, ExecutorPlugin.class, "local", LocalExecutorPlugin.class);
-
-        // serde
-        ObjectMapperModule mapper = new ObjectMapperModule();
-        DateTimeZoneSerDe.configure(mapper);
-        TimestampSerDe.configure(mapper);
-        CharsetSerDe.configure(mapper);
-        LocalFileSerDe.configure(mapper);
-        mapper.registerModule(new GuavaModule());  // jackson-datatype-guava
-        mapper.registerModule(new JodaModule());  // jackson-datatype-joda
-        mapper.configure(binder);
-    }
-}

data/embulk-core/src/main/java/org/embulk/exec/ExecutionInterruptedException.java

@@ -1,10 +0,0 @@
-package org.embulk.exec;
-
-public class ExecutionInterruptedException
-        extends RuntimeException
-{
-    public ExecutionInterruptedException(Exception cause)
-    {
-        super(cause);
-    }
-}

data/embulk-core/src/main/java/org/embulk/exec/ExecutionResult.java

@@ -1,33 +0,0 @@
-package org.embulk.exec;
-
-import java.util.List;
-import org.embulk.config.ConfigDiff;
-
-public class ExecutionResult
-{
-    private final ConfigDiff configDiff;
-    private final boolean skipped;
-    private final List<Throwable> ignoredExceptions;
-
-    public ExecutionResult(ConfigDiff configDiff, boolean skipped, List<Throwable> ignoredExceptions)
-    {
-        this.configDiff = configDiff;
-        this.skipped = skipped;
-        this.ignoredExceptions = ignoredExceptions;
-    }
-
-    public ConfigDiff getConfigDiff()
-    {
-        return configDiff;
-    }
-
-    public boolean isSkipped()
-    {
-        return skipped;
-    }
-
-    public List<Throwable> getIgnoredExceptions()
-    {
-        return ignoredExceptions;
-    }
-}

data/embulk-core/src/main/java/org/embulk/exec/ExtensionServiceLoaderModule.java

@@ -1,43 +0,0 @@
-package org.embulk.exec;
-
-import java.util.ServiceLoader;
-import com.google.inject.Module;
-import com.google.inject.Binder;
-import org.embulk.config.ConfigSource;
-import org.embulk.spi.Extension;
-
-/**
- * ExtensionServiceLoaderModule loads Extensions using java.util.ServiceLoader
- * mechanism.
- * Jar packages providing an extension need to include
- * META-INF/services/org.embulk.exec.Extension file. Contents of the file is
- * one-line text of the extension class name (e.g. com.example.MyPluginSourceExtension).
- */
-public class ExtensionServiceLoaderModule
-        implements Module
-{
-    private final ClassLoader classLoader;
-    private final ConfigSource systemConfig;
-
-    public ExtensionServiceLoaderModule(ConfigSource systemConfig)
-    {
-        this(ExtensionServiceLoaderModule.class.getClassLoader(), systemConfig);
-    }
-
-    public ExtensionServiceLoaderModule(ClassLoader classLoader, ConfigSource systemConfig)
-    {
-        this.classLoader = classLoader;
-        this.systemConfig = systemConfig;
-    }
-
-    @Override
-    public void configure(Binder binder)
-    {
-        ServiceLoader<Extension> serviceLoader = ServiceLoader.load(Extension.class, classLoader);
-        for (Extension extension : serviceLoader) {
-            for (Module module : extension.getModules(systemConfig)) {
-                module.configure(binder);
-            }
-        }
-    }
-}

data/embulk-core/src/main/java/org/embulk/exec/ForGuess.java

@@ -1,16 +0,0 @@
-package org.embulk.exec;
-
-import javax.inject.Qualifier;
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-@Retention(RUNTIME)
-@Target({FIELD, PARAMETER, METHOD})
-@Qualifier
-public @interface ForGuess
-{
-}

data/embulk-core/src/main/java/org/embulk/exec/ForSystemConfig.java

@@ -1,16 +0,0 @@
-package org.embulk.exec;
-
-import javax.inject.Qualifier;
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-@Retention(RUNTIME)
-@Target({FIELD, PARAMETER, METHOD})
-@Qualifier
-public @interface ForSystemConfig
-{
-}

data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java

@@ -1,309 +0,0 @@
-package org.embulk.exec;
-
-import java.util.List;
-import java.util.Set;
-import java.util.ArrayList;
-import java.util.concurrent.ExecutionException;
-import com.google.common.collect.ImmutableList;
-import com.google.common.base.Throwables;
-import com.google.inject.Inject;
-import com.google.inject.Binder;
-import com.google.inject.multibindings.Multibinder;
-import org.embulk.plugin.PluginType;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.TaskReport;
-import org.embulk.spi.Schema;
-import org.embulk.spi.Page;
-import org.embulk.spi.Buffer;
-import org.embulk.spi.InputPlugin;
-import org.embulk.spi.FileInputPlugin;
-import org.embulk.spi.ParserPlugin;
-import org.embulk.spi.GuessPlugin;
-import org.embulk.spi.Exec;
-import org.embulk.spi.ExecAction;
-import org.embulk.spi.ExecSession;
-import org.embulk.spi.FileInput;
-import org.embulk.spi.PageOutput;
-import org.embulk.spi.TransactionalFileInput;
-import org.embulk.spi.FileInputRunner;
-import static org.embulk.spi.util.Inputs.each;
-
-public class GuessExecutor
-{
-    private final List<PluginType> defaultGuessPlugins;
-
-    private interface GuessExecutorSystemTask
-            extends Task
-    {
-        @Config("guess_plugins")
-        @ConfigDefault("[]")
-        public List<PluginType> getGuessPlugins();
-    }
-
-    private interface GuessExecutorTask
-            extends Task
-    {
-        @Config("guess_plugins")
-        @ConfigDefault("[]")
-        public List<PluginType> getGuessPlugins();
-
-        @Config("exclude_guess_plugins")
-        @ConfigDefault("[]")
-        public List<PluginType> getExcludeGuessPlugins();
-
-        @Config("guess_sample_buffer_bytes")
-        @ConfigDefault("32768") // 32 * 1024
-        public int getSampleBufferBytes();
-    }
-
-    public static void registerDefaultGuessPluginTo(Binder binder, PluginType type)
-    {
-        Multibinder<PluginType> multibinder = Multibinder.newSetBinder(binder, PluginType.class, ForGuess.class);
-        multibinder.addBinding().toInstance(type);
-    }
-
-    // Used by FileInputRunner#guess(..)
-    public static ConfigSource createSampleBufferConfigFromExecConfig(ConfigSource execConfig)
-    {
-        final GuessExecutorTask execTask = execConfig.loadConfig(GuessExecutorTask.class);
-        return Exec.newConfigSource().set("sample_buffer_bytes", execTask.getSampleBufferBytes());
-    }
-
-    @Inject
-    public GuessExecutor(@ForSystemConfig ConfigSource systemConfig,
-            @ForGuess Set<PluginType> defaultGuessPlugins)
-    {
-        GuessExecutorSystemTask systemTask = systemConfig.loadConfig(GuessExecutorSystemTask.class);
-
-        ImmutableList.Builder<PluginType> list = ImmutableList.builder();
-        list.addAll(defaultGuessPlugins);
-        list.addAll(systemTask.getGuessPlugins());
-        this.defaultGuessPlugins = list.build();
-    }
-
-    public ConfigDiff guess(ExecSession exec, final ConfigSource config)
-    {
-        try {
-            return Exec.doWith(exec, new ExecAction<ConfigDiff>() {
-                public ConfigDiff run()
-                {
-                    try (SetCurrentThreadName dontCare = new SetCurrentThreadName("guess")) {
-                        return doGuess(config);
-                    }
-                }
-            });
-        } catch (ExecutionException ex) {
-            throw Throwables.propagate(ex.getCause());
-        }
-    }
-
-    protected InputPlugin newInputPlugin(ConfigSource inputConfig)
-    {
-        return Exec.newPlugin(InputPlugin.class, inputConfig.get(PluginType.class, "type"));
-    }
-
-    private ConfigDiff doGuess(ConfigSource config)
-    {
-        ConfigSource inputConfig = config.getNested("in");
-        ConfigSource execConfig = config.getNestedOrGetEmpty("exec");
-
-        InputPlugin input = newInputPlugin(inputConfig);
-
-        ConfigDiff inputGuessed;
-        if (input instanceof ConfigurableGuessInputPlugin) {
-            inputGuessed = ((ConfigurableGuessInputPlugin) input).guess(execConfig, inputConfig);
-        }
-        else {
-            try {
-                inputGuessed = input.guess(inputConfig);
-            } catch (AbstractMethodError ex) {
-                // for backward compatibility with embulk v0.4 interface
-                throw new UnsupportedOperationException(input.getClass().getSimpleName()+".guess(ConfigSource) is not implemented. This input plugin does not support guessing.");
-            }
-        }
-
-        ConfigDiff wrapped = Exec.newConfigDiff();
-        wrapped.getNestedOrSetEmpty("in").merge(inputGuessed);
-        return wrapped;
-    }
-
-    // called by FileInputRunner
-    public ConfigDiff guessParserConfig(Buffer sample, ConfigSource inputConfig, ConfigSource execConfig)
-    {
-        List<PluginType> guessPlugins = new ArrayList<PluginType>(defaultGuessPlugins);
-
-        GuessExecutorTask task = execConfig.loadConfig(GuessExecutorTask.class);
-        guessPlugins.addAll(task.getGuessPlugins());
-        guessPlugins.removeAll(task.getExcludeGuessPlugins());
-
-        return guessParserConfig(sample, inputConfig, guessPlugins);
-    }
-
-    private ConfigDiff guessParserConfig(Buffer sample,
-            ConfigSource config, List<PluginType> guessPlugins)
-    {
-        // repeat guessing upto 10 times
-        ConfigDiff lastGuessed = Exec.newConfigDiff();
-        for (int i=0; i < 10; i++) {
-            // include last-guessed config to run guess input
-            ConfigSource originalConfig = config.deepCopy().merge(lastGuessed);
-            ConfigSource guessInputConfig = originalConfig.deepCopy();
-            guessInputConfig.getNestedOrSetEmpty("parser")
-                .set("type", "system_guess")  // override in.parser.type so that FileInputRunner.run uses GuessParserPlugin
-                .set("guess_plugins", guessPlugins)
-                .set("orig_config", originalConfig);
-
-            // run FileInputPlugin
-            final FileInputRunner input = new FileInputRunner(new BufferFileInputPlugin(sample));
-            ConfigDiff guessed;
-            try {
-                input.transaction(guessInputConfig, new InputPlugin.Control() {
-                    public List<TaskReport> run(TaskSource inputTaskSource, Schema schema, int taskCount)
-                    {
-                        if (taskCount == 0) {
-                            throw new NoSampleException("No input files to guess");
-                        }
-                        input.run(inputTaskSource, null, 0, new PageOutput() {
-                            @Override
-                            public void add(Page page)
-                            {
-                                throw new RuntimeException("Input plugin must be a FileInputPlugin to guess parser configuration");  // TODO exception class
-                            }
-
-                            @Override
-                            public void finish() { }
-
-                            @Override
-                            public void close() { }
-                        });
-                        throw new AssertionError("Guess executor must throw GuessedNoticeError");
-                    }
-                });
-
-                throw new AssertionError("Guess executor must throw GuessedNoticeError");
-
-            } catch (GuessedNoticeError error) {
-                guessed = lastGuessed.deepCopy().merge(error.getGuessedConfig());
-            }
-
-            // merge to the last-guessed config
-            if (lastGuessed.equals(guessed)) {
-                // not changed
-                return lastGuessed;
-            }
-            lastGuessed = guessed;
-        }
-
-        return lastGuessed;
-    }
-
-    public static class GuessParserPlugin
-            implements ParserPlugin
-    {
-        private interface PluginTask
-                extends Task
-        {
-            @Config("guess_plugins")
-            public List<PluginType> getGuessPluginTypes();
-
-            @Config("orig_config")
-            public ConfigSource getOriginalConfig();
-        }
-
-        @Override
-        public void transaction(ConfigSource config, ParserPlugin.Control control)
-        {
-            PluginTask task = config.loadConfig(PluginTask.class);
-            control.run(task.dump(), null);
-        }
-
-        @Override
-        public void run(TaskSource taskSource, Schema schema,
-                FileInput input, PageOutput pageOutput)
-        {
-            PluginTask task = taskSource.loadTask(PluginTask.class);
-            final ConfigSource originalConfig = task.getOriginalConfig();
-
-            // get sample buffer
-            Buffer sample = readSample(input, 32*1024);  // TODO get sample size from system config. See also SamplingParserPlugin().
-
-            // load guess plugins
-            ImmutableList.Builder<GuessPlugin> builder = ImmutableList.builder();
-            for (PluginType guessType : task.getGuessPluginTypes()) {
-                GuessPlugin guess = Exec.newPlugin(GuessPlugin.class, guessType);
-                builder.add(guess);
-            }
-            List<GuessPlugin> guesses = builder.build();
-
-            // run guess plugins
-            ConfigSource mergedConfig = originalConfig.deepCopy();
-            ConfigDiff mergedGuessed = Exec.newConfigDiff();
-            for (int i=0; i < guesses.size(); i++) {
-                ConfigDiff guessed = guesses.get(i).guess(originalConfig, sample);
-                guessed = addAssumedDecoderConfigs(originalConfig, guessed);
-                mergedGuessed.merge(guessed);
-                mergedConfig.merge(mergedGuessed);
-                if (!mergedConfig.equals(originalConfig)) {
-                    // config updated
-                    throw new GuessedNoticeError(mergedGuessed);
-                }
-            }
-            throw new GuessedNoticeError(mergedGuessed);
-        }
-
-        private static Buffer readSample(FileInput fileInput, int sampleSize)
-        {
-            Buffer sample = Buffer.allocate(sampleSize);
-            try {
-                SamplingParserPlugin.readSample(fileInput, sample, 0, sampleSize);
-            } catch (RuntimeException ex) {
-                // ignores exceptions because FileDecoderPlugin can throw exceptions
-                // such as "Unexpected end of ZLIB input stream" if decoder plugin
-                // is wrongly guessed.
-            }
-            if (sample.limit() > 0) {
-                return sample;
-            }
-            throw new NoSampleException("No input buffer to guess");
-        }
-
-        private static class ConfigSourceList extends ArrayList<ConfigSource> { };
-
-        private static ConfigDiff addAssumedDecoderConfigs(ConfigSource originalConfig, ConfigDiff guessed)
-        {
-            List<ConfigSource> guessedDecoders = guessed.get(ConfigSourceList.class, "decoders", null);
-            if (guessedDecoders == null) {
-                return guessed;
-            } else {
-                List<ConfigSource> assumedDecoders = originalConfig.get(ConfigSourceList.class, "decoders", new ConfigSourceList());
-                ImmutableList.Builder<ConfigSource> added = ImmutableList.builder();
-                for (ConfigSource assuemed : assumedDecoders) {
-                    added.add(Exec.newConfigSource());
-                }
-                added.addAll(guessedDecoders);
-                return guessed.set("decoders", added.build());
-            }
-        }
-    }
-
-    public static class GuessedNoticeError
-            extends Error
-    {
-        private final ConfigDiff guessedConfig;
-
-        public GuessedNoticeError(ConfigDiff guessedConfig)
-        {
-            this.guessedConfig = guessedConfig;
-        }
-
-        public ConfigDiff getGuessedConfig()
-        {
-            return guessedConfig;
-        }
-    }
-}