embulk 0.7.0-java

data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderFactory.java

@@ -0,0 +1,9 @@
+package org.embulk.plugin;
+
+import java.util.Collection;
+import java.net.URL;
+
+public interface PluginClassLoaderFactory
+{
+    PluginClassLoader create(Collection<URL> urls, ClassLoader parentClassLoader);
+}

data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderModule.java

@@ -0,0 +1,71 @@
+package org.embulk.plugin;
+
+import java.util.Collection;
+import java.util.Properties;
+import java.net.URL;
+import java.io.InputStream;
+import java.io.IOException;
+import com.google.inject.Module;
+import com.google.inject.Binder;
+import com.google.inject.Scopes;
+import com.google.inject.Provider;
+import org.embulk.config.ConfigSource;
+
+public class PluginClassLoaderModule
+        implements Module
+{
+    public PluginClassLoaderModule(ConfigSource systemConfig)
+    { }
+
+    @Override
+    public void configure(Binder binder)
+    {
+        binder.bind(PluginClassLoaderFactory.class).toProvider(new FactoryProvider()).in(Scopes.SINGLETON);;
+    }
+
+    private static class FactoryProvider
+            implements Provider<PluginClassLoaderFactory>
+    {
+        private final Collection<String> parentFirstPackages;
+        private final Collection<String> parentFirstResources;
+        private final PluginClassLoaderFactory factory;
+
+        public FactoryProvider()
+        {
+            // TODO make these paths customizable using ConfigSource
+            this.parentFirstPackages = readPropertyKeys("/embulk/parent_first_packages.properties");
+            this.parentFirstResources = readPropertyKeys("/embulk/parent_first_resources.properties");
+
+            this.factory = new Factory();
+        }
+
+        private static Collection<String> readPropertyKeys(String name)
+        {
+            try (InputStream in = PluginClassLoaderModule.class.getResourceAsStream(name)) {
+                if (in == null) {
+                    throw new NullPointerException(String.format("Resource '%s' is not found in classpath. Jar file or classloader is broken.", name));
+                }
+                Properties prop = new Properties();
+                prop.load(in);
+                return prop.stringPropertyNames();
+            } catch (IOException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+
+        @Override
+        public PluginClassLoaderFactory get()
+        {
+            return factory;
+        }
+
+        private class Factory implements PluginClassLoaderFactory
+        {
+            public PluginClassLoader create(Collection<URL> urls, ClassLoader parentClassLoader)
+            {
+                return new PluginClassLoader(urls, parentClassLoader,
+                        parentFirstPackages, parentFirstResources);
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/PluginManager.java

@@ -0,0 +1,78 @@
+package org.embulk.plugin;
+
+import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import org.embulk.config.ConfigException;
+import org.embulk.spi.InputPlugin;
+import org.embulk.plugin.compat.PluginWrappers;
+
+public class PluginManager
+{
+    private final List<PluginSource> sources;
+    private final Injector injector;
+
+    // Set<PluginSource> is injected by BuiltinPluginSourceModule or extensions
+    // using Multibinder<PluginSource>.
+    @Inject
+    public PluginManager(Set<PluginSource> pluginSources, Injector injector)
+    {
+        this.sources = ImmutableList.copyOf(pluginSources);
+        this.injector = injector;
+    }
+
+    @SuppressWarnings("unchecked")
+    public <T> T newPlugin(Class<T> iface, PluginType type)
+    {
+        T t = newPluginWithoutWrapper(iface, type);
+        if (t instanceof InputPlugin) {
+            return (T) PluginWrappers.inputPlugin((InputPlugin) t);
+        }
+        return t;
+    }
+
+    private <T> T newPluginWithoutWrapper(Class<T> iface, PluginType type)
+    {
+        if (sources.isEmpty()) {
+            throw new ConfigException("No PluginSource is installed");
+        }
+
+        if (type == null) {
+            throw new ConfigException(String.format("%s type is not set (if you intend to use NullOutputPlugin, you should enclose null in quotes such as {type: \"null\"}.", iface.getSimpleName()));
+        }
+
+        List<PluginSourceNotMatchException> exceptions = new ArrayList<>();
+        for (PluginSource source : sources) {
+            try {
+                return source.newPlugin(iface, type);
+            }
+            catch (PluginSourceNotMatchException e) {
+                exceptions.add(e);
+            }
+        }
+
+        throw buildPluginNotFoundException(iface, type, exceptions);
+    }
+
+    private static ConfigException buildPluginNotFoundException(Class<?> iface, PluginType type,
+            List<PluginSourceNotMatchException> exceptions)
+    {
+        StringBuilder message = new StringBuilder();
+        message.append(String.format("%s '%s' is not found.", iface.getSimpleName(), type.getName()));
+        for (PluginSourceNotMatchException exception : exceptions) {
+            Throwable cause = (exception.getCause() == null ? exception : exception.getCause());
+            if (cause.getMessage() != null) {
+                message.append(String.format("%n"));
+                message.append(cause.getMessage());
+            }
+        }
+        ConfigException e = new ConfigException(message.toString());
+        for (PluginSourceNotMatchException exception : exceptions) {
+            e.addSuppressed(exception);
+        }
+        return e;
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/PluginSource.java

@@ -0,0 +1,6 @@
+package org.embulk.plugin;
+
+public interface PluginSource
+{
+    <T> T newPlugin(Class<T> iface, PluginType type) throws PluginSourceNotMatchException;
+}

data/embulk-core/src/main/java/org/embulk/plugin/PluginSourceNotMatchException.java

@@ -0,0 +1,19 @@
+package org.embulk.plugin;
+
+public class PluginSourceNotMatchException
+        extends Exception
+{
+    public PluginSourceNotMatchException()
+    {
+    }
+
+    public PluginSourceNotMatchException(String message)
+    {
+        super(message);
+    }
+
+    public PluginSourceNotMatchException(Throwable cause)
+    {
+        super(cause);
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/PluginType.java

@@ -0,0 +1,47 @@
+package org.embulk.plugin;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+public class PluginType
+{
+    private final String name;
+
+    // TODO accept isObject()/ObjectNode for complex PluginSource
+    @JsonCreator
+    public PluginType(String name)
+    {
+        if (name == null) {
+            throw new NullPointerException("name must not be null");
+        }
+        this.name = name;
+    }
+
+    @JsonValue
+    public String getName()
+    {
+        return name;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return name.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object other)
+    {
+        if (!(other instanceof PluginType)) {
+            return false;
+        }
+        PluginType o = (PluginType) other;
+        return name.equals(o.name);
+    }
+
+    @Override
+    public String toString()
+    {
+        return name;
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/compat/InputPluginWrapper.java

@@ -0,0 +1,102 @@
+package org.embulk.plugin.compat;
+
+import java.util.List;
+import java.lang.reflect.Method;
+import java.lang.reflect.InvocationTargetException;
+import com.google.common.base.Throwables;
+import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.Schema;
+import org.embulk.spi.InputPlugin;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.TaskSource;
+
+public class InputPluginWrapper
+        implements InputPlugin
+{
+    public static InputPlugin wrapIfNecessary(InputPlugin object)
+    {
+        Method runMethod = wrapRunMethod(object);
+        if (runMethod != null) {
+            return new InputPluginWrapper(object, runMethod);
+        }
+        return object;
+    }
+
+    private static Method wrapRunMethod(InputPlugin object)
+    {
+        try {
+            Method m = object.getClass().getMethod("run", TaskSource.class, Schema.class, int.class, PageOutput.class);
+            if (m.getReturnType().equals(CommitReport.class)) {
+                return m;
+            } else {
+                return null;
+            }
+        }
+        catch (NoSuchMethodException ex) {
+            return null;
+        }
+    }
+
+    private final InputPlugin object;
+    private final Method runMethod;
+
+    private InputPluginWrapper(InputPlugin object,
+            Method runMethod)
+    {
+        this.object = object;
+        this.runMethod = runMethod;
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config,
+            InputPlugin.Control control)
+    {
+        return object.transaction(config, control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            Schema schema, int taskCount,
+            InputPlugin.Control control)
+    {
+        return object.resume(taskSource, schema, taskCount, control);
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+            Schema schema, int taskCount,
+            List<TaskReport> successTaskReports)
+    {
+        object.cleanup(taskSource, schema, taskCount, successTaskReports);
+    }
+
+    @Override
+    public TaskReport run(TaskSource taskSource,
+            Schema schema, int taskIndex,
+            PageOutput output)
+    {
+        if (runMethod != null) {
+            try {
+                return (TaskReport) runMethod.invoke(object, taskSource, schema, taskIndex, output);
+            }
+            catch (IllegalAccessException | IllegalArgumentException ex) {
+                throw Throwables.propagate(ex);
+            }
+            catch (InvocationTargetException ex) {
+                throw Throwables.propagate(ex.getCause());
+            }
+
+        } else {
+            return object.run(taskSource, schema, taskIndex, output);
+        }
+    }
+
+    @Override
+    public ConfigDiff guess(ConfigSource config)
+    {
+        return object.guess(config);
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/compat/PluginWrappers.java

@@ -0,0 +1,30 @@
+package org.embulk.plugin.compat;
+
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.TransactionalFileInput;
+import org.embulk.spi.TransactionalFileOutput;
+import org.embulk.spi.TransactionalPageOutput;
+import org.embulk.plugin.compat.InputPluginWrapper;
+
+public class PluginWrappers
+{
+    public static InputPlugin inputPlugin(InputPlugin input)
+    {
+        return InputPluginWrapper.wrapIfNecessary(input);
+    }
+
+    public static TransactionalFileInput transactionalFileInput(TransactionalFileInput tran)
+    {
+        return TransactionalFileInputWrapper.wrapIfNecessary(tran);
+    }
+
+    public static TransactionalFileOutput transactionalFileOutput(TransactionalFileOutput tran)
+    {
+        return TransactionalFileOutputWrapper.wrapIfNecessary(tran);
+    }
+
+    public static TransactionalPageOutput transactionalPageOutput(TransactionalPageOutput tran)
+    {
+        return TransactionalPageOutputWrapper.wrapIfNecessary(tran);
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalFileInputWrapper.java

@@ -0,0 +1,96 @@
+package org.embulk.plugin.compat;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.InvocationTargetException;
+import com.google.common.base.Throwables;
+import org.embulk.config.TaskReport;
+import org.embulk.config.CommitReport;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.TransactionalFileInput;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TransactionalFileInputWrapper
+        implements TransactionalFileInput
+{
+    private final Logger logger = LoggerFactory.getLogger(PluginWrappers.class);
+
+    public static TransactionalFileInput wrapIfNecessary(TransactionalFileInput object)
+    {
+        Method runMethod = wrapCommitMethod(object);
+        if (runMethod != null) {
+            return new TransactionalFileInputWrapper(object, runMethod);
+        }
+        return object;
+    }
+
+    private static Method wrapCommitMethod(TransactionalFileInput object)
+    {
+        try {
+            Method m = object.getClass().getMethod("commit");
+            if (m.getReturnType().equals(CommitReport.class)) {
+                return m;
+            } else {
+                return null;
+            }
+        }
+        catch (NoSuchMethodException ex) {
+            return null;
+        }
+    }
+
+    private final TransactionalFileInput object;
+    private final Method commitMethod;
+
+    private TransactionalFileInputWrapper(TransactionalFileInput object,
+            Method commitMethod)
+    {
+        this.object = object;
+        this.commitMethod = commitMethod;
+        logger.warn("A file input plugin is compiled with old Embulk plugin API. Please update the plugin version using \"embulk gem install\" command, or contact a developer of the plugin to upgrade the plugin code using \"embulk migrate\" command: {}", object.getClass());
+    }
+
+    @Override
+    public Buffer poll()
+    {
+        return object.poll();
+    }
+
+    @Override
+    public boolean nextFile()
+    {
+        return object.nextFile();
+    }
+
+    @Override
+    public void close()
+    {
+        object.close();
+    }
+
+    @Override
+    public void abort()
+    {
+        object.abort();
+    }
+
+    @Override
+    public TaskReport commit()
+    {
+        if (commitMethod != null) {
+            try {
+                return (TaskReport) commitMethod.invoke(object);
+            }
+            catch (IllegalAccessException | IllegalArgumentException ex) {
+                throw Throwables.propagate(ex);
+            }
+            catch (InvocationTargetException ex) {
+                throw Throwables.propagate(ex.getCause());
+            }
+
+        } else {
+            return object.commit();
+        }
+    }
+}
+