embulk 0.8.25-java → 0.8.26-java

data/embulk-core/src/main/java/org/embulk/plugin/maven/MavenArtifactFinder.java

@@ -0,0 +1,134 @@
+package org.embulk.plugin.maven;
+
+import java.io.IOError;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.NotDirectoryException;
+import java.nio.file.Path;
+import org.apache.maven.repository.internal.MavenRepositorySystemUtils;
+import org.eclipse.aether.DefaultRepositorySystemSession;
+import org.eclipse.aether.RepositorySystem;
+import org.eclipse.aether.RepositorySystemSession;
+import org.eclipse.aether.artifact.Artifact;
+import org.eclipse.aether.artifact.DefaultArtifact;
+import org.eclipse.aether.impl.DefaultServiceLocator;
+import org.eclipse.aether.repository.LocalRepository;
+import org.eclipse.aether.resolution.ArtifactRequest;
+import org.eclipse.aether.resolution.ArtifactResolutionException;
+import org.eclipse.aether.resolution.ArtifactResult;
+
+public class MavenArtifactFinder
+{
+    private MavenArtifactFinder(final Path givenLocalMavenRepositoryPath,
+                                final Path absoluteLocalMavenRepositoryPath,
+                                final RepositorySystem repositorySystem,
+                                final RepositorySystemSession repositorySystemSession)
+    {
+        this.givenLocalMavenRepositoryPath = givenLocalMavenRepositoryPath;
+        this.absoluteLocalMavenRepositoryPath = absoluteLocalMavenRepositoryPath;
+        this.repositorySystem = repositorySystem;
+        this.repositorySystemSession = repositorySystemSession;
+    }
+
+    public static MavenArtifactFinder create(final Path localMavenRepositoryPath)
+        throws MavenRepositoryNotFoundException
+    {
+        final Path absolutePath;
+        try {
+            absolutePath = localMavenRepositoryPath.normalize().toAbsolutePath();
+        }
+        catch (IOError ex) {
+            throw new MavenRepositoryNotFoundException(localMavenRepositoryPath, ex);
+        }
+        catch (SecurityException ex) {
+            throw new MavenRepositoryNotFoundException(localMavenRepositoryPath, ex);
+        }
+
+        if (!Files.exists(absolutePath)) {
+            throw new MavenRepositoryNotFoundException(localMavenRepositoryPath,
+                                                       absolutePath,
+                                                       new NoSuchFileException(absolutePath.toString()));
+        }
+        if (!Files.isDirectory(absolutePath)) {
+            throw new MavenRepositoryNotFoundException(localMavenRepositoryPath,
+                                                       absolutePath,
+                                                       new NotDirectoryException(absolutePath.toString()));
+        }
+
+        final RepositorySystem repositorySystem = createRepositorySystem();
+
+        return new MavenArtifactFinder(localMavenRepositoryPath,
+                                       absolutePath,
+                                       repositorySystem,
+                                       createRepositorySystemSession(repositorySystem, absolutePath));
+    }
+
+    public final Path findMavenArtifactJar(
+            final String groupId,
+            final String artifactId,
+            final String classifier,
+            final String version)
+        throws MavenArtifactNotFoundException
+    {
+        return findMavenArtifact(groupId, artifactId, classifier, "jar", version);
+    }
+
+    public Path findMavenArtifact(
+            final String groupId,
+            final String artifactId,
+            final String classifier,
+            final String extension,
+            final String version)
+        throws MavenArtifactNotFoundException
+    {
+        final ArtifactResult result;
+        try {
+            result = resolveMavenArtifact(groupId, artifactId, classifier, extension, version);
+        }
+        catch (ArtifactResolutionException ex) {
+            throw new MavenArtifactNotFoundException(groupId, artifactId, classifier, version,
+                                                     this.givenLocalMavenRepositoryPath,
+                                                     this.absoluteLocalMavenRepositoryPath,
+                                                     ex);
+        }
+        return result.getArtifact().getFile().toPath();
+    }
+
+    private ArtifactResult resolveMavenArtifact(
+            final String groupId,
+            final String artifactId,
+            final String classifier,
+            final String extension,
+            final String version)
+        throws ArtifactResolutionException
+    {
+        // |classifier| can be null for |org.eclipse.aether.artifact.DefaultArtifact|.
+        final ArtifactRequest artifactRequest = new ArtifactRequest().setArtifact(
+            new DefaultArtifact(groupId, artifactId, classifier, extension, version));
+
+        return this.repositorySystem.resolveArtifact(this.repositorySystemSession, artifactRequest);
+    }
+
+    private static RepositorySystem createRepositorySystem()
+    {
+        final DefaultServiceLocator locator = MavenRepositorySystemUtils.newServiceLocator();
+        return locator.getService(RepositorySystem.class);
+    }
+
+    private static RepositorySystemSession createRepositorySystemSession(
+            final RepositorySystem repositorySystem, final Path localRepositoryPath)
+    {
+        final DefaultRepositorySystemSession session = MavenRepositorySystemUtils.newSession();
+
+        final LocalRepository repository = new LocalRepository(localRepositoryPath.toString());
+        session.setLocalRepositoryManager(repositorySystem.newLocalRepositoryManager(session, repository));
+        return session;
+    }
+
+    // Paths are kept just for hinting in Exceptions.
+    private final Path givenLocalMavenRepositoryPath;
+    private final Path absoluteLocalMavenRepositoryPath;
+
+    private final RepositorySystem repositorySystem;
+    private final RepositorySystemSession repositorySystemSession;
+}

data/embulk-core/src/main/java/org/embulk/plugin/maven/MavenArtifactNotFoundException.java

@@ -0,0 +1,20 @@
+package org.embulk.plugin.maven;
+
+import java.nio.file.Path;
+
+public class MavenArtifactNotFoundException
+        extends Exception {
+    public MavenArtifactNotFoundException(final String groupId,
+                                          final String artifactId,
+                                          final String classifier,
+                                          final String version,
+                                          final Path givenRepositoryPath,
+                                          final Path absoluteRepositoryPath,
+                                          final Throwable cause)
+    {
+        super("Maven artifact \"" + groupId + ":" + artifactId + ":" + version +
+              (classifier != null ? (":" + classifier) : "") + "\" is not found: at \"" +
+              givenRepositoryPath.toString() + "\" (\"" + absoluteRepositoryPath.toString() + "\").",
+              cause);
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/maven/MavenPluginSource.java

@@ -0,0 +1,187 @@
+package org.embulk.plugin.maven;
+
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.embulk.plugin.MavenPluginType;
+import org.embulk.plugin.PluginClassLoaderFactory;
+import org.embulk.plugin.PluginSource;
+import org.embulk.plugin.PluginSourceNotMatchException;
+import org.embulk.plugin.PluginType;
+import org.embulk.plugin.jar.JarPluginLoader;
+import org.embulk.plugin.jar.InvalidJarPluginException;
+import org.embulk.spi.DecoderPlugin;
+import org.embulk.spi.EncoderPlugin;
+import org.embulk.spi.ExecutorPlugin;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.FileInputRunner;
+import org.embulk.spi.FileOutputPlugin;
+import org.embulk.spi.FileOutputRunner;
+import org.embulk.spi.FilterPlugin;
+import org.embulk.spi.FormatterPlugin;
+import org.embulk.spi.GuessPlugin;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.ParserPlugin;
+
+public class MavenPluginSource
+        implements PluginSource
+{
+    @Inject
+    public MavenPluginSource(Injector injector)
+    {
+        this.injector = injector;
+    }
+
+    @Override
+    public <T> T newPlugin(Class<T> pluginInterface, PluginType pluginType)
+            throws PluginSourceNotMatchException
+    {
+        final String category;
+        if (InputPlugin.class.isAssignableFrom(pluginInterface)) {
+            category = "input";
+        } else if (OutputPlugin.class.isAssignableFrom(pluginInterface)) {
+            category = "output";
+        } else if (ParserPlugin.class.isAssignableFrom(pluginInterface)) {
+            category = "parser";
+        } else if (FormatterPlugin.class.isAssignableFrom(pluginInterface)) {
+            category = "formatter";
+        } else if (DecoderPlugin.class.isAssignableFrom(pluginInterface)) {
+            category = "decoder";
+        } else if (EncoderPlugin.class.isAssignableFrom(pluginInterface)) {
+            category = "encoder";
+        } else if (FilterPlugin.class.isAssignableFrom(pluginInterface)) {
+            category = "filter";
+        } else if (GuessPlugin.class.isAssignableFrom(pluginInterface)) {
+            category = "guess";
+        } else if (ExecutorPlugin.class.isAssignableFrom(pluginInterface)) {
+            category = "executor";
+        } else {
+            // unsupported plugin category
+            throw new PluginSourceNotMatchException("Plugin interface " + pluginInterface + " is not supported.");
+        }
+
+        if (pluginType.getSourceType() != PluginSource.Type.MAVEN) {
+            throw new PluginSourceNotMatchException();
+        }
+        final MavenPluginType mavenPluginType = (MavenPluginType) pluginType;
+
+        final PluginClassLoaderFactory pluginClassLoaderFactory =
+            this.injector.getInstance(PluginClassLoaderFactory.class);
+
+        final MavenArtifactFinder mavenArtifactFinder;
+        try {
+            mavenArtifactFinder = MavenArtifactFinder.create(getLocalMavenRepository());
+        }
+        catch (MavenRepositoryNotFoundException ex) {
+            throw new PluginSourceNotMatchException(ex);
+        }
+
+        final Path jarPath;
+        try {
+            jarPath = mavenArtifactFinder.findMavenArtifactJar(
+                mavenPluginType.getGroup(),
+                "embulk-" + category + "-" + mavenPluginType.getName(),
+                mavenPluginType.getClassifier(),
+                mavenPluginType.getVersion());
+        }
+        catch (MavenArtifactNotFoundException ex) {
+            throw new PluginSourceNotMatchException(ex);
+        }
+
+        final Class<?> pluginMainClass;
+        try (JarPluginLoader loader = JarPluginLoader.load(jarPath, pluginClassLoaderFactory)) {
+            pluginMainClass = loader.getPluginMainClass();
+        }
+        catch (InvalidJarPluginException ex) {
+            throw new PluginSourceNotMatchException(ex);
+        }
+
+        final Object pluginMainObject;
+        try {
+            // Unlike InjectedPluginSource and JRubyPluginSource,
+            // MavenPluginSource does not have "registration" before creating an instance of the plugin class.
+            // FileInputPlugin and FileOutputPlugin are wrapped with FileInputRunner and FileOutputRunner here.
+            if (FileInputPlugin.class.isAssignableFrom(pluginMainClass)) {
+                final FileInputPlugin fileInputPluginMainObject;
+                try {
+                    fileInputPluginMainObject = (FileInputPlugin) this.injector.getInstance(pluginMainClass);
+                }
+                catch (ClassCastException ex) {
+                    throw new PluginSourceNotMatchException(
+                        "[FATAL/INTERNAL] Plugin class \"" + pluginMainClass.getName() + "\" is not file-input.", ex);
+                }
+                pluginMainObject = new FileInputRunner(fileInputPluginMainObject);
+            } else if (FileOutputPlugin.class.isAssignableFrom(pluginMainClass)) {
+                final FileOutputPlugin fileOutputPluginMainObject;
+                try {
+                    fileOutputPluginMainObject = (FileOutputPlugin) this.injector.getInstance(pluginMainClass);
+                }
+                catch (ClassCastException ex) {
+                    throw new PluginSourceNotMatchException(
+                        "[FATAL/INTERNAL] Plugin class \"" + pluginMainClass.getName() + "\" is not file-output.", ex);
+                }
+                pluginMainObject = new FileOutputRunner(fileOutputPluginMainObject);
+            } else {
+                if (!pluginInterface.isAssignableFrom(pluginMainClass)) {
+                    throw new PluginSourceNotMatchException(
+                        "Plugin class \"" + pluginMainClass.getName() + "\" is not a valid " + category + " plugin.");
+                }
+                pluginMainObject = this.injector.getInstance(pluginMainClass);
+            }
+        }
+        catch (ExceptionInInitializerError ex) {
+            throw new PluginSourceNotMatchException(
+                "Plugin class \"" + pluginMainClass.getName() +
+                "\" is not instantiatable due to exception in initialization.", ex);
+        }
+        catch (SecurityException ex) {
+            throw new PluginSourceNotMatchException(
+                "Plugin class \"" + pluginMainClass.getName() +
+                "\" is not instantiatable due to security manager.", ex);
+        }
+
+        try {
+            return pluginInterface.cast(pluginMainObject);
+        }
+        catch (ClassCastException ex) {
+            throw new PluginSourceNotMatchException(
+                "[FATAL/INTERNAL] Plugin class \"" + pluginMainClass.getName() + "\" is not " + category + " actually.",
+                ex);
+        }
+    }
+
+    private Path getLocalMavenRepository()
+            throws PluginSourceNotMatchException
+    {
+        String env;
+        try {
+            env = System.getenv("M2_REPO");
+        }
+        catch (NullPointerException | SecurityException ex) {
+            // The Exceptions are just ignored, and the default local Maven repository is used.
+            // TODO: Log?
+            env = null;
+        }
+
+        if (env == null) {
+            return getEmbulkHome().resolve("m2").resolve("repository");
+        }
+
+        return Paths.get(env);
+    }
+
+    private Path getEmbulkHome()
+            throws PluginSourceNotMatchException
+    {
+        final String propertyHome = System.getProperty("user.home");
+        if (propertyHome == null) {
+            throw new PluginSourceNotMatchException();
+        }
+
+        return Paths.get(propertyHome, ".embulk");
+    }
+
+    private final Injector injector;
+}

data/embulk-core/src/main/java/org/embulk/plugin/maven/MavenPluginSourceModule.java

@@ -0,0 +1,22 @@
+package org.embulk.plugin.maven;
+
+import com.google.inject.Binder;
+import com.google.inject.Module;
+import com.google.inject.multibindings.Multibinder;
+import org.embulk.config.ConfigSource;
+import org.embulk.plugin.PluginSource;
+
+public class MavenPluginSourceModule
+        implements Module
+{
+    public MavenPluginSourceModule(ConfigSource systemConfig)
+    {
+    }
+
+    @Override
+    public void configure(Binder binder)
+    {
+        Multibinder<PluginSource> multibinder = Multibinder.newSetBinder(binder, PluginSource.class);
+        multibinder.addBinding().to(MavenPluginSource.class);
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/maven/MavenRepositoryNotFoundException.java

@@ -0,0 +1,31 @@
+package org.embulk.plugin.maven;
+
+import java.nio.file.Path;
+
+public class MavenRepositoryNotFoundException
+        extends Exception {
+    public MavenRepositoryNotFoundException(final Path givenPath,
+                                            final Throwable cause)
+    {
+        super("Maven repository specified is not found at \"" + givenPath.toString() + "\".", cause);
+    }
+
+    public MavenRepositoryNotFoundException(final Path givenPath,
+                                            final Path absolutePath,
+                                            final Throwable cause)
+    {
+        super("Maven repository specified is not found at \"" + givenPath.toString() +
+              "\" (\"" + absolutePath.toString() + "\").",
+              cause);
+    }
+
+    public MavenRepositoryNotFoundException(final String message,
+                                            final Path givenPath,
+                                            final Path absolutePath,
+                                            final Throwable cause)
+    {
+        super("Maven repository specified is not found at \"" + givenPath.toString() +
+              "\" (\"" + absolutePath.toString() + "\"): " + message,
+              cause);
+    }
+}

data/embulk-core/src/main/resources/embulk/parent_first_packages.properties

@@ -1,4 +1,4 @@
-# generated by './gradlew updateResources' at 2015-06-30 23:52:09 UTC
+# generated by './gradlew updateResources' at 2017-06-24 00:47:26 UTC
 ch.qos.logback.classic
 ch.qos.logback.core
 com.fasterxml.jackson.annotation
@@ -65,6 +65,8 @@ org.jcodings
 org.joda.time
 org.joni
 org.jruby
+org.msgpack.core
+org.msgpack.value
 org.slf4j
 org.yaml.snakeyaml
 org.yecht

data/embulk-core/src/main/resources/embulk/parent_first_resources.properties

@@ -1,4 +1,4 @@
-# generated by './gradlew updateResources' at 2015-06-30 23:52:09 UTC
+# generated by './gradlew updateResources' at 2017-06-24 00:47:26 UTC
 ch/qos/logback/classic/boolex
 ch/qos/logback/classic/db/script
 com/ibm/icu
@@ -18,6 +18,7 @@ jni/x86_64-Linux
 jni/x86_64-SunOS
 jni/x86_64-Windows
 jruby
+msgpack
 net/jcip/annotations
 okay
 org/apache/bval/jsr303

data/embulk-core/src/test/java/org/embulk/EmbulkTestRuntime.java

@@ -13,6 +13,8 @@ import org.embulk.exec.SystemConfigModule;
 import org.embulk.exec.ExecModule;
 import org.embulk.exec.ExtensionServiceLoaderModule;
 import org.embulk.plugin.BuiltinPluginSourceModule;
+import org.embulk.plugin.PluginClassLoaderFactory;
+import org.embulk.plugin.PluginClassLoaderModule;
 import org.embulk.jruby.JRubyScriptingModule;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Exec;
@@ -40,6 +42,7 @@ public class EmbulkTestRuntime
             new ExtensionServiceLoaderModule(systemConfig).configure(binder);
             new BuiltinPluginSourceModule().configure(binder);
             new JRubyScriptingModule(systemConfig).configure(binder);
+            new PluginClassLoaderModule(systemConfig).configure(binder);
             new TestUtilityModule().configure(binder);
             new TestPluginSourceModule().configure(binder);
         }
@@ -75,6 +78,11 @@ public class EmbulkTestRuntime
         return getInstance(RandomManager.class).getRandom();
     }
 
+    public PluginClassLoaderFactory getPluginClassLoaderFactory()
+    {
+        return getInstance(PluginClassLoaderFactory.class);
+    }
+
     @Override
     public Statement apply(Statement base, Description description)
     {