embulk 0.8.25-java → 0.8.26-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a2368360f1531d901df3fb7f5d0fe6069ad3362c
-  data.tar.gz: ef5f5347e9cfabb4ea31d9e29fd69695c9d4ebe4
+  metadata.gz: 67e9c5003f422b52f56fd7347f77dda2ea07d86e
+  data.tar.gz: ac13f8e09bcc9998ba716fc96e6e98e0546605ef
 SHA512:
-  metadata.gz: af58995886e862f1cc30dcbe23813da71f72bd443eb4b9af62584ff5c4aa49ab85a6c53d717e3258f331020e28f4e7b2ff9701c1fde24f1e58d93bff89705387
-  data.tar.gz: 791c332fcac833460e2a5458d72d8f937ec5881af769fe4100f1b30e225e7b6ed4db0c058f0eec5274f8d3cef56f86a73ff79e95d1987f5f479ad56330f77135
+  metadata.gz: 6a275c7189877da26bed04a116dab321b0bf5d9c8f378143aa743c2fa05ae0806fb8d27eabaad76c6bdd7b06971a1b8f5ea2aa8a52152aa7e67a129545e417bc
+  data.tar.gz: 6ce56d2cfec57fc2db846f7be7aaf5209e30ac5180b28f0d9930f7cebd333f614e443dcb1e57b49bc58683e4b1a595239809bbe50d4915f9bd21e5d59597abb7

data/build.gradle

@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards"), p
 
 allprojects {
     group = 'org.embulk'
-    version = '0.8.25'
+    version = '0.8.26'
 
     ext {
         jrubyVersion = '9.1.5.0'
@@ -100,6 +100,9 @@ subprojects {
         tasks.withType(JavaCompile) {
             options.compilerArgs << "-Xlint:unchecked" //<< "-Xlint:deprecation"
         }
+        tasks.withType(Test) {
+            systemProperties System.properties.findAll { it.key.startsWith("org.embulk") }
+        }
         tasks.withType(FindBugs) {
             reports {
                 xml.enabled = false
@@ -227,6 +230,18 @@ project(':embulk-cli') {
     apply plugin: 'com.github.johnrengelman.shadow'
 
     shadowJar {
+        // 'org.apache.maven' is used in:
+        // - embulk-cli: EmbulkMigrate and EmbulkSelfUpdate (from v0.8.17)
+        // - embulk-core: MavenPluginSource and the related (from v0.8.26)
+        relocate 'org.apache.maven', 'org.embulk.third_party.org.apache.maven'
+
+        // 'org.codehaus.plexus' is used from 'org.apache.maven'.
+        relocate 'org.codehaus.plexus', 'org.embulk.third_party.org.codehaus.plexus'
+
+        // 'org.eclipse.aether' is used in:
+        // - embulk-core: MavenPluginSource and the related (from v0.8.26)
+        relocate 'org.eclipse.aether', 'org.embulk.third_party.org.eclipse.aether'
+
         // NOTE: This 'Implementation-Version' in the manifest is referred to provide the Embulk version at runtime.
         // See also: embulk-core/src/main/java/org/embulk/EmbulkVersion.java
         manifest {

data/embulk-cli/src/main/java/org/embulk/cli/EmbulkMigrate.java

@@ -10,6 +10,7 @@ import java.nio.file.Path;
 import java.nio.file.PathMatcher;
 import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
+import java.nio.file.StandardCopyOption;
 import java.nio.file.StandardOpenOption;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
@@ -76,7 +77,7 @@ public class EmbulkMigrate
             System.out.println("Done. No files are modified.");
         }
         else {
-            System.out.println("Done. Please check modifieid files.");
+            System.out.println("Done. Please check modified files.");
         }
     }
 
@@ -236,7 +237,9 @@ public class EmbulkMigrate
         {
             Path destinationPath = this.basePath.resolve(destinationFileName);
             Files.createDirectories(destinationPath.getParent());
-            Files.copy(EmbulkMigrate.class.getClassLoader().getResourceAsStream(sourceResourcePath), destinationPath);
+            Files.copy(EmbulkMigrate.class.getClassLoader().getResourceAsStream(sourceResourcePath), destinationPath,
+                    StandardCopyOption.REPLACE_EXISTING);
+            this.modifiedFiles.add(destinationPath);
         }
 
         public boolean match(String filePath, Pattern pattern)
@@ -424,7 +427,7 @@ public class EmbulkMigrate
     private static final Pattern OLD_EMBULK_IN_GEMSPEC = Pattern.compile(
         "embulk-");
     private static final Pattern GRADLE_VERSION_IN_WRAPPER = Pattern.compile(
-        "gradle-[23]\\.\\d+(\\.\\d+)?-/");
+        "gradle-[23]\\.\\d+(\\.\\d+)?-");
     private static final Pattern JSON_COLUMN_METHOD_IN_ALL_JAVA = Pattern.compile(
         "void\\s+jsonColumn");
     private static final Pattern TIMESTAMP_COLUMN_METHOD_IN_ALL_JAVA = Pattern.compile(

data/embulk-core/build.gradle

@@ -43,6 +43,13 @@ dependencies {
     // For embulk/guess/charset.rb. See also embulk.gemspec
     compile 'com.ibm.icu:icu4j:54.1.1'
 
+    // For MavenPluginSource / MavenArtifactFinder.
+    compile 'org.eclipse.aether:aether-api:1.1.0'
+    compile 'org.eclipse.aether:aether-spi:1.1.0'
+    compile 'org.eclipse.aether:aether-util:1.1.0'
+    compile 'org.eclipse.aether:aether-impl:1.1.0'
+    compile 'org.apache.maven:maven-aether-provider:3.3.9'
+
     gems 'rubygems:bundler:1.10.6'
     gems 'rubygems:msgpack:1.1.0'
     gems 'rubygems:liquid:3.0.6'

data/embulk-core/src/main/java/org/embulk/EmbulkService.java

@@ -11,6 +11,7 @@ import org.embulk.exec.ExecModule;
 import org.embulk.exec.ExtensionServiceLoaderModule;
 import org.embulk.plugin.PluginClassLoaderModule;
 import org.embulk.plugin.BuiltinPluginSourceModule;
+import org.embulk.plugin.maven.MavenPluginSourceModule;
 import org.embulk.jruby.JRubyScriptingModule;
 import static com.google.common.base.Preconditions.checkState;
 
@@ -45,6 +46,7 @@ public class EmbulkService
                 new ExtensionServiceLoaderModule(systemConfig),
                 new PluginClassLoaderModule(systemConfig),
                 new BuiltinPluginSourceModule(),
+                new MavenPluginSourceModule(systemConfig),
                 new JRubyScriptingModule(systemConfig));
     }
 

data/embulk-core/src/main/java/org/embulk/plugin/InjectedPluginSource.java

@@ -37,7 +37,7 @@ public class InjectedPluginSource
         this.injector = injector;
     }
 
-    public static interface PluginFactory <T>
+    private static interface PluginFactory <T>
     {
         public T newPlugin(Injector injector);
     }

data/embulk-core/src/main/java/org/embulk/plugin/MavenPluginType.java

@@ -10,10 +10,11 @@ import java.util.Objects;
 public final class MavenPluginType
         extends PluginType
 {
-    private MavenPluginType(final String name, final String group, final String version)
+    private MavenPluginType(final String name, final String group, final String classifier, final String version)
     {
         super("maven", name);
         this.group = group;
+        this.classifier = classifier;
         this.version = version;
 
         final StringBuilder fullNameBuilder = new StringBuilder();
@@ -23,6 +24,10 @@ public final class MavenPluginType
         fullNameBuilder.append(name);
         fullNameBuilder.append(":");
         fullNameBuilder.append(version);
+        if (classifier != null) {
+            fullNameBuilder.append(":");
+            fullNameBuilder.append(classifier);
+        }
         this.fullName = fullNameBuilder.toString();
 
         final HashMap<String, String> fullMapMutable = new HashMap<String, String>();
@@ -33,12 +38,13 @@ public final class MavenPluginType
         this.fullMap = Collections.unmodifiableMap(fullMapMutable);
     }
 
-    public static MavenPluginType create(final String name, final String group, final String version)
+    public static MavenPluginType create(
+            final String name, final String group, final String classifier, final String version)
     {
-        if (name == null) {
-            throw new NullPointerException("\"name\" must not present.");
+        if (name == null || group == null || version == null) {
+            throw new NullPointerException("\"name\", \"group\" and \"version\" must be present.");
         }
-        return new MavenPluginType(name, group, version);
+        return new MavenPluginType(name, group, classifier, version);
     }
 
     @JsonValue
@@ -53,6 +59,12 @@ public final class MavenPluginType
         return this.group;
     }
 
+    @JsonProperty("classifier")
+    public final String getClassifier()
+    {
+        return this.classifier;
+    }
+
     @JsonProperty("version")
     public final String getVersion()
     {
@@ -67,7 +79,7 @@ public final class MavenPluginType
     @Override
     public final int hashCode()
     {
-        return Objects.hash(getSourceType(), getName(), this.group, this.version);
+        return Objects.hash(getSourceType(), getName(), this.group, this.classifier, this.version);
     }
 
     @Override
@@ -80,6 +92,8 @@ public final class MavenPluginType
         return (this.getSourceType().equals(other.getSourceType()) &&
                 this.getName().equals(other.getName()) &&
                 this.getGroup().equals(other.getGroup()) &&
+                ((this.getClassifier() == null && other.getClassifier() == null) ||
+                 (this.getClassifier() != null && this.getClassifier().equals(other.getClassifier()))) &&
                 this.getVersion().equals(other.getVersion()));
     }
 
@@ -91,6 +105,7 @@ public final class MavenPluginType
     }
 
     private final String group;
+    private final String classifier;  // |classifier| can be null.
     private final String version;
     private final String fullName;
     private final Map<String, String> fullMap;

data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoader.java

@@ -42,6 +42,15 @@ public class PluginClassLoader
                 }));
     }
 
+    /**
+     * Adds the specified path to the list of URLs (for {@code URLClassLoader}) to search for classes and resources.
+     *
+     * It internally calls {@code URLClassLoader#addURL}.
+     *
+     * Some plugins (embulk-input-jdbc, for example) are calling this method to load external JAR files.
+     *
+     * @see https://github.com/embulk/embulk-input-jdbc/blob/ebfff0b249d507fc730c87e08b56e6aa492060ca/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java#L586-L595
+     */
     public void addPath(Path path)
     {
         try {
@@ -56,17 +65,45 @@ public class PluginClassLoader
         super.addURL(url);
     }
 
+    /**
+     * Loads the class with the specified binary name prioritized by the "parent-first" condition.
+     *
+     * It copy-cats {@code ClassLoader#loadClass} while the "parent-first" priorities are considered.
+     *
+     * If the specified class is "parent-first", it behaves the same as {@code ClassLoader#loadClass} ordered as below.
+     *
+     * <ol>
+     *
+     * <li><p>Invoke the {@code #findLoadedClass} method to check if the class has already been loaded.</p></li>
+     *
+     * <li><p>Invoke the parent's {@code #loadClass} method.
+     *
+     * <li><p>Invoke the {@code #findClass} method of this class loader to find the class.</p></li>
+     *
+     * </ol>
+     *
+     * If the specified class is "NOT parent-first", the 2nd and 3rd actions are swapped.
+     *
+     * @see https://docs.oracle.com/javase/7/docs/api/java/lang/ClassLoader.html#loadClass(java.lang.String,%20boolean)
+     * @see http://hg.openjdk.java.net/jdk7u/jdk7u/jdk/file/jdk7u141-b02/src/share/classes/java/lang/ClassLoader.java
+     */
     @Override
     protected Class<?> loadClass(String name, boolean resolve)
             throws ClassNotFoundException
     {
         synchronized (getClassLoadingLock(name)) {
-            Class<?> loadedClass = findLoadedClass(name);
+            // If the class has already been loaded by this {@code ClassLoader} or the parent's {@code ClassLoader},
+            // find the loaded class and return it.
+            final Class<?> loadedClass = findLoadedClass(name);
+
             if (loadedClass != null) {
                 return resolveClass(loadedClass, resolve);
             }
 
-            boolean parentFirst = isParentFirstPackage(name);
+            final boolean parentFirst = isParentFirstPackage(name);
+
+            // If the class is "not parent-first" (not to be loaded by the parent at first),
+            // try {@code #findClass} of the child's ({@code PluginClassLoader}'s).
             if (!parentFirst) {
                 try {
                     return resolveClass(findClass(name), resolve);
@@ -74,11 +111,14 @@ public class PluginClassLoader
                 }
             }
 
+            // If the class is "parent-first" (to be loaded by the parent at first), try this part at first.
+            // If the class is "not parent-first" (not to be loaded by the parent at first), the above part runs first.
             try {
                 return resolveClass(getParent().loadClass(name), resolve);
             } catch (ClassNotFoundException ignored) {
             }
 
+            // If the class is "parent-first" (to be loaded by the parent at first), this part runs after the above.
             if (parentFirst) {
                 return resolveClass(findClass(name), resolve);
             }

data/embulk-core/src/main/java/org/embulk/plugin/PluginSourceNotMatchException.java

@@ -5,6 +5,7 @@ public class PluginSourceNotMatchException
 {
     public PluginSourceNotMatchException()
     {
+        super();
     }
 
     public PluginSourceNotMatchException(String message)
@@ -16,4 +17,9 @@ public class PluginSourceNotMatchException
     {
         super(cause);
     }
+
+    public PluginSourceNotMatchException(String message, Throwable cause)
+    {
+        super(message, cause);
+    }
 }

data/embulk-core/src/main/java/org/embulk/plugin/PluginType.java

@@ -79,8 +79,9 @@ public abstract class PluginType
             {
                 final String name = stringMap.get("name");
                 final String group = stringMap.get("group");
+                final String classifier = stringMap.get("classifier");
                 final String version = stringMap.get("version");
-                return MavenPluginType.create(name, group, version);
+                return MavenPluginType.create(name, group, classifier, version);
             }
         default:
             throw new IllegalArgumentException("\"source\" must be one of: [\"default\", \"maven\"]");

data/embulk-core/src/main/java/org/embulk/plugin/jar/InvalidJarPluginException.java

@@ -0,0 +1,14 @@
+package org.embulk.plugin.jar;
+
+public class InvalidJarPluginException
+        extends Exception {
+    public InvalidJarPluginException(final String message)
+    {
+        super(message);
+    }
+
+    public InvalidJarPluginException(final String message, final Throwable cause)
+    {
+        super(message, cause);
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/jar/JarPluginLoader.java

@@ -0,0 +1,205 @@
+package org.embulk.plugin.jar;
+
+import com.google.common.collect.ImmutableList;
+import java.io.IOError;
+import java.io.IOException;
+import java.net.JarURLConnection;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.jar.Attributes;
+import java.util.jar.Manifest;
+import org.embulk.plugin.PluginClassLoader;
+import org.embulk.plugin.PluginClassLoaderFactory;
+
+/**
+ * JarPluginLoader loads a JAR-based Embulk plugin class.
+ *
+ * It implements {@code AutoCloseable} so that it can load the contents lazily in the future.
+ * It is recommended to try-with-resources to use this class.
+ */
+public class JarPluginLoader
+        implements AutoCloseable
+{
+    public static final String MANIFEST_PLUGIN_MAIN_CLASS = "Embulk-Plugin-Main-Class";
+    public static final String MANIFEST_PLUGIN_SPI_VERSION = "Embulk-Plugin-Spi-Version";
+
+    private JarPluginLoader(final Manifest pluginManifest,
+                            final Attributes pluginManifestAttributes,
+                            final Class pluginMainClass)
+    {
+        this.pluginManifest = pluginManifest;
+        this.pluginManifestAttributes = pluginManifestAttributes;
+        this.pluginMainClass = pluginMainClass;
+    }
+
+    public static JarPluginLoader load(final Path jarPath, final PluginClassLoaderFactory classLoaderFactory)
+            throws InvalidJarPluginException
+    {
+        final JarURLConnection connection = openJarURLConnection(jarPath);
+        final Manifest manifest = loadJarPluginManifest(connection, jarPath);
+        final Attributes manifestAttributes = manifest.getMainAttributes();
+        final int spiVersion = getPluginSpiVersionFromManifest(manifestAttributes);
+
+        if (spiVersion == 0) {
+            final String mainClassName = getPluginMainClassNameFromManifest(manifestAttributes);
+            final Class mainClass = loadJarPluginMainClass(jarPath, mainClassName, classLoaderFactory);
+            return new JarPluginLoader(manifest, manifestAttributes, mainClass);
+        }
+
+        throw new InvalidJarPluginException("Unknown SPI version of JAR plugin: " + spiVersion);
+    }
+
+    public Class getPluginMainClass()
+            throws InvalidJarPluginException
+    {
+        return this.pluginMainClass;
+    }
+
+    public int getPluginSpiVersion()
+            throws InvalidJarPluginException
+    {
+        return getPluginSpiVersionFromManifest(this.pluginManifestAttributes);
+    }
+
+    @Override
+    public void close()
+            throws InvalidJarPluginException
+    {
+    }
+
+    /**
+     * Opens JarURLConnection for the given JAR file path.
+     *
+     * Note that JarURLConnection does not have {@code disconnect} nor {@code close}.
+     */
+    private static JarURLConnection openJarURLConnection(final Path jarPath)
+            throws InvalidJarPluginException
+    {
+        // jar:file:/...
+        final URL jarUrl;
+        try {
+            jarUrl = new URL("jar:" + jarPath.toUri().toURL().toString() + "!/");
+        }
+        catch (MalformedURLException ex) {
+            throw new InvalidJarPluginException("JAR plugin path specified is invalid: " + jarPath.toString(), ex);
+        }
+
+        try {
+            return (JarURLConnection) jarUrl.openConnection();
+        }
+        catch (IOException ex) {
+            throw new InvalidJarPluginException("JAR plugin specified is formatted wrongly: " + jarPath.toString(), ex);
+        }
+    }
+
+    private static Manifest loadJarPluginManifest(final JarURLConnection connection, final Path jarPath)
+            throws InvalidJarPluginException
+    {
+        try {
+            return connection.getManifest();
+        }
+        catch (IOException ex) {
+            throw new InvalidJarPluginException("Manifest in JAR plugin specified is invalid: " + jarPath.toString(),
+                                                ex);
+        }
+    }
+
+    private static Class loadJarPluginMainClass(final Path jarPath,
+                                                final String pluginMainClassName,
+                                                final PluginClassLoaderFactory pluginClassLoaderFactory)
+            throws InvalidJarPluginException
+    {
+        final URI fileUriJar;
+        try {
+            fileUriJar = jarPath.toUri();
+        }
+        catch (IOError ex) {
+            throw new InvalidJarPluginException("[FATAL] JAR plugin path specified is invalid: " + jarPath.toString(),
+                                                ex);
+        }
+        catch (SecurityException ex) {
+            throw new InvalidJarPluginException("Security manager prohibits getting the working directory: " +
+                                                jarPath.toString() +
+                                                ". Specifying an absolute path for JAR plugin may solve this.",
+                                                ex);
+        }
+
+        // file:/...
+        final URL fileUrlJar;
+        try {
+            fileUrlJar = fileUriJar.toURL();
+        }
+        catch (IllegalArgumentException ex) {
+            throw new InvalidJarPluginException("[FATAL/INTERNAL] JAR plugin path as URI is not absolute.", ex);
+        }
+        catch (MalformedURLException ex) {
+            throw new InvalidJarPluginException("JAR plugin path specified is invalid: " + jarPath.toString(), ex);
+        }
+
+
+        final PluginClassLoader pluginClassLoader =
+            pluginClassLoaderFactory.create(ImmutableList.of(fileUrlJar), JarPluginLoader.class.getClassLoader());
+
+        final Class pluginMainClass;
+        try {
+            pluginMainClass = pluginClassLoader.loadClass(pluginMainClassName);
+        }
+        catch (ClassNotFoundException ex) {
+            throw new InvalidJarPluginException("Class " + pluginMainClassName + " not found in " + jarPath.toString(),
+                                                ex);
+        }
+
+        return pluginMainClass;
+    }
+
+    private static int getPluginSpiVersionFromManifest(final Attributes manifestAttributes)
+            throws InvalidJarPluginException
+    {
+        final String spiVersionString = getAttributeFromManifest(manifestAttributes, MANIFEST_PLUGIN_SPI_VERSION);
+
+        if (spiVersionString == null) {
+            throw new InvalidJarPluginException("SPI version of JAR plugin is not specified.");
+        }
+
+        try {
+            return Integer.parseInt(spiVersionString);
+        }
+        catch (NumberFormatException ex) {
+            throw new InvalidJarPluginException("SPI version of JAR plugin is not an integer: \"" +
+                                                spiVersionString +
+                                                "\"", ex);
+        }
+    }
+
+    private static String getPluginMainClassNameFromManifest(final Attributes manifestAttributes)
+            throws InvalidJarPluginException
+    {
+        final String pluginMainClassName = getAttributeFromManifest(manifestAttributes, MANIFEST_PLUGIN_MAIN_CLASS);
+
+        if (pluginMainClassName == null) {
+            throw new InvalidJarPluginException("Main class name of JAR plugin is not specified.");
+        }
+
+        return pluginMainClassName;
+    }
+
+    private static String getAttributeFromManifest(final Attributes manifestAttributes, final String attributeName)
+            throws InvalidJarPluginException
+    {
+        try {
+            return manifestAttributes.getValue(attributeName);
+        }
+        catch (IllegalArgumentException ex) {
+            throw new InvalidJarPluginException("[FATAL/INTERNAL] " +
+                                                attributeName +
+                                                " is considered invalid as a manifest attribute.",
+                                                ex);
+        }
+    }
+
+    private final Manifest pluginManifest;
+    private final Attributes pluginManifestAttributes;
+    private final Class pluginMainClass;
+}