embulk 0.8.35-java → 0.8.36-java

data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderFactory.java

@@ -5,5 +5,12 @@ import java.net.URL;
 
 public interface PluginClassLoaderFactory
 {
-    PluginClassLoader create(Collection<URL> urls, ClassLoader parentClassLoader);
+    PluginClassLoader create(Collection<URL> flatJarUrls, ClassLoader parentClassLoader);
+    PluginClassLoader createForNestedJar(
+        final ClassLoader parentClassLoader,
+        final URL oneNestedJarUrl);
+    PluginClassLoader createForNestedJar(
+        final ClassLoader parentClassLoader,
+        final URL oneNestedJarUrl,
+        final Collection<String> embeddedJarPathsInNestedJar);
 }

data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderModule.java

@@ -61,10 +61,41 @@ public class PluginClassLoaderModule
 
         private class Factory implements PluginClassLoaderFactory
         {
+            @Override
             public PluginClassLoader create(Collection<URL> urls, ClassLoader parentClassLoader)
             {
-                return new PluginClassLoader(urls, parentClassLoader,
-                        parentFirstPackages, parentFirstResources);
+                return PluginClassLoader.createForFlatJars(
+                    parentClassLoader,
+                    urls,
+                    parentFirstPackages,
+                    parentFirstResources);
+            }
+
+            @Override
+            public PluginClassLoader createForNestedJar(
+                    final ClassLoader parentClassLoader,
+                    final URL oneNestedJarUrl)
+            {
+                return PluginClassLoader.createForNestedJar(
+                    parentClassLoader,
+                    oneNestedJarUrl,
+                    null,
+                    parentFirstPackages,
+                    parentFirstResources);
+            }
+
+            @Override
+            public PluginClassLoader createForNestedJar(
+                    final ClassLoader parentClassLoader,
+                    final URL oneNestedJarUrl,
+                    final Collection<String> embeddedJarPathsInNestedJar)
+            {
+                return PluginClassLoader.createForNestedJar(
+                    parentClassLoader,
+                    oneNestedJarUrl,
+                    embeddedJarPathsInNestedJar,
+                    parentFirstPackages,
+                    parentFirstResources);
             }
         }
     }

data/embulk-core/src/main/java/org/embulk/plugin/jar/JarPluginLoader.java

@@ -1,6 +1,5 @@
 package org.embulk.plugin.jar;
 
-import com.google.common.collect.ImmutableList;
 import java.io.IOError;
 import java.io.IOException;
 import java.net.JarURLConnection;
@@ -8,6 +7,9 @@ import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import java.util.jar.Attributes;
 import java.util.jar.Manifest;
 import org.embulk.plugin.PluginClassLoader;
@@ -24,6 +26,7 @@ public class JarPluginLoader
 {
     public static final String MANIFEST_PLUGIN_MAIN_CLASS = "Embulk-Plugin-Main-Class";
     public static final String MANIFEST_PLUGIN_SPI_VERSION = "Embulk-Plugin-Spi-Version";
+    public static final String MANIFEST_PLUGIN_CLASS_PATH = "Embulk-Plugin-Class-Path";
 
     private JarPluginLoader(final Manifest pluginManifest,
                             final Attributes pluginManifestAttributes,
@@ -44,7 +47,8 @@ public class JarPluginLoader
 
         if (spiVersion == 0) {
             final String mainClassName = getPluginMainClassNameFromManifest(manifestAttributes);
-            final Class mainClass = loadJarPluginMainClass(jarPath, mainClassName, classLoaderFactory);
+            final List<String> pluginClassPath = getPluginClassPathFromManifest(manifestAttributes);
+            final Class mainClass = loadJarPluginMainClass(jarPath, mainClassName, pluginClassPath, classLoaderFactory);
             return new JarPluginLoader(manifest, manifestAttributes, mainClass);
         }
 
@@ -108,6 +112,7 @@ public class JarPluginLoader
 
     private static Class loadJarPluginMainClass(final Path jarPath,
                                                 final String pluginMainClassName,
+                                                final List<String> pluginClassPath,
                                                 final PluginClassLoaderFactory pluginClassLoaderFactory)
             throws InvalidJarPluginException
     {
@@ -138,9 +143,15 @@ public class JarPluginLoader
             throw new InvalidJarPluginException("JAR plugin path specified is invalid: " + jarPath.toString(), ex);
         }
 
-
-        final PluginClassLoader pluginClassLoader =
-            pluginClassLoaderFactory.create(ImmutableList.of(fileUrlJar), JarPluginLoader.class.getClassLoader());
+        final PluginClassLoader pluginClassLoader;
+        if (pluginClassPath.isEmpty()) {
+            pluginClassLoader = pluginClassLoaderFactory.createForNestedJar(
+                JarPluginLoader.class.getClassLoader(), fileUrlJar);
+        }
+        else {
+            pluginClassLoader = pluginClassLoaderFactory.createForNestedJar(
+                JarPluginLoader.class.getClassLoader(), fileUrlJar, pluginClassPath);
+        }
 
         final Class pluginMainClass;
         try {
@@ -185,6 +196,22 @@ public class JarPluginLoader
         return pluginMainClassName;
     }
 
+    private static List<String> getPluginClassPathFromManifest(final Attributes manifestAttributes)
+            throws InvalidJarPluginException
+    {
+        final String pluginClassPathJoined = getAttributeFromManifest(manifestAttributes, MANIFEST_PLUGIN_CLASS_PATH);
+
+        if (pluginClassPathJoined == null) {
+            return Collections.<String>emptyList();
+        }
+
+        final List<String> pluginClassPath = new ArrayList<String>();
+        for (final String splitPluginClassPath : pluginClassPathJoined.split(" +", 0)) {
+            pluginClassPath.add(splitPluginClassPath);
+        }
+        return pluginClassPath;
+    }
+
     private static String getAttributeFromManifest(final Attributes manifestAttributes, final String attributeName)
             throws InvalidJarPluginException
     {

data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java

@@ -20,7 +20,6 @@ import org.embulk.plugin.PluginType;
 import org.embulk.plugin.PluginManager;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampFormatter;
-import org.embulk.spi.time.TimestampFormatter.FormatterTask;
 
 public class ExecSession
 {
@@ -201,11 +200,7 @@ public class ExecSession
 
     public TimestampFormatter newTimestampFormatter(String format, DateTimeZone timezone)
     {
-        ConfigSource config = Exec.newConfigSource();
-        config.set("timezone", timezone.getID());
-        // TODO: Stop creating the |FormatterTask| as |TimestampFormater.FormatterTask| is deprecated since v0.6.14.
-        FormatterTask formatterTask = config.loadConfig(FormatterTask.class);
-        return new TimestampFormatter(format, formatterTask.getTimeZone());
+        return new TimestampFormatter(format, timezone);
     }
 
     public TempFileSpace getTempFileSpace()

data/embulk-core/src/main/java/org/embulk/spi/json/RubyValueApi.java

@@ -13,9 +13,24 @@ import org.jruby.runtime.builtin.IRubyObject;
 import org.jruby.util.ByteList;
 import org.jcodings.specific.ASCIIEncoding;
 
+@Deprecated
 public class RubyValueApi
 {
+    @Deprecated
     public static Value fromMessagePack(RubyString content)
+    {
+        if (!fromMessagePackDeprecationWarned) {
+            System.err.println("[WARN] Plugin uses deprecated org.embulk.spi.json.RubyValueApi.fromMessagePack");
+            System.err.println("[WARN] Report plugins in your config at: https://github.com/embulk/embulk/issues/802");
+            // The |fromMessagePackDeprecationWarned| flag is used only for warning messages.
+            // Even in case of race conditions, messages are just duplicated -- should be acceptable.
+            fromMessagePackDeprecationWarned = true;
+        }
+
+        return fromMessagePackInternal(content);
+    }
+
+    private static Value fromMessagePackInternal(RubyString content)
     {
         ByteList list = content.getByteList();
         try {
@@ -40,8 +55,17 @@ public class RubyValueApi
         }
     }
 
+    @Deprecated
     public static RubyString toMessagePack(Ruby runtime, Value value)
     {
+        if (!toMessagePackDeprecationWarned) {
+            System.err.println("[WARN] Plugin uses deprecated org.embulk.spi.json.RubyValueApi.toMessagePack");
+            System.err.println("[WARN] Report plugins in your config at: https://github.com/embulk/embulk/issues/802");
+            // The |toMessagePackDeprecationWarned| flag is used only for warning messages.
+            // Even in case of race conditions, messages are just duplicated -- should be acceptable.
+            toMessagePackDeprecationWarned = true;
+        }
+
         try {
             MessageBufferPacker packer = MessagePack.newDefaultBufferPacker();
             packer.packValue(value);
@@ -54,9 +78,23 @@ public class RubyValueApi
         }
     }
 
+    @Deprecated
     public static Value toValue(Ruby runtime, IRubyObject object)
     {
+        if (!toValueDeprecationWarned) {
+            System.err.println("[WARN] Plugin uses deprecated org.embulk.spi.json.RubyValueApi.toValue");
+            System.err.println("[WARN] Report plugins in your config at: https://github.com/embulk/embulk/issues/802");
+            // The |toValueDeprecationWarned| flag is used only for warning messages.
+            // Even in case of race conditions, messages are just duplicated -- should be acceptable.
+            toValueDeprecationWarned = true;
+        }
+
         RubyString string = (RubyString) object.callMethod(runtime.getCurrentContext(), "to_msgpack");
-        return fromMessagePack(string);
+        return fromMessagePackInternal(string);
     }
+
+    private static boolean fromMessagePackDeprecationWarned = false;
+    private static boolean toMessagePackDeprecationWarned = false;
+    private static boolean toValueDeprecationWarned = false;
+
 }

data/embulk-core/src/main/java/org/embulk/spi/time/Timestamp.java

@@ -88,15 +88,33 @@ public class Timestamp
         }
     }
 
+    @Deprecated
     public RubyTime getRubyTime(Ruby runtime)
     {
+        if (!getRubyTimeDeprecationWarned) {
+            System.err.println("[WARN] Plugin uses deprecated org.embulk.spi.time.Timestamp.getRubyTime");
+            System.err.println("[WARN] Report plugins in your config at: https://github.com/embulk/embulk/issues/820");
+            // The |getRubyTimeDeprecationWarned| flag is used only for warning messages.
+            // Even in case of race conditions, messages are just duplicated -- should be acceptable.
+            getRubyTimeDeprecationWarned = true;
+        }
+
         RubyTime time = new RubyTime(runtime, runtime.getClass("Time"), new DateTime(toEpochMilli())).gmtime();
         time.setNSec(nano % 1000000);
         return time;
     }
 
+    @Deprecated
     public static Timestamp fromRubyTime(RubyTime time)
     {
+        if (!fromRubyTimeDeprecationWarned) {
+            System.err.println("[WARN] Plugin uses deprecated org.embulk.spi.time.Timestamp.fromRubyTime");
+            System.err.println("[WARN] Report plugins in your config at: https://github.com/embulk/embulk/issues/820");
+            // The |fromRubyTimeDeprecationWarned| flag is used only for warning messages.
+            // Even in case of race conditions, messages are just duplicated -- should be acceptable.
+            fromRubyTimeDeprecationWarned = true;
+        }
+
         long msec = time.getDateTime().getMillis();
         long sec = msec / 1000;
         long nsec = time.getNSec() + (msec % 1000) * 1000000;
@@ -156,4 +174,7 @@ public class Timestamp
 
         return new Timestamp(seconds, nano);
     }
+
+    private static boolean getRubyTimeDeprecationWarned = false;
+    private static boolean fromRubyTimeDeprecationWarned = false;
 }

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormat.java

@@ -26,12 +26,30 @@ public class TimestampFormat
     @Deprecated
     public TimestampFormatter newFormatter(TimestampFormatter.FormatterTask task)
     {
+        // NOTE: Its deprecation is not actually from ScriptingContainer, though.
+        // TODO: Notify users about deprecated calls through the notification reporter.
+        if (!newFormatterDeprecationWarned) {
+            System.err.println("[WARN] Plugin uses deprecated method org.embulk.spi.time.TimestampFormat.newFormatter");
+            System.err.println("[WARN] Report plugins in your config at: https://github.com/embulk/embulk/issues/828");
+            // The |newFormatterDeprecationWarned| flag is used only for warning messages.
+            // Even in case of race conditions, messages are just duplicated -- should be acceptable.
+            newFormatterDeprecationWarned = true;
+        }
         return new TimestampFormatter(format, task.getTimeZone());
     }
 
     @Deprecated
     public TimestampParser newParser(TimestampParser.ParserTask task)
     {
+        // NOTE: Its deprecation is not actually from ScriptingContainer, though.
+        // TODO: Notify users about deprecated calls through the notification reporter.
+        if (!newParserDeprecationWarned) {
+            System.err.println("[WARN] Plugin uses deprecated method org.embulk.spi.time.TimestampFormat.newParser");
+            System.err.println("[WARN] Report plugins in your config at: https://github.com/embulk/embulk/issues/828");
+            // The |newParserDeprecationWarned| flag is used only for warning messages.
+            // Even in case of race conditions, messages are just duplicated -- should be acceptable.
+            newParserDeprecationWarned = true;
+        }
         return new TimestampParser(format, task.getDefaultTimeZone());
     }
 
@@ -69,6 +87,9 @@ public class TimestampFormat
         }
     }
 
+    private static boolean newFormatterDeprecationWarned = false;
+    private static boolean newParserDeprecationWarned = false;
+
     //// Java standard TimeZone
     //static TimeZone parseDateTimeZone(String s)
     //{

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetterFactory.java

@@ -24,17 +24,34 @@ import org.embulk.spi.Column;
 import org.embulk.spi.PageBuilder;
 import org.embulk.config.ConfigException;
 
-public class DynamicColumnSetterFactory
+class DynamicColumnSetterFactory
 {
     private final DefaultValueSetter defaultValue;
     private final DynamicPageBuilder.BuilderTask task;
+    private final boolean useColumnForTimestampMetadata;
 
-    DynamicColumnSetterFactory(
-            DynamicPageBuilder.BuilderTask task,
-            DefaultValueSetter defaultValue)
+    private DynamicColumnSetterFactory(
+            final DynamicPageBuilder.BuilderTask task,
+            final DefaultValueSetter defaultValue,
+            final boolean useColumnForTimestampMetadata)
     {
         this.defaultValue = defaultValue;
         this.task = task;
+        this.useColumnForTimestampMetadata = useColumnForTimestampMetadata;
+    }
+
+    static DynamicColumnSetterFactory createWithTimestampMetadataFromBuilderTask(
+            final DynamicPageBuilder.BuilderTask task,
+            final DefaultValueSetter defaultValue)
+    {
+        return new DynamicColumnSetterFactory(task, defaultValue, false);
+    }
+
+    static DynamicColumnSetterFactory createWithTimestampMetadataFromColumn(
+            final DynamicPageBuilder.BuilderTask task,
+            final DefaultValueSetter defaultValue)
+    {
+        return new DynamicColumnSetterFactory(task, defaultValue, true);
     }
 
     public static DefaultValueSetter nullDefaultValue()
@@ -53,22 +70,29 @@ public class DynamicColumnSetterFactory
             return new DoubleColumnSetter(pageBuilder, column, defaultValue);
         } else if (type instanceof StringType) {
             TimestampFormatter formatter = new TimestampFormatter(
-                    getTimestampFormat(column).getFormat(), getTimeZone(column));
+                    getTimestampFormatForFormatter(column).getFormat(), getTimeZone(column));
             return new StringColumnSetter(pageBuilder, column, defaultValue, formatter);
         } else if (type instanceof TimestampType) {
             // TODO use flexible time format like Ruby's Time.parse
-            TimestampParser parser = new TimestampParser(
-                    getTimestampFormat(column).getFormat(), getTimeZone(column));
+            final TimestampParser parser;
+            if (this.useColumnForTimestampMetadata) {
+                final TimestampType timestampType = (TimestampType) type;
+                parser = new TimestampParser(timestampType.getFormat(), getTimeZone(column));
+            }
+            else {
+                parser = new TimestampParser(
+                        getTimestampFormatForParser(column).getFormat(), getTimeZone(column));
+            }
             return new TimestampColumnSetter(pageBuilder, column, defaultValue, parser);
         } else if (type instanceof JsonType) {
             TimestampFormatter formatter = new TimestampFormatter(
-                    getTimestampFormat(column).getFormat(), getTimeZone(column));
+                    getTimestampFormatForFormatter(column).getFormat(), getTimeZone(column));
             return new JsonColumnSetter(pageBuilder, column, defaultValue, formatter);
         }
         throw new ConfigException("Unknown column type: "+type);
     }
 
-    private TimestampFormat getTimestampFormat(Column column)
+    private TimestampFormat getTimestampFormatForFormatter(Column column)
     {
         DynamicPageBuilder.ColumnOption option = getColumnOption(column);
         if (option != null) {
@@ -78,6 +102,16 @@ public class DynamicColumnSetterFactory
         }
     }
 
+    private TimestampFormat getTimestampFormatForParser(Column column)
+    {
+        DynamicPageBuilder.ColumnOption option = getColumnOption(column);
+        if (option != null) {
+            return option.getTimestampFormat();
+        } else {
+            return new TimestampFormat("%Y-%m-%d %H:%M:%S.%N");
+        }
+    }
+
     private DateTimeZone getTimeZone(Column column)
     {
         DynamicPageBuilder.ColumnOption option = getColumnOption(column);

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicPageBuilder.java

@@ -50,8 +50,10 @@ public class DynamicPageBuilder
     public static interface ColumnOption
             extends Task
     {
+        // DynamicPageBuilder is used for inputs, then datetime parsing.
+        // Ruby's strptime does not accept numeric prefixes in specifiers such as "%6N".
         @Config("timestamp_format")
-        @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%6N\"")
+        @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%N\"")
         public TimestampFormat getTimestampFormat();
 
         @Config("timezone")
@@ -59,16 +61,14 @@ public class DynamicPageBuilder
         public Optional<DateTimeZone> getTimeZone();
     }
 
-    public DynamicPageBuilder(BuilderTask task,
-            BufferAllocator allocator, Schema schema, PageOutput output)
+    private DynamicPageBuilder(
+            final DynamicColumnSetterFactory factory,
+            final BufferAllocator allocator,
+            final Schema schema,
+            final PageOutput output)
     {
         this.pageBuilder = new PageBuilder(allocator, schema, output);
         this.schema = schema;
-
-        // TODO configurable default value
-        DynamicColumnSetterFactory factory = new DynamicColumnSetterFactory(task,
-                DynamicColumnSetterFactory.nullDefaultValue());
-
         ImmutableList.Builder<DynamicColumnSetter> setters = ImmutableList.builder();
         ImmutableMap.Builder<String, DynamicColumnSetter> lookup = ImmutableMap.builder();
         for (Column c : schema.getColumns()) {
@@ -80,6 +80,44 @@ public class DynamicPageBuilder
         this.columnLookup = lookup.build();
     }
 
+    @Deprecated  // The static creator method "createWithTimestampMetadataFromBuilderTask" is preferred.
+    public DynamicPageBuilder(
+            BuilderTask task,
+            BufferAllocator allocator,
+            Schema schema,
+            PageOutput output)
+    {
+        this(DynamicColumnSetterFactory.createWithTimestampMetadataFromBuilderTask(
+                 task, DynamicColumnSetterFactory.nullDefaultValue()),  // TODO configurable default value
+             allocator,
+             schema,
+             output);
+    }
+
+    public static DynamicPageBuilder createWithTimestampMetadataFromBuilderTask(
+            BuilderTask task,
+            BufferAllocator allocator,
+            Schema schema,
+            PageOutput output)
+    {
+        // TODO configurable default value
+        DynamicColumnSetterFactory factory = DynamicColumnSetterFactory.createWithTimestampMetadataFromBuilderTask(
+                task, DynamicColumnSetterFactory.nullDefaultValue());
+        return new DynamicPageBuilder(factory, allocator, schema, output);
+    }
+
+    public static DynamicPageBuilder createWithTimestampMetadataFromColumn(
+            BuilderTask task,
+            BufferAllocator allocator,
+            Schema schema,
+            PageOutput output)
+    {
+        // TODO configurable default value
+        DynamicColumnSetterFactory factory = DynamicColumnSetterFactory.createWithTimestampMetadataFromColumn(
+                task, DynamicColumnSetterFactory.nullDefaultValue());
+        return new DynamicPageBuilder(factory, allocator, schema, output);
+    }
+
     public List<Column> getColumns()
     {
         return schema.getColumns();