embulk 0.6.15 → 0.6.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 48e00af06c823ff60ec39a9d020896c6b16442a5
-  data.tar.gz: 711922cebca874837dce9d100cbee5d52e5c833f
+  metadata.gz: 0923308ef177929c5e59513c16d318379c5d4993
+  data.tar.gz: ff6459b193b9635b5f666ffc3f4c878577c06e64
 SHA512:
-  metadata.gz: 714466b1fbd8794ace91fcc3b074dac84b2c72d5504e9751da9f04045f86861c653b0cd9088e443b3b53fda56b400ba7bb8e1d1a75ca8e0c2e2ba2dcbd66e745
-  data.tar.gz: 7ffa66ba254cd40e2cb4427d356648eec8e61e194c00d94b2f93817f22bb70d182154fe88ce0d38c7088507b8a4c0c445e45f6ab8da37627a8158113824921a5
+  metadata.gz: 7a5ee5b44def0c21d59f9a4ea27ba643c086626d681ad342630cd899005c05e1e1ad6fea55eb3aefd21e817cfa371156481522e59f6d5af560bfd16cab0e6f6a
+  data.tar.gz: cf930364ad7f307871700e9637490d3a445b1a0e78e479727828924e9e947301041c1c87834bbcb3d763e81f39afb98204c8da2fc5a2a4f7cd880b53b25c84a2

data/build.gradle

@@ -11,7 +11,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.6.15'
+    version = '0.6.16'
 
     ext {
         jrubyVersion = '1.7.19'
@@ -149,6 +149,40 @@ dependencies {
     jrubyExec 'rubygems:test-unit:3.0.+'
 }
 
+import java.util.zip.ZipFile
+import java.util.zip.ZipEntry
+task updateResources(dependsOn: 'classpath') {
+    doFirst {
+        List<String> packages = []
+        List<String> dirs = []
+        file("classpath").listFiles().each { jarFile ->
+            List<String> names = new ZipFile(jarFile.toString()).entries().iterator().collect { it.getName() }
+            packages.addAll(names.findAll { it.endsWith(".class") }.collect { it.replaceFirst(/(?:^|\/)[^\/]*\.class$/, '').replaceAll(/\//, '.') }.unique())
+            dirs.addAll(names.findAll { !it.endsWith("/") && !it.endsWith(".class") }.collect { it.replaceFirst(/(?:^|\/)[^\/]*$/, '') }.unique())
+        }
+        packages = packages.unique()
+        dirs = dirs.unique()
+        List<String> uniquePackages = packages.clone()
+        List<String> uniqueDirs = dirs.clone()
+        packages.each { pk -> uniquePackages.removeAll { it.startsWith(pk + '.') } }
+        dirs.each { dir -> uniqueDirs.removeAll { it.startsWith(dir + '/') } }
+        uniquePackages.removeAll { it.isEmpty() }
+        uniqueDirs.removeAll { it.isEmpty() || it == "META-INF" }
+        String date = new Date().format('yyyy-MM-dd HH:mm:ss z', TimeZone.getTimeZone("UTC"))
+        file("embulk-core/src/main/resources/embulk/parent_first_packages.properties").withWriter { out ->
+            out.println("# generated by './gradlew updateResources' at $date")
+            uniquePackages.sort().each { out.println it }
+        }
+        file("embulk-core/src/main/resources/embulk/parent_first_resources.properties").withWriter { out ->
+            out.println("# generated by './gradlew updateResources' at $date")
+            uniqueDirs.sort().each { out.println it }
+        }
+    }
+    doLast {
+        classpath.execute()
+    }
+}
+
 //
 // classpath task
 //

data/embulk-core/src/main/java/org/embulk/EmbulkService.java

@@ -8,6 +8,7 @@ import org.embulk.config.ConfigSource;
 import org.embulk.exec.SystemConfigModule;
 import org.embulk.exec.ExecModule;
 import org.embulk.exec.ExtensionServiceLoaderModule;
+import org.embulk.plugin.PluginClassLoaderModule;
 import org.embulk.plugin.BuiltinPluginSourceModule;
 import org.embulk.jruby.JRubyScriptingModule;
 
@@ -21,6 +22,7 @@ public class EmbulkService
         modules.add(new SystemConfigModule(systemConfig));
         modules.add(new ExecModule());
         modules.add(new ExtensionServiceLoaderModule(systemConfig));
+        modules.add(new PluginClassLoaderModule(systemConfig));
         modules.add(new BuiltinPluginSourceModule());
         modules.add(new JRubyScriptingModule(systemConfig));
         modules.addAll(getAdditionalModules(systemConfig));

data/embulk-core/src/main/java/org/embulk/command/Runner.java

@@ -143,6 +143,7 @@ public class Runner
         ExecutionResult result;
         try {
             if (resume != null) {
+                // exec is not used here
                 result = loader.resume(config, resume);
             } else {
                 result = loader.run(exec, config);
@@ -156,6 +157,11 @@ public class Runner
                 } catch (Throwable ex) {
                     partial.addSuppressed(ex);
                 }
+                try {
+                    exec.cleanup();
+                } catch (Throwable ex) {
+                    partial.addSuppressed(ex);
+                }
                 throw partial;
             }
             // save the resume state
@@ -169,6 +175,7 @@ public class Runner
         if (options.getResumeStatePath() != null) {
             boolean dontCare = new File(options.getResumeStatePath()).delete();
         }
+        exec.cleanup();
 
         // write next config
         ConfigDiff configDiff = result.getConfigDiff();
@@ -187,7 +194,6 @@ public class Runner
         ConfigSource resumeConfig = loadYamlConfig(resumePath);
         ResumeState resume = resumeConfig.loadConfig(ResumeState.class);
 
-        //ExecSession exec = newExecSession(config);  // not necessary
         BulkLoader loader = injector.getInstance(BulkLoader.class);
         loader.cleanup(config, resume);
 
@@ -200,9 +206,16 @@ public class Runner
         ConfigSource config = loadYamlConfig(partialConfigPath);
         checkFileWritable(options.getNextConfigOutputPath());
 
-        ExecSession exec = newExecSession(config);
-        GuessExecutor guess = injector.getInstance(GuessExecutor.class);
-        ConfigDiff configDiff = guess.guess(exec, config);
+        ConfigDiff configDiff;
+        {
+            ExecSession exec = newExecSession(config);
+            try {
+                GuessExecutor guess = injector.getInstance(GuessExecutor.class);
+                configDiff = guess.guess(exec, config);
+            } finally {
+                exec.cleanup();
+            }
+        }
 
         String yml = writeNextConfig(options.getNextConfigOutputPath(), config, configDiff);
         System.err.println(yml);
@@ -244,10 +257,17 @@ public class Runner
 
     public void preview(String partialConfigPath)
     {
-        ConfigSource config = loadYamlConfig(partialConfigPath);
-        ExecSession exec = newExecSession(config);
-        PreviewExecutor preview = injector.getInstance(PreviewExecutor.class);
-        PreviewResult result = preview.preview(exec, config);
+        PreviewResult result;
+        {
+            ConfigSource config = loadYamlConfig(partialConfigPath);
+            ExecSession exec = newExecSession(config);
+            try {
+                PreviewExecutor preview = injector.getInstance(PreviewExecutor.class);
+                result = preview.preview(exec, config);
+            } finally {
+                exec.cleanup();
+            }
+        }
         ModelManager modelManager = injector.getInstance(ModelManager.class);
 
         PreviewPrinter printer;
@@ -294,7 +314,8 @@ public class Runner
 
     private ExecSession newExecSession(ConfigSource config)
     {
-        return new ExecSession(injector, config.getNestedOrSetEmpty("exec"));
+        ConfigSource execConfig = config.deepCopy().getNestedOrSetEmpty("exec");
+        return ExecSession.builder(injector).fromExecConfig(execConfig).build();
     }
 
     private static class MapType extends HashMap<String, Object> {

data/embulk-core/src/main/java/org/embulk/exec/BulkLoader.java

@@ -320,7 +320,7 @@ public class BulkLoader
         public ResumeState buildResumeState(ExecSession exec)
         {
             return new ResumeState(
-                    exec.getSessionConfigSource(),
+                    exec.getSessionExecConfig(),
                     inputTaskSource, outputTaskSource,
                     first(schemas), executorSchema,
                     getInputCommitReports(), getOutputCommitReports());
@@ -351,8 +351,8 @@ public class BulkLoader
     public ExecutionResult resume(final ConfigSource config, final ResumeState resume)
     {
         try {
-            ExecSession exec = new ExecSession(injector, resume.getExecSessionConfigSource());
-            return Exec.doWith(exec, new ExecAction<ExecutionResult>() {
+            ExecSession exec = ExecSession.builder(injector).fromExecConfig(resume.getExecSessionConfigSource()).build();
+            ExecutionResult result = Exec.doWith(exec, new ExecAction<ExecutionResult>() {
                 public ExecutionResult run()
                 {
                     try (SetCurrentThreadName dontCare = new SetCurrentThreadName("resume")) {
@@ -360,6 +360,8 @@ public class BulkLoader
                     }
                 }
             });
+            exec.cleanup();
+            return result;
         } catch (ExecutionException ex) {
             throw Throwables.propagate(ex.getCause());
         }
@@ -368,7 +370,7 @@ public class BulkLoader
     public void cleanup(final ConfigSource config, final ResumeState resume)
     {
         try {
-            ExecSession exec = new ExecSession(injector, resume.getExecSessionConfigSource());
+            ExecSession exec = ExecSession.builder(injector).fromExecConfig(resume.getExecSessionConfigSource()).build();
             Exec.doWith(exec, new ExecAction<Void>() {
                 public Void run()
                 {
@@ -378,6 +380,7 @@ public class BulkLoader
                     }
                 }
             });
+            exec.cleanup();
         } catch (ExecutionException ex) {
             throw Throwables.propagate(ex.getCause());
         }

data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java

@@ -16,6 +16,7 @@ import org.embulk.spi.ParserPlugin;
 import org.embulk.spi.ExecutorPlugin;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.util.CharsetSerDe;
+import org.embulk.spi.unit.LocalFileSerDe;
 import static org.embulk.plugin.InjectedPluginSource.registerPluginTo;
 
 public class ExecModule
@@ -29,6 +30,7 @@ public class ExecModule
         binder.bind(ILoggerFactory.class).toProvider(LoggerProvider.class);
         binder.bind(ModelManager.class).in(Scopes.SINGLETON);
         binder.bind(BufferAllocator.class).to(PooledBufferAllocator.class).in(Scopes.SINGLETON);
+        binder.bind(TempFileAllocator.class).in(Scopes.SINGLETON);
 
         // GuessExecutor
         registerPluginTo(binder, ParserPlugin.class, "system_guess", GuessExecutor.GuessParserPlugin.class);
@@ -43,6 +45,7 @@ public class ExecModule
         DateTimeZoneSerDe.configure(mapper);
         TimestampSerDe.configure(mapper);
         CharsetSerDe.configure(mapper);
+        LocalFileSerDe.configure(mapper);
         mapper.registerModule(new GuavaModule());  // jackson-datatype-guava
         mapper.registerModule(new JodaModule());  // jackson-datatype-joda
         mapper.configure(binder);

data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java

@@ -60,7 +60,7 @@ public class PreviewExecutor
     public PreviewResult preview(ExecSession exec, final ConfigSource config)
     {
         try {
-            return Exec.doWith(exec.copyForPreview(), new ExecAction<PreviewResult>() {
+            return Exec.doWith(exec.forPreview(), new ExecAction<PreviewResult>() {
                 public PreviewResult run()
                 {
                     try (SetCurrentThreadName dontCare = new SetCurrentThreadName("preview")) {

data/embulk-core/src/main/java/org/embulk/exec/TempFileAllocator.java

@@ -0,0 +1,32 @@
+package org.embulk.exec;
+
+import java.io.File;
+import com.google.inject.Inject;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.TempFileSpace;
+
+// TODO change this class to interface
+// TODO don't use this class directly. Use spi.Exec.getTempFileSpace() instead.
+public class TempFileAllocator
+{
+    private final File[] dirs;
+
+    @Inject
+    public TempFileAllocator(@ForSystemConfig ConfigSource systemConfig)
+    {
+        // TODO get `temp_dirs` from system config
+        String s = System.getProperty("java.io.tmpdir");
+        if (s == null || s.isEmpty()) {
+            s = "/tmp";
+        }
+        this.dirs = new File[] {
+            new File(s, "embulk")
+        };
+    }
+
+    public TempFileSpace newSpace(String subdir)
+    {
+        // TODO support multiple files
+        return new TempFileSpace(new File(dirs[0], subdir));
+    }
+}

data/embulk-core/src/main/java/org/embulk/{spi → plugin}/PluginClassLoader.java

@@ -1,6 +1,7 @@
-package org.embulk.spi;
+package org.embulk.plugin;
 
 import java.util.List;
+import java.util.Collection;
 import java.util.Iterator;
 import java.util.ArrayList;
 import java.util.Enumeration;
@@ -9,33 +10,36 @@ import java.nio.file.Path;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.net.MalformedURLException;
+import com.google.common.base.Function;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Iterators;
-import org.jruby.Ruby;
 
 public class PluginClassLoader
         extends URLClassLoader
 {
-    private static final String[] CHILD_FIRST_PACKAGES = new String[] {
-        "io.netty.",
-        "org.yaml.",
-        "com.ibm.icu.",
-    };
-
-    private static final String[] CHILD_FIRST_PATHS = new String[] {
-        "io/netty/",
-        "org/yaml/",
-        "com/ibm/icu/",
-    };
-
-    public PluginClassLoader(Ruby pluginJRubyRuntime, List<URL> urls)
-    {
-        this(urls, pluginJRubyRuntime.getJRubyClassLoader());
-    }
+    private final List<String> parentFirstPackagePrefixes;
+    private final List<String> parentFirstResourcePrefixes;
 
-    public PluginClassLoader(List<URL> urls, ClassLoader parent)
+    public PluginClassLoader(Collection<URL> urls, ClassLoader parent,
+            Collection<String> parentFirstPackages,
+            Collection<String> parentFirstResources)
     {
         super(urls.toArray(new URL[urls.size()]), parent);
+        this.parentFirstPackagePrefixes = ImmutableList.copyOf(
+                Iterables.transform(parentFirstPackages, new Function<String, String>() {
+                    public String apply(String pkg)
+                    {
+                        return pkg + ".";
+                    }
+                }));
+        this.parentFirstResourcePrefixes = ImmutableList.copyOf(
+                Iterables.transform(parentFirstResources, new Function<String, String>() {
+                    public String apply(String pkg)
+                    {
+                        return pkg + "/";
+                    }
+                }));
     }
 
     public void addPath(Path path)
@@ -62,8 +66,8 @@ public class PluginClassLoader
                 return resolveClass(loadedClass, resolve);
             }
 
-            boolean childFirst = isInChildFirstPackage(name);
-            if (childFirst) {
+            boolean parentFirst = isParentFirstPackage(name);
+            if (!parentFirst) {
                 try {
                     return resolveClass(findClass(name), resolve);
                 } catch (ClassNotFoundException ignored) {
@@ -75,7 +79,7 @@ public class PluginClassLoader
             } catch (ClassNotFoundException ignored) {
             }
 
-            if (!childFirst) {
+            if (parentFirst) {
                 return resolveClass(findClass(name), resolve);
             }
 
@@ -94,7 +98,7 @@ public class PluginClassLoader
     @Override
     public URL getResource(String name)
     {
-        boolean childFirst = isInChildFirstPath(name);
+        boolean childFirst = isParentFirstPath(name);
 
         if (childFirst) {
             URL childUrl = findResource(name);
@@ -124,9 +128,9 @@ public class PluginClassLoader
     {
         List<Iterator<URL>> resources = new ArrayList<>();
 
-        boolean childFirst = isInChildFirstPath(name);
+        boolean parentFirst = isParentFirstPath(name);
 
-        if (childFirst) {
+        if (!parentFirst) {
             Iterator<URL> childResources = Iterators.forEnumeration(findResources(name));
             resources.add(childResources);
         }
@@ -134,7 +138,7 @@ public class PluginClassLoader
         Iterator<URL> parentResources = Iterators.forEnumeration(getParent().getResources(name));
         resources.add(parentResources);
 
-        if (!childFirst) {
+        if (parentFirst) {
             Iterator<URL> childResources = Iterators.forEnumeration(findResources(name));
             resources.add(childResources);
         }
@@ -142,9 +146,9 @@ public class PluginClassLoader
         return Iterators.asEnumeration(Iterators.concat(resources.iterator()));
     }
 
-    private boolean isInChildFirstPackage(String name)
+    private boolean isParentFirstPackage(String name)
     {
-        for (String pkg : CHILD_FIRST_PACKAGES) {
+        for (String pkg : parentFirstPackagePrefixes) {
             if (name.startsWith(pkg)) {
                 return true;
             }
@@ -152,9 +156,9 @@ public class PluginClassLoader
         return false;
     }
 
-    private boolean isInChildFirstPath(String name)
+    private boolean isParentFirstPath(String name)
     {
-        for (String path : CHILD_FIRST_PATHS) {
+        for (String path : parentFirstResourcePrefixes) {
             if (name.startsWith(path)) {
                 return true;
             }

data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderFactory.java

@@ -0,0 +1,9 @@
+package org.embulk.plugin;
+
+import java.util.Collection;
+import java.net.URL;
+
+public interface PluginClassLoaderFactory
+{
+    public PluginClassLoader create(Collection<URL> urls, ClassLoader parentClassLoader);
+}

data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderModule.java

@@ -0,0 +1,72 @@
+package org.embulk.plugin;
+
+import java.util.Collection;
+import java.util.Properties;
+import java.net.URL;
+import java.io.InputStream;
+import java.io.IOException;
+import com.google.inject.Inject;
+import com.google.inject.Module;
+import com.google.inject.Binder;
+import com.google.inject.Scopes;
+import com.google.inject.Provider;
+import org.embulk.config.ConfigSource;
+
+public class PluginClassLoaderModule
+        implements Module
+{
+    public PluginClassLoaderModule(ConfigSource systemConfig)
+    { }
+
+    @Override
+    public void configure(Binder binder)
+    {
+        binder.bind(PluginClassLoaderFactory.class).toProvider(new FactoryProvider()).in(Scopes.SINGLETON);;
+    }
+
+    private static class FactoryProvider
+            implements Provider<PluginClassLoaderFactory>
+    {
+        private final Collection<String> parentFirstPackages;
+        private final Collection<String> parentFirstResources;
+        private final PluginClassLoaderFactory factory;
+
+        public FactoryProvider()
+        {
+            // TODO make these paths customizable using ConfigSource
+            this.parentFirstPackages = readPropertyKeys("/embulk/parent_first_packages.properties");
+            this.parentFirstResources = readPropertyKeys("/embulk/parent_first_resources.properties");
+
+            this.factory = new Factory();
+        }
+
+        private static Collection<String> readPropertyKeys(String name)
+        {
+            try (InputStream in = PluginClassLoaderModule.class.getResourceAsStream(name)) {
+                if (in == null) {
+                    throw new NullPointerException(String.format("Resource '%s' is not found in classpath. Jar file or classloader is broken.", name));
+                }
+                Properties prop = new Properties();
+                prop.load(in);
+                return prop.stringPropertyNames();
+            } catch (IOException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+
+        @Override
+        public PluginClassLoaderFactory get()
+        {
+            return factory;
+        }
+
+        private class Factory implements PluginClassLoaderFactory
+        {
+            public PluginClassLoader create(Collection<URL> urls, ClassLoader parentClassLoader)
+            {
+                return new PluginClassLoader(urls, parentClassLoader,
+                        parentFirstPackages, parentFirstResources);
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/Exec.java

@@ -88,6 +88,12 @@ public class Exec
         return session().newTaskSource();
     }
 
+    // TODO this method is still beta
+    public static TempFileSpace getTempFileSpace()
+    {
+        return session().getTempFileSpace();
+    }
+
     public static boolean isPreview()
     {
         return session().isPreview();

data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java

@@ -1,5 +1,6 @@
 package org.embulk.spi;
 
+import java.io.File;
 import org.joda.time.DateTimeZone;
 import org.slf4j.Logger;
 import org.slf4j.ILoggerFactory;
@@ -14,6 +15,7 @@ import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskSource;
 import org.embulk.config.DataSourceImpl;
+import org.embulk.exec.TempFileAllocator;
 import org.embulk.plugin.PluginType;
 import org.embulk.plugin.PluginManager;
 import org.embulk.spi.time.Timestamp;
@@ -28,25 +30,67 @@ public class ExecSession
     private final PluginManager pluginManager;
     private final BufferAllocator bufferAllocator;
 
-    private final ConfigSource execConfig;
     private final Timestamp transactionTime;
-    private final DateTimeZone transactionTimeZone;
+    private final TempFileSpace tempFileSpace;
 
     private final boolean preview;
 
+    @Deprecated
     public interface SessionTask
             extends Task
     {
         @Config("transaction_time")
         @ConfigDefault("null")
         Optional<Timestamp> getTransactionTime();
+    }
+
+    public static class Builder
+    {
+        private final Injector injector;
+        private Timestamp transactionTime;
+
+        public Builder(Injector injector)
+        {
+            this.injector = injector;
+        }
 
-        @Config("transaction_time_zone")
-        @ConfigDefault("\"UTC\"")
-        DateTimeZone getTransactionTimeZone();
+        public Builder fromExecConfig(ConfigSource configSource)
+        {
+            this.transactionTime = configSource.get(Timestamp.class, "transaction_time", null);
+            return this;
+        }
+
+        public Builder setTransactionTime(Timestamp timestamp)
+        {
+            this.transactionTime = timestamp;
+            return this;
+        }
+
+        public ExecSession build()
+        {
+            if (transactionTime == null) {
+                transactionTime = Timestamp.ofEpochMilli(System.currentTimeMillis());  // TODO get nanoseconds for default
+            }
+            return new ExecSession(injector, transactionTime);
+        }
     }
 
-    public ExecSession(Injector injector, ConfigSource execConfig)
+    public static Builder builder(Injector injector)
+    {
+        return new Builder(injector);
+    }
+
+    @Deprecated
+    public ExecSession(Injector injector, ConfigSource configSource)
+    {
+        this(injector,
+                configSource.loadConfig(SessionTask.class).getTransactionTime().or(
+                    Timestamp.ofEpochMilli(System.currentTimeMillis())
+                )
+            );  // TODO get nanoseconds for default
+    }
+
+    private ExecSession(Injector injector, Timestamp transactionTime)
     {
         this.injector = injector;
         this.loggerFactory = injector.getInstance(ILoggerFactory.class);
@@ -54,10 +98,10 @@ public class ExecSession
         this.pluginManager = injector.getInstance(PluginManager.class);
         this.bufferAllocator = injector.getInstance(BufferAllocator.class);
 
-        this.execConfig = execConfig.deepCopy();
-        SessionTask task = execConfig.loadConfig(SessionTask.class);
-        this.transactionTime = task.getTransactionTime().or(Timestamp.ofEpochMilli(System.currentTimeMillis()));  // TODO get nanoseconds for default
-        this.transactionTimeZone = task.getTransactionTimeZone();
+        this.transactionTime = transactionTime;
+
+        TempFileAllocator tempFileAllocator = injector.getInstance(TempFileAllocator.class);
+        this.tempFileSpace = tempFileAllocator.newSpace(transactionTime.toString());
 
         this.preview = false;
     }
@@ -70,23 +114,21 @@ public class ExecSession
         this.pluginManager = copy.pluginManager;
         this.bufferAllocator = copy.bufferAllocator;
 
-        this.execConfig = copy.execConfig;
         this.transactionTime = copy.transactionTime;
-        this.transactionTimeZone = copy.transactionTimeZone;
+        this.tempFileSpace = copy.tempFileSpace;
 
         this.preview = preview;
     }
 
-    public ExecSession copyForPreview()
+    public ExecSession forPreview()
     {
         return new ExecSession(this, true);
     }
 
-    public ConfigSource getSessionConfigSource()
+    public ConfigSource getSessionExecConfig()
     {
         return newConfigSource()
-            .set("transaction_time", transactionTime)
-            .set("transaction_time_zone", transactionTimeZone);
+            .set("transaction_time", transactionTime);
     }
 
     public Injector getInjector()
@@ -99,11 +141,6 @@ public class ExecSession
         return transactionTime;
     }
 
-    public DateTimeZone getTransactionTimeZone()
-    {
-        return transactionTimeZone;
-    }
-
     public Logger getLogger(String name)
     {
         return loggerFactory.getLogger(name);
@@ -124,11 +161,6 @@ public class ExecSession
         return modelManager;
     }
 
-    public ConfigSource getExecConfig()
-    {
-        return execConfig;
-    }
-
     public <T> T newPlugin(Class<T> iface, PluginType type)
     {
         return pluginManager.newPlugin(iface, type);
@@ -162,8 +194,18 @@ public class ExecSession
         return new TimestampFormatter(format, formatterTask);
     }
 
+    public TempFileSpace getTempFileSpace()
+    {
+        return tempFileSpace;
+    }
+
     public boolean isPreview()
     {
         return preview;
     }
+
+    public void cleanup()
+    {
+        tempFileSpace.cleanup();
+    }
 }

data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java

@@ -81,7 +81,7 @@ public class FileInputRunner
         }
 
         GuessExecutor guessExecutor = Exec.getInjector().getInstance(GuessExecutor.class);
-        return guessExecutor.guessParserConfig(sample, config, Exec.session().getExecConfig());
+        return guessExecutor.guessParserConfig(sample, config, Exec.session().getSessionExecConfig());
     }
 
     private class RunnerControl

data/embulk-core/src/main/java/org/embulk/spi/TaskState.java

@@ -30,11 +30,7 @@ public class TaskState
     public void setException(Throwable exception)
     {
         this.started = true;
-        if (exception == null) {
-            this.exception = Optional.absent();
-        } else {
-            this.exception = Optional.of(exception);
-        }
+        this.exception = Optional.fromNullable(exception);
     }
 
     public void resetException()

data/embulk-core/src/main/java/org/embulk/spi/TempFileException.java

@@ -0,0 +1,19 @@
+package org.embulk.spi;
+
+import java.io.IOException;
+
+public class TempFileException
+        extends RuntimeException
+{
+    public TempFileException(IOException cause)
+    {
+        super(cause);
+    }
+
+
+    @Override
+    public IOException getCause()
+    {
+        return (IOException) super.getCause();
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/TempFileSpace.java

@@ -0,0 +1,56 @@
+package org.embulk.spi;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.io.File;
+import java.io.IOException;
+import com.google.common.base.Preconditions;
+
+public class TempFileSpace
+{
+    private final File dir;
+    private boolean dirCreated;
+
+    public TempFileSpace(File dir)
+    {
+        Preconditions.checkArgument(dir != null, "dir is null");
+        this.dir = dir;
+    }
+
+    public File createTempFile()
+    {
+        return createTempFile("tmp");
+    }
+
+    public File createTempFile(String fileExt)
+    {
+        return createTempFile(Thread.currentThread().getName()+"_", fileExt);
+    }
+
+    public File createTempFile(String prefix, String fileExt)
+    {
+        try {
+            if (!dirCreated) {
+                dir.mkdirs();
+                dirCreated = true;
+            }
+            return File.createTempFile(prefix, "."+fileExt, dir);
+        } catch (IOException ex) {
+            throw new TempFileException(ex);
+        }
+    }
+
+    public void cleanup()
+    {
+        File[] files = dir.listFiles();
+        if (files != null) {
+            for (File e : files) {
+                e.delete();
+                // TODO delete directory recursively
+            }
+        }
+        dir.delete();
+        dirCreated = false;
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/unit/LocalFile.java

@@ -0,0 +1,106 @@
+package org.embulk.spi.unit;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ByteArrayInputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.Files;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import org.embulk.spi.Exec;
+import org.embulk.spi.TempFileSpace;
+import org.embulk.spi.TempFileException;
+
+public class LocalFile
+{
+    public static LocalFile of(File path) throws IOException
+    {
+        return of(path.toPath());
+    }
+
+    public static LocalFile of(Path path) throws IOException
+    {
+        return new LocalFile(path, Files.readAllBytes(path));
+    }
+
+    public static LocalFile of(String path) throws IOException
+    {
+        return of(Paths.get(path));
+    }
+
+    public static LocalFile ofContent(byte[] content)
+    {
+        return new LocalFile(content);
+    }
+
+    public static LocalFile ofContent(String content)
+    {
+        return new LocalFile(content.getBytes(StandardCharsets.UTF_8));
+    }
+
+    private Path path;
+    private final byte[] content;
+
+    private LocalFile(Path path, byte[] content)
+    {
+        this.path = path;
+        this.content = content;
+    }
+
+    private LocalFile(byte[] content)
+    {
+        this.path = null;
+        this.content = content;
+    }
+
+    public File getFile()
+    {
+        return getPath(Exec.getTempFileSpace()).toFile();
+    }
+
+    public File getFile(TempFileSpace space)
+    {
+        return getPath(space).toFile();
+    }
+
+    public Path getPath()
+    {
+        return getPath(Exec.getTempFileSpace());
+    }
+
+    public synchronized Path getPath(TempFileSpace tempFileSpace)
+    {
+        if (path == null) {
+            Path temp = tempFileSpace.createTempFile().toPath();
+            try {
+                Files.write(temp, content);
+            } catch (IOException ex) {
+                throw new TempFileException(ex);
+            }
+            this.path = temp;
+        }
+        return path;
+    }
+
+    public byte[] getContent()
+    {
+        return content;
+    }
+
+    public String getContentAsString()
+    {
+        return new String(content);
+    }
+
+    public String getContentAsString(Charset charset)
+    {
+        return new String(content, charset);
+    }
+
+    public InputStream newContentInputStream()
+    {
+        return new ByteArrayInputStream(content);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/unit/LocalFileSerDe.java

@@ -0,0 +1,116 @@
+package org.embulk.spi.unit;
+
+import java.io.File;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.module.guice.ObjectMapperModule;
+
+public class LocalFileSerDe
+{
+    public static void configure(ObjectMapperModule mapper)
+    {
+        SimpleModule module = new SimpleModule();
+        module.addSerializer(LocalFile.class, new LocalFileSerializer());
+        module.addDeserializer(LocalFile.class, new LocalFileDeserializer());
+        mapper.registerModule(module);
+    }
+
+    private static class LocalFileSerializer
+            extends JsonSerializer<LocalFile>
+    {
+        @Override
+        public void serialize(LocalFile value, JsonGenerator jgen, SerializerProvider provider)
+                throws IOException
+        {
+            jgen.writeStartObject();
+            jgen.writeFieldName("base64");
+            jgen.writeBinary(value.getContent());
+            jgen.writeEndObject();
+        }
+    }
+
+    private static class LocalFileDeserializer
+            extends JsonDeserializer<LocalFile>
+    {
+        private final File tempDir;
+
+        public LocalFileDeserializer()
+        {
+            this(new File(System.getProperty("java.io.tmpdir")));
+        }
+
+        public LocalFileDeserializer(File tempDir)
+        {
+            this.tempDir = tempDir;
+        }
+
+        @Override
+        public LocalFile deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException
+        {
+            JsonToken t = jp.getCurrentToken();
+            if (t == JsonToken.START_OBJECT) {
+                t = jp.nextToken();
+            }
+
+            switch(t) {
+            case VALUE_NULL:
+                return null;
+
+            case FIELD_NAME:
+                {
+                    LocalFile result;
+
+                    String keyName = jp.getCurrentName();
+                    if ("content".equals(keyName)) {
+                        jp.nextToken();
+                        result = LocalFile.ofContent(jp.getValueAsString());
+                    } else if ("base64".equals(keyName)) {
+                        jp.nextToken();
+                        ByteArrayOutputStream out = new ByteArrayOutputStream();
+                        jp.readBinaryValue(ctxt.getBase64Variant(), out);
+                        result = LocalFile.ofContent(out.toByteArray());
+                    } else {
+                        throw ctxt.mappingException("Unknown key '"+keyName+"' to deserialize LocalFile");
+                    }
+
+                    t = jp.nextToken();
+                    if (t != JsonToken.END_OBJECT) {
+                        throw ctxt.mappingException("Unexpected extra map keys to LocalFile");
+                    }
+                    return result;
+                }
+
+            case END_OBJECT:
+            case START_ARRAY:
+            case END_ARRAY:
+                throw ctxt.mappingException("Attempted unexpected map or array to LocalFile");
+
+            case VALUE_EMBEDDED_OBJECT:
+                {
+                    Object obj = jp.getEmbeddedObject();
+                    if (obj == null) {
+                        return null;
+                    }
+                    if (LocalFile.class.isAssignableFrom(obj.getClass())) {
+                        return (LocalFile) obj;
+                    }
+                    throw ctxt.mappingException("Don't know how to convert embedded Object of type "+obj.getClass().getName()+" into LocalFile");
+                }
+
+            default:
+                return LocalFile.of(jp.getValueAsString());
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/Inputs.java

@@ -62,13 +62,4 @@ public class Inputs
             }
         };
     }
-
-    public static String formatPath(String pathFormat)
-    {
-        Timestamp timestamp = Exec.session().getTransactionTime();
-        DateTimeZone timezone = Exec.session().getTransactionTimeZone();
-        // newTimestampFormatter (eventually calls org.jruby.util.RubyDateFormat.<init> doesn't throw exceptions
-        TimestampFormatter formatter = Exec.session().newTimestampFormatter(pathFormat, timezone);
-        return formatter.format(timestamp);  // TimestampFormatter.format doesn't throw exceptions
-    }
 }

data/embulk-core/src/main/resources/embulk/parent_first_packages.properties

@@ -0,0 +1,70 @@
+# generated by './gradlew updateResources' at 2015-06-30 23:52:09 UTC
+ch.qos.logback.classic
+ch.qos.logback.core
+com.fasterxml.jackson.annotation
+com.fasterxml.jackson.core
+com.fasterxml.jackson.databind
+com.fasterxml.jackson.datatype.guava
+com.fasterxml.jackson.datatype.joda
+com.fasterxml.jackson.module.guice
+com.google.common.annotations
+com.google.common.base
+com.google.common.cache
+com.google.common.collect
+com.google.common.escape
+com.google.common.eventbus
+com.google.common.hash
+com.google.common.html
+com.google.common.io
+com.google.common.math
+com.google.common.net
+com.google.common.primitives
+com.google.common.reflect
+com.google.common.util.concurrent
+com.google.common.xml
+com.google.inject
+com.google.thirdparty.publicsuffix
+com.headius.invokebinder
+com.headius.options
+com.ibm.icu.impl
+com.ibm.icu.lang
+com.ibm.icu.math
+com.ibm.icu.text
+com.ibm.icu.util
+com.jcraft.jzlib
+com.kenai.constantine
+com.kenai.jffi
+com.kenai.jnr.x86asm
+com.martiansoftware.nailgun
+edu.umd.cs.findbugs.annotations
+io.airlift.slice
+io.netty.buffer
+io.netty.util
+javax.annotation
+javax.inject
+javax.validation
+jnr.constants
+jnr.enxio.channels
+jnr.enxio.example
+jnr.ffi
+jnr.netdb
+jnr.posix
+jnr.unixsocket
+jnr.x86asm
+net.jcip.annotations
+org.aopalliance.aop
+org.aopalliance.intercept
+org.apache.bval
+org.apache.commons.beanutils
+org.apache.commons.collections
+org.apache.commons.lang3
+org.embulk
+org.fusesource.hawtjni.runtime
+org.fusesource.jansi
+org.jcodings
+org.joda.time
+org.joni
+org.jruby
+org.slf4j
+org.yaml.snakeyaml
+org.yecht

data/embulk-core/src/main/resources/embulk/parent_first_resources.properties

@@ -0,0 +1,28 @@
+# generated by './gradlew updateResources' at 2015-06-30 23:52:09 UTC
+ch/qos/logback/classic/boolex
+ch/qos/logback/classic/db/script
+com/ibm/icu
+com/martiansoftware/nailgun/builtins
+edu/umd/cs/findbugs/annotations
+embulk
+javax/annotation
+jni/Darwin
+jni/arm-Linux
+jni/i386-Linux
+jni/i386-SunOS
+jni/i386-Windows
+jni/ppc-AIX
+jni/sparcv9-SunOS
+jni/x86_64-FreeBSD
+jni/x86_64-Linux
+jni/x86_64-SunOS
+jni/x86_64-Windows
+jruby
+net/jcip/annotations
+okay
+org/apache/bval/jsr303
+org/joda/time/format
+org/joda/time/tz/data
+tables
+templates
+yaml

data/embulk-core/src/test/java/org/embulk/EmbulkTestRuntime.java

@@ -52,7 +52,7 @@ public class EmbulkTestRuntime
         super(new TestRuntimeModule());
         Injector injector = getInjector();
         ConfigSource execConfig = new DataSourceImpl(injector.getInstance(ModelManager.class));
-        this.exec = new ExecSession(injector, execConfig);
+        this.exec = ExecSession.builder(injector).fromExecConfig(execConfig).build();
     }
 
     public ExecSession getExec()
@@ -96,6 +96,8 @@ public class EmbulkTestRuntime
                     });
                 } catch (RuntimeException ex) {
                     throw ex.getCause();
+                } finally {
+                    exec.cleanup();
                 }
             }
         };

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.6.16
     release/release-0.6.15
     release/release-0.6.14
     release/release-0.6.13

data/embulk-docs/src/release/release-0.6.16.rst

@@ -0,0 +1,26 @@
+Release 0.6.16
+==================================
+
+General Changes
+------------------
+
+* Class loader of plugins lookup classes from plugin's class loader first. This change solves the problem where a plugin uses newer (or older) version of jar library which conflicts with a jar library used by applications embedding embulk.
+* Added ``ExecSession#cleanup()`` method. Applications embedding embulk as a library needs to call this method at the end of bulk load transaction.
+
+
+Plugin API
+------------------
+
+* Added ``spi.util.TempFileSpace`` and ``spi.Exec.getTempFileSpace``. This utility allows plugins to create temporary files. Because the behavior of cleaning files up is not completely defined yet, it is recommended to test the behavior before deploying code using this API to production.
+* Added ``spi.unit.LocalFile`` class. Plugins can use this class as a mapped value of configuration file when it needs a path to a local file. This class solves a problem that plugins don't work with distributed executor plugins (such as embulk-executor-mapreduce) if the plugins use ``String`` to get a local file path. Using this class, users can also embed contents of the file in the configuration file instead of setting path to a file.
+
+
+Note
+------------------
+
+* This release includes changes around ClassLoader. Most of plugins should work without modification but it's still highly recommended to test your plugins localy before deploying this version to production systems.
+
+
+Release Date
+------------------
+2015-07-01

data/lib/embulk/java/imports.rb

@@ -28,10 +28,10 @@ module Embulk::Java
   java_import 'org.embulk.spi.Column'
   java_import 'org.embulk.spi.type.Type'
   java_import 'org.embulk.spi.type.Types'
-  java_import 'org.embulk.spi.PluginClassLoader'
   java_import 'org.embulk.spi.FileInputRunner'
   java_import 'org.embulk.spi.FileOutputRunner'
   java_import 'org.embulk.spi.Exec'
+  java_import 'org.embulk.plugin.PluginClassLoaderFactory'
 
   # TODO
 end

data/lib/embulk/java_plugin.rb

@@ -6,7 +6,8 @@ module Embulk
     def self.classloader(dir)
       jars = Dir["#{dir}/**/*.jar"]
       urls = jars.map {|jar| java.io.File.new(File.expand_path(jar)).toURI.toURL }
-      classloader = Java::PluginClassLoader.new(JRuby.runtime, urls)
+      factory = Java.injector.getInstance(Java::PluginClassLoaderFactory.java_class)
+      factory.create(urls, JRuby.runtime.getJRubyClassLoader())
     end
 
     def self.register_input(name, class_fqdn, jar_dir)

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.6.15'
+  VERSION = '0.6.16'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.6.15
+  version: 0.6.16
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-29 00:00:00.000000000 Z
+date: 2015-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -148,10 +148,14 @@ files:
 - embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java
 - embulk-core/src/main/java/org/embulk/exec/SetCurrentThreadName.java
 - embulk-core/src/main/java/org/embulk/exec/SystemConfigModule.java
+- embulk-core/src/main/java/org/embulk/exec/TempFileAllocator.java
 - embulk-core/src/main/java/org/embulk/jruby/JRubyPluginSource.java
 - embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.java
 - embulk-core/src/main/java/org/embulk/plugin/BuiltinPluginSourceModule.java
 - embulk-core/src/main/java/org/embulk/plugin/InjectedPluginSource.java
+- embulk-core/src/main/java/org/embulk/plugin/PluginClassLoader.java
+- embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderFactory.java
+- embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderModule.java
 - embulk-core/src/main/java/org/embulk/plugin/PluginManager.java
 - embulk-core/src/main/java/org/embulk/plugin/PluginSource.java
 - embulk-core/src/main/java/org/embulk/plugin/PluginSourceNotMatchException.java
@@ -185,13 +189,14 @@ files:
 - embulk-core/src/main/java/org/embulk/spi/PageOutput.java
 - embulk-core/src/main/java/org/embulk/spi/PageReader.java
 - embulk-core/src/main/java/org/embulk/spi/ParserPlugin.java
-- embulk-core/src/main/java/org/embulk/spi/PluginClassLoader.java
 - embulk-core/src/main/java/org/embulk/spi/ProcessState.java
 - embulk-core/src/main/java/org/embulk/spi/ProcessTask.java
 - embulk-core/src/main/java/org/embulk/spi/Schema.java
 - embulk-core/src/main/java/org/embulk/spi/SchemaConfig.java
 - embulk-core/src/main/java/org/embulk/spi/SchemaConfigException.java
 - embulk-core/src/main/java/org/embulk/spi/TaskState.java
+- embulk-core/src/main/java/org/embulk/spi/TempFileException.java
+- embulk-core/src/main/java/org/embulk/spi/TempFileSpace.java
 - embulk-core/src/main/java/org/embulk/spi/Transactional.java
 - embulk-core/src/main/java/org/embulk/spi/TransactionalFileInput.java
 - embulk-core/src/main/java/org/embulk/spi/TransactionalFileOutput.java
@@ -215,6 +220,8 @@ files:
 - embulk-core/src/main/java/org/embulk/spi/type/TypeDeserializer.java
 - embulk-core/src/main/java/org/embulk/spi/type/Types.java
 - embulk-core/src/main/java/org/embulk/spi/unit/ByteSize.java
+- embulk-core/src/main/java/org/embulk/spi/unit/LocalFile.java
+- embulk-core/src/main/java/org/embulk/spi/unit/LocalFileSerDe.java
 - embulk-core/src/main/java/org/embulk/spi/util/CharsetSerDe.java
 - embulk-core/src/main/java/org/embulk/spi/util/Decoders.java
 - embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnNotFoundException.java
@@ -249,6 +256,8 @@ files:
 - embulk-core/src/main/resources/embulk/logback-color.xml
 - embulk-core/src/main/resources/embulk/logback-console.xml
 - embulk-core/src/main/resources/embulk/logback-file.xml
+- embulk-core/src/main/resources/embulk/parent_first_packages.properties
+- embulk-core/src/main/resources/embulk/parent_first_resources.properties
 - embulk-core/src/test/java/org/embulk/EmbulkTestRuntime.java
 - embulk-core/src/test/java/org/embulk/GuiceBinder.java
 - embulk-core/src/test/java/org/embulk/RandomManager.java
@@ -319,6 +328,7 @@ files:
 - embulk-docs/src/release/release-0.6.13.rst
 - embulk-docs/src/release/release-0.6.14.rst
 - embulk-docs/src/release/release-0.6.15.rst
+- embulk-docs/src/release/release-0.6.16.rst
 - embulk-docs/src/release/release-0.6.2.rst
 - embulk-docs/src/release/release-0.6.3.rst
 - embulk-docs/src/release/release-0.6.4.rst
@@ -435,8 +445,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.6.15.jar
-- classpath/embulk-standards-0.6.15.jar
+- classpath/embulk-core-0.6.16.jar
+- classpath/embulk-standards-0.6.16.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar