embulk 0.7.0-java

data/embulk-core/src/main/java/org/embulk/guice/LifeCycleMethodsMap.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2010 Proofpoint, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * Copyright 2015 Sadayuki Furuhashi
+ */
+package org.embulk.guice;
+
+import com.google.common.collect.Maps;
+
+import java.util.Map;
+
+class LifeCycleMethodsMap
+{
+    private final Map<Class<?>, LifeCycleMethods> map = Maps.newHashMap();
+
+    synchronized LifeCycleMethods get(Class<?> clazz)
+    {
+        LifeCycleMethods methods = map.get(clazz);
+        if (methods == null) {
+            methods = new LifeCycleMethods(clazz);
+            map.put(clazz, methods);
+        }
+        return methods;
+    }
+}

data/embulk-core/src/main/java/org/embulk/guice/LifeCycleModule.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2010 Proofpoint, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * Copyright 2015 Sadayuki Furuhashi
+ */
+package org.embulk.guice;
+
+import com.google.common.collect.Lists;
+import com.google.inject.Binder;
+import com.google.inject.Module;
+import com.google.inject.Provides;
+import com.google.inject.Singleton;
+import com.google.inject.TypeLiteral;
+import com.google.inject.spi.InjectionListener;
+import com.google.inject.spi.TypeEncounter;
+import com.google.inject.spi.TypeListener;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static com.google.inject.matcher.Matchers.any;
+
+/**
+ * Guice module for binding the LifeCycle manager
+ */
+public class LifeCycleModule implements Module
+{
+    private final List<Object> injectedInstances = Lists.newArrayList();
+    private final LifeCycleMethodsMap lifeCycleMethodsMap = new LifeCycleMethodsMap();
+    private final AtomicReference<LifeCycleManager> lifeCycleManagerRef = new AtomicReference<LifeCycleManager>(null);
+
+    @Override
+    public void configure(Binder binder)
+    {
+        binder.disableCircularProxies();
+
+        binder.bindListener(any(), new TypeListener()
+        {
+            @Override
+            public <T> void hear(TypeLiteral<T> type, TypeEncounter<T> encounter)
+            {
+                encounter.register(new InjectionListener<T>()
+                {
+                    @Override
+                    public void afterInjection(T obj)
+                    {
+                        if (isLifeCycleClass(obj.getClass())) {
+                            LifeCycleManager lifeCycleManager = lifeCycleManagerRef.get();
+                            if (lifeCycleManager != null) {
+                                try {
+                                    lifeCycleManager.addInstance(obj);
+                                }
+                                catch (Exception e) {
+                                    throw new Error(e);
+                                }
+                            }
+                            else {
+                                injectedInstances.add(obj);
+                            }
+                        }
+                    }
+                });
+            }
+        });
+    }
+
+    @Provides
+    @Singleton
+    public LifeCycleManager getServerManager()
+            throws Exception
+    {
+        LifeCycleManager lifeCycleManager = new LifeCycleManager(injectedInstances, lifeCycleMethodsMap);
+        lifeCycleManagerRef.set(lifeCycleManager);
+        return lifeCycleManager;
+    }
+
+    private boolean isLifeCycleClass(Class<?> clazz)
+    {
+        LifeCycleMethods methods = lifeCycleMethodsMap.get(clazz);
+        return methods.hasFor(PostConstruct.class) || methods.hasFor(PreDestroy.class);
+    }
+}

data/embulk-core/src/main/java/org/embulk/jruby/JRubyPluginSource.java

@@ -0,0 +1,72 @@
+package org.embulk.jruby;
+
+import com.google.inject.Inject;
+import org.jruby.embed.ScriptingContainer;
+import org.jruby.embed.InvokeFailedException;
+import org.embulk.plugin.PluginType;
+import org.embulk.plugin.PluginSource;
+import org.embulk.plugin.PluginSourceNotMatchException;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.FormatterPlugin;
+import org.embulk.spi.DecoderPlugin;
+import org.embulk.spi.EncoderPlugin;
+import org.embulk.spi.FilterPlugin;
+import org.embulk.spi.GuessPlugin;
+import org.embulk.spi.ExecutorPlugin;
+
+public class JRubyPluginSource
+        implements PluginSource
+{
+    private final ScriptingContainer jruby;
+    private final Object rubyPluginManager;
+
+    @Inject
+    public JRubyPluginSource(ScriptingContainer jruby)
+    {
+        this.jruby = jruby;
+
+        // get Embulk::Plugin
+        //this.rubyPluginManager = ((RubyModule) jruby.get("Embulk")).const_get(
+        //        RubySymbol.newSymbol(
+        //            jruby.getProvider().getRuntime(), "Plugin"));
+        this.rubyPluginManager = jruby.runScriptlet("Embulk::Plugin");
+    }
+
+    public <T> T newPlugin(Class<T> iface, PluginType type) throws PluginSourceNotMatchException
+    {
+        String name = type.getName();
+
+        String category;
+        if (InputPlugin.class.isAssignableFrom(iface)) {
+            category = "input";
+        } else if (OutputPlugin.class.isAssignableFrom(iface)) {
+            category = "output";
+        } else if (ParserPlugin.class.isAssignableFrom(iface)) {
+            category = "parser";
+        } else if (FormatterPlugin.class.isAssignableFrom(iface)) {
+            category = "formatter";
+        } else if (DecoderPlugin.class.isAssignableFrom(iface)) {
+            category = "decoder";
+        } else if (EncoderPlugin.class.isAssignableFrom(iface)) {
+            category = "encoder";
+        } else if (FilterPlugin.class.isAssignableFrom(iface)) {
+            category = "filter";
+        } else if (GuessPlugin.class.isAssignableFrom(iface)) {
+            category = "guess";
+        } else if (ExecutorPlugin.class.isAssignableFrom(iface)) {
+            category = "executor";
+        } else {
+            // unsupported plugin category
+            throw new PluginSourceNotMatchException("Plugin interface "+iface+" is not supported in JRuby");
+        }
+
+        String methodName = "new_java_" + category;
+        try {
+            return jruby.callMethod(rubyPluginManager, methodName, name, iface);
+        } catch (InvokeFailedException ex) {
+            throw new PluginSourceNotMatchException(ex.getCause());
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.java

@@ -0,0 +1,119 @@
+package org.embulk.jruby;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Set;
+import org.slf4j.ILoggerFactory;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Module;
+import com.google.inject.Binder;
+import com.google.inject.Scopes;
+import com.google.inject.multibindings.Multibinder;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.spi.Dependency;
+import com.google.inject.spi.ProviderWithDependencies;
+import org.jruby.CompatVersion;
+import org.jruby.embed.LocalContextScope;
+import org.jruby.embed.ScriptingContainer;
+import org.embulk.plugin.PluginSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ModelManager;
+import org.embulk.exec.ForSystemConfig;
+import org.embulk.spi.BufferAllocator;
+
+public class JRubyScriptingModule
+        implements Module
+{
+    public JRubyScriptingModule(ConfigSource systemConfig)
+    {
+    }
+
+    @Override
+    public void configure(Binder binder)
+    {
+        binder.bind(ScriptingContainer.class).toProvider(ScriptingContainerProvider.class).in(Scopes.SINGLETON);
+
+        Multibinder<PluginSource> multibinder = Multibinder.newSetBinder(binder, PluginSource.class);
+        multibinder.addBinding().to(JRubyPluginSource.class);
+    }
+
+    private static class ScriptingContainerProvider
+            implements ProviderWithDependencies<ScriptingContainer>
+    {
+        private final Injector injector;
+        private final boolean useGlobalRubyRuntime;
+
+        @Inject
+        public ScriptingContainerProvider(Injector injector, @ForSystemConfig ConfigSource systemConfig)
+        {
+            this.injector = injector;
+
+            // use_global_ruby_runtime is valid only when it's guaranteed that just one Injector is
+            // instantiated in this JVM.
+            this.useGlobalRubyRuntime = systemConfig.get(boolean.class, "use_global_ruby_runtime", false);
+
+            // TODO get jruby-home from systemConfig to call jruby.container.setHomeDirectory
+            // TODO get jruby-load-paths from systemConfig to call jruby.container.setLoadPaths
+        }
+
+        public ScriptingContainer get()
+        {
+            LocalContextScope scope = (useGlobalRubyRuntime ? LocalContextScope.SINGLETON : LocalContextScope.SINGLETHREAD);
+            ScriptingContainer jruby = new ScriptingContainer(scope);
+            jruby.setCompatVersion(CompatVersion.RUBY1_9);
+
+            // Search embulk/java/bootstrap.rb from a $LOAD_PATH.
+            // $LOAD_PATH is set by lib/embulk/command/embulk_run.rb if Embulk starts
+            // using embulk-cli but it's not set if Embulk is embedded in an application.
+            // Here adds this jar's internal resources to $LOAD_PATH for those applciations.
+
+//            List<String> loadPaths = new ArrayList<String>(jruby.getLoadPaths());
+//            String coreJarPath = JRubyScriptingModule.class.getProtectionDomain().getCodeSource().getLocation().getPath();
+//            if (!loadPaths.contains(coreJarPath)) {
+//                loadPaths.add(coreJarPath);
+//            }
+//            jruby.setLoadPaths(loadPaths);
+
+            // load embulk.rb
+            jruby.runScriptlet("require 'embulk'");
+
+            // jruby searches embulk/java/bootstrap.rb from the beginning of $LOAD_PATH.
+            jruby.runScriptlet("require 'embulk/java/bootstrap'");
+
+            // TODO validate Embulk::Java::Injected::Injector doesn't exist? If it already exists,
+            //      Injector is created more than once in this JVM although use_global_ruby_runtime
+            //      is set to true.
+
+            // set some constants
+            jruby.callMethod(
+                    jruby.runScriptlet("Embulk::Java::Injected"),
+                    "const_set", "Injector", injector);
+            jruby.callMethod(
+                    jruby.runScriptlet("Embulk::Java::Injected"),
+                    "const_set", "ModelManager", injector.getInstance(ModelManager.class));
+            jruby.callMethod(
+                    jruby.runScriptlet("Embulk::Java::Injected"),
+                    "const_set", "BufferAllocator", injector.getInstance(BufferAllocator.class));
+
+            // initialize logger
+            jruby.callMethod(
+                    jruby.runScriptlet("Embulk"),
+                    "logger=",
+                        jruby.callMethod(
+                            jruby.runScriptlet("Embulk::Logger"),
+                            "new", injector.getInstance(ILoggerFactory.class).getLogger("ruby")));
+
+            return jruby;
+        }
+
+        public Set<Dependency<?>> getDependencies()
+        {
+            // get() depends on other modules
+            return ImmutableSet.of(
+                Dependency.get(Key.get(ModelManager.class)),
+                Dependency.get(Key.get(BufferAllocator.class)));
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/BuiltinPluginSourceModule.java

@@ -0,0 +1,17 @@
+package org.embulk.plugin;
+
+import com.google.inject.Module;
+import com.google.inject.Binder;
+import com.google.inject.multibindings.Multibinder;
+
+public class BuiltinPluginSourceModule
+        implements Module
+{
+    @Override
+    public void configure(Binder binder)
+    {
+        Multibinder<PluginSource> multibinder = Multibinder.newSetBinder(binder, PluginSource.class);
+        //multibinder.addBinding().to(LocalDirectoryPluginSource.class);  // TODO
+        multibinder.addBinding().to(InjectedPluginSource.class);
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/InjectedPluginSource.java

@@ -0,0 +1,96 @@
+package org.embulk.plugin;
+
+import com.google.inject.Binder;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.name.Names;
+import com.google.inject.name.Named;
+import com.google.common.base.Preconditions;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.FileInputRunner;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.FileOutputPlugin;
+import org.embulk.spi.FileOutputRunner;
+
+/**
+ * InjectedPluginSource loads plugins bound by Guice.
+ * This plugin source is intended to be used in test cases.
+ * Plugins need to be bound to Binder following:
+ *
+ * // Module
+ * public void configure(Binder binder)
+ * {
+ *     InjectedPluginSource.registerPluginTo(InputPluginclass, "my", MyInputPlugin.class);
+ * }
+ *
+ */
+public class InjectedPluginSource
+        implements PluginSource
+{
+    private final Injector injector;
+
+    @Inject
+    public InjectedPluginSource(Injector injector)
+    {
+        this.injector = injector;
+    }
+
+    public static interface PluginFactory <T>
+    {
+        public T newPlugin(Injector injector);
+    }
+
+    public <T> T newPlugin(Class<T> iface, PluginType type) throws PluginSourceNotMatchException
+    {
+        String name = type.getName();
+        try {
+            @SuppressWarnings("unchecked")
+            PluginFactory<T> factory = (PluginFactory<T>) injector.getInstance(
+                    Key.get(PluginFactory.class, pluginFactoryName(iface, name)));
+            return factory.newPlugin(injector);
+        } catch (com.google.inject.ConfigurationException ex) {
+            throw new PluginSourceNotMatchException();
+        }
+    }
+
+    public static <T> void registerPluginTo(Binder binder, Class<T> iface, String name, final Class<?> impl)
+    {
+        PluginFactory<T> factory;
+        if (FileInputPlugin.class.isAssignableFrom(impl)) {
+            Preconditions.checkArgument(InputPlugin.class.equals(iface));
+            factory = new PluginFactory<T>() {
+                @SuppressWarnings("unchecked")
+                public T newPlugin(Injector injector)
+                {
+                    return (T) new FileInputRunner((FileInputPlugin) injector.getInstance(impl));
+                }
+            };
+        } else if (FileOutputPlugin.class.isAssignableFrom(impl)) {
+            Preconditions.checkArgument(OutputPlugin.class.equals(iface));
+            factory = new PluginFactory<T>() {
+                @SuppressWarnings("unchecked")
+                public T newPlugin(Injector injector)
+                {
+                    return (T) new FileOutputRunner((FileOutputPlugin) injector.getInstance(impl));
+                }
+            };
+        } else {
+            Preconditions.checkArgument(iface.isAssignableFrom(impl));
+            factory = new PluginFactory<T>() {
+                @SuppressWarnings("unchecked")
+                public T newPlugin(Injector injector)
+                {
+                    return (T) injector.getInstance(impl);
+                }
+            };
+        }
+        binder.bind(PluginFactory.class).annotatedWith(pluginFactoryName(iface, name)).toInstance(factory);
+    }
+
+    private static Named pluginFactoryName(Class<?> iface, String name)
+    {
+        return Names.named(iface.getName() + "." + name);
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoader.java

@@ -0,0 +1,168 @@
+package org.embulk.plugin;
+
+import java.util.List;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.net.MalformedURLException;
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+
+public class PluginClassLoader
+        extends URLClassLoader
+{
+    private final List<String> parentFirstPackagePrefixes;
+    private final List<String> parentFirstResourcePrefixes;
+
+    public PluginClassLoader(Collection<URL> urls, ClassLoader parent,
+            Collection<String> parentFirstPackages,
+            Collection<String> parentFirstResources)
+    {
+        super(urls.toArray(new URL[urls.size()]), parent);
+        this.parentFirstPackagePrefixes = ImmutableList.copyOf(
+                Iterables.transform(parentFirstPackages, new Function<String, String>() {
+                    public String apply(String pkg)
+                    {
+                        return pkg + ".";
+                    }
+                }));
+        this.parentFirstResourcePrefixes = ImmutableList.copyOf(
+                Iterables.transform(parentFirstResources, new Function<String, String>() {
+                    public String apply(String pkg)
+                    {
+                        return pkg + "/";
+                    }
+                }));
+    }
+
+    public void addPath(Path path)
+    {
+        try {
+            addUrl(path.toUri().toURL());
+        } catch (MalformedURLException ex) {
+            throw new IllegalArgumentException(ex);
+        }
+    }
+
+    public void addUrl(URL url)
+    {
+        super.addURL(url);
+    }
+
+    @Override
+    protected Class<?> loadClass(String name, boolean resolve)
+            throws ClassNotFoundException
+    {
+        synchronized (getClassLoadingLock(name)) {
+            Class<?> loadedClass = findLoadedClass(name);
+            if (loadedClass != null) {
+                return resolveClass(loadedClass, resolve);
+            }
+
+            boolean parentFirst = isParentFirstPackage(name);
+            if (!parentFirst) {
+                try {
+                    return resolveClass(findClass(name), resolve);
+                } catch (ClassNotFoundException ignored) {
+                }
+            }
+
+            try {
+                return resolveClass(getParent().loadClass(name), resolve);
+            } catch (ClassNotFoundException ignored) {
+            }
+
+            if (parentFirst) {
+                return resolveClass(findClass(name), resolve);
+            }
+
+            throw new ClassNotFoundException(name);
+        }
+    }
+
+    private Class<?> resolveClass(Class<?> clazz, boolean resolve)
+    {
+        if (resolve) {
+            resolveClass(clazz);
+        }
+        return clazz;
+    }
+
+    @Override
+    public URL getResource(String name)
+    {
+        boolean childFirst = isParentFirstPath(name);
+
+        if (childFirst) {
+            URL childUrl = findResource(name);
+            if (childUrl != null) {
+                return childUrl;
+            }
+        }
+
+        URL parentUrl = getParent().getResource(name);
+        if (parentUrl != null) {
+            return parentUrl;
+        }
+
+        if (!childFirst) {
+            URL childUrl = findResource(name);
+            if (childUrl != null) {
+                return childUrl;
+            }
+        }
+
+        return null;
+    }
+
+    @Override
+    public Enumeration<URL> getResources(String name)
+            throws IOException
+    {
+        List<Iterator<URL>> resources = new ArrayList<>();
+
+        boolean parentFirst = isParentFirstPath(name);
+
+        if (!parentFirst) {
+            Iterator<URL> childResources = Iterators.forEnumeration(findResources(name));
+            resources.add(childResources);
+        }
+
+        Iterator<URL> parentResources = Iterators.forEnumeration(getParent().getResources(name));
+        resources.add(parentResources);
+
+        if (parentFirst) {
+            Iterator<URL> childResources = Iterators.forEnumeration(findResources(name));
+            resources.add(childResources);
+        }
+
+        return Iterators.asEnumeration(Iterators.concat(resources.iterator()));
+    }
+
+    private boolean isParentFirstPackage(String name)
+    {
+        for (String pkg : parentFirstPackagePrefixes) {
+            if (name.startsWith(pkg)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private boolean isParentFirstPath(String name)
+    {
+        for (String path : parentFirstResourcePrefixes) {
+            if (name.startsWith(path)) {
+                return true;
+            }
+        }
+        return false;
+    }
+}