embulk 0.1.0

data/embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.java

@@ -0,0 +1,100 @@
+package org.embulk.jruby;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Set;
+import java.io.File;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Module;
+import com.google.inject.Binder;
+import com.google.inject.Scopes;
+import com.google.inject.Provider;
+import com.google.inject.multibindings.Multibinder;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.spi.Dependency;
+import com.google.inject.spi.ProviderWithDependencies;
+import org.jruby.CompatVersion;
+import org.jruby.embed.ScriptingContainer;
+import org.embulk.plugin.PluginSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ModelManager;
+import org.embulk.spi.BufferAllocator;
+
+public class JRubyScriptingModule
+        implements Module
+{
+    public JRubyScriptingModule(ConfigSource systemConfig)
+    {
+        // TODO get jruby-home from systemConfig to call jruby.container.setHomeDirectory
+        // TODO get jruby-load-paths from systemConfig to call jruby.container.setLoadPaths
+    }
+
+    @Override
+    public void configure(Binder binder)
+    {
+        binder.bind(ScriptingContainer.class).toProvider(ScriptingContainerProvider.class).in(Scopes.SINGLETON);
+        //binder.bind(JRubyModule.class).in(Scopes.SINGLETON);
+
+        Multibinder<PluginSource> multibinder = Multibinder.newSetBinder(binder, PluginSource.class);
+        multibinder.addBinding().to(JRubyPluginSource.class);
+    }
+
+    private static class ScriptingContainerProvider
+            implements ProviderWithDependencies<ScriptingContainer>
+    {
+        private final Injector injector;
+
+        @Inject
+        public ScriptingContainerProvider(Injector injector)
+        {
+            this.injector = injector;
+        }
+
+        public ScriptingContainer get()
+        {
+            ScriptingContainer jruby = new ScriptingContainer();
+            jruby.setCompatVersion(CompatVersion.RUBY1_9);
+
+            // Search embulk/java/bootstrap.rb from a $LOAD_PATH.
+            // $LOAD_PATH is set by lib/embulk/command/embulk.rb if Embulk starts
+            // using embulk-cli but it's not set if Embulk is embedded in an application.
+            // Here adds this jar's internal resources to $LOAD_PATH for those applciations.
+
+            List<String> loadPaths = new ArrayList<String>(jruby.getLoadPaths());
+            String coreJarPath = JRubyScriptingModule.class.getProtectionDomain().getCodeSource().getLocation().getPath();
+            if (!loadPaths.contains(coreJarPath)) {
+                loadPaths.add(coreJarPath);
+            }
+            jruby.setLoadPaths(loadPaths);
+
+            // jruby searches embulk/java/bootstrap.rb from the beginning of $LOAD_PATH.
+            jruby.runScriptlet("require 'embulk/java/bootstrap'");
+
+            // set some constants
+            jruby.callMethod(
+                    jruby.runScriptlet("Embulk::Java"),
+                    "const_set", "Injector", injector);
+            jruby.callMethod(
+                    jruby.runScriptlet("Embulk::Java::Injected"),
+                    "const_set", "ModelManager", injector.getInstance(ModelManager.class));
+            jruby.callMethod(
+                    jruby.runScriptlet("Embulk::Java::Injected"),
+                    "const_set", "BufferAllocator", injector.getInstance(BufferAllocator.class));
+
+            // load embulk.rb
+            jruby.runScriptlet("require 'embulk'");
+
+            return jruby;
+        }
+
+        public Set<Dependency<?>> getDependencies()
+        {
+            // get() depends on other modules
+            return ImmutableSet.of(
+                Dependency.get(Key.get(ModelManager.class)),
+                Dependency.get(Key.get(BufferAllocator.class)));
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/BuiltinPluginSourceModule.java

@@ -0,0 +1,17 @@
+package org.embulk.plugin;
+
+import com.google.inject.Module;
+import com.google.inject.Binder;
+import com.google.inject.multibindings.Multibinder;
+
+public class BuiltinPluginSourceModule
+        implements Module
+{
+    @Override
+    public void configure(Binder binder)
+    {
+        Multibinder<PluginSource> multibinder = Multibinder.newSetBinder(binder, PluginSource.class);
+        //multibinder.addBinding().to(LocalDirectoryPluginSource.class);  // TODO
+        multibinder.addBinding().to(InjectedPluginSource.class);
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/InjectedPluginSource.java

@@ -0,0 +1,92 @@
+package org.embulk.plugin;
+
+import com.google.inject.Binder;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.name.Names;
+import com.google.inject.name.Named;
+import com.google.common.base.Preconditions;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.FileInputRunner;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.FileOutputPlugin;
+import org.embulk.spi.FileOutputRunner;
+
+/**
+ * InjectedPluginSource loads plugins bound by Guice.
+ * This plugin source is intended to be used in test cases.
+ * Plugins need to be bound to Binder following:
+ *
+ * // Module
+ * public void configure(Binder binder)
+ * {
+ *     InjectedPluginSource.registerPluginTo(InputPluginclass, "my", MyInputPlugin.class);
+ * }
+ *
+ */
+public class InjectedPluginSource
+        implements PluginSource
+{
+    private final Injector injector;
+
+    @Inject
+    public InjectedPluginSource(Injector injector)
+    {
+        this.injector = injector;
+    }
+
+    public static interface PluginFactory <T>
+    {
+        public T newPlugin(Injector injector);
+    }
+
+    public <T> T newPlugin(Class<T> iface, PluginType type) throws PluginSourceNotMatchException
+    {
+        String name = type.getName();
+        try {
+            PluginFactory<T> factory = (PluginFactory<T>) injector.getInstance(
+                    Key.get(PluginFactory.class, pluginFactoryName(iface, name)));
+            return factory.newPlugin(injector);
+        } catch (com.google.inject.ConfigurationException ex) {
+            throw new PluginSourceNotMatchException();
+        }
+    }
+
+    public static <T> void registerPluginTo(Binder binder, Class<T> iface, String name, final Class<?> impl)
+    {
+        PluginFactory<T> factory;
+        if (FileInputPlugin.class.isAssignableFrom(impl)) {
+            Preconditions.checkArgument(InputPlugin.class.equals(iface));
+            factory = new PluginFactory<T>() {
+                public T newPlugin(Injector injector)
+                {
+                    return (T) new FileInputRunner((FileInputPlugin) injector.getInstance(impl));
+                }
+            };
+        } else if (FileOutputPlugin.class.isAssignableFrom(impl)) {
+            Preconditions.checkArgument(OutputPlugin.class.equals(iface));
+            factory = new PluginFactory<T>() {
+                public T newPlugin(Injector injector)
+                {
+                    return (T) new FileOutputRunner((FileOutputPlugin) injector.getInstance(impl));
+                }
+            };
+        } else {
+            Preconditions.checkArgument(iface.isAssignableFrom(impl));
+            factory = new PluginFactory<T>() {
+                public T newPlugin(Injector injector)
+                {
+                    return (T) injector.getInstance(impl);
+                }
+            };
+        }
+        binder.bind(PluginFactory.class).annotatedWith(pluginFactoryName(iface, name)).toInstance(factory);
+    }
+
+    private static Named pluginFactoryName(Class<?> iface, String name)
+    {
+        return Names.named(iface.getName() + "." + name);
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/PluginManager.java

@@ -0,0 +1,34 @@
+package org.embulk.plugin;
+
+import java.util.Set;
+import java.util.List;
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import org.embulk.config.ConfigException;
+
+public class PluginManager
+{
+    private final List<PluginSource> sources;
+    private final Injector injector;
+
+    // Set<PluginSource> is injected BuiltinPluginSourceModule or extensions
+    // using Multibinder<PluginSource>.
+    @Inject
+    public PluginManager(Set<PluginSource> pluginSources, Injector injector)
+    {
+        this.sources = ImmutableList.copyOf(pluginSources);
+        this.injector = injector;
+    }
+
+    public <T> T newPlugin(Class<T> iface, PluginType type)
+    {
+        for (PluginSource source : sources) {
+            try {
+                return source.newPlugin(iface, type);
+            } catch (PluginSourceNotMatchException e) {
+            }
+        }
+        throw new ConfigException("Plugin not found");  // TODO exception message should include type in original format
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/PluginSource.java

@@ -0,0 +1,6 @@
+package org.embulk.plugin;
+
+public interface PluginSource
+{
+    public <T> T newPlugin(Class<T> iface, PluginType type) throws PluginSourceNotMatchException;
+}

data/embulk-core/src/main/java/org/embulk/plugin/PluginSourceNotMatchException.java

@@ -0,0 +1,19 @@
+package org.embulk.plugin;
+
+public class PluginSourceNotMatchException
+        extends Exception
+{
+    public PluginSourceNotMatchException()
+    {
+    }
+
+    public PluginSourceNotMatchException(String message)
+    {
+        super(message);
+    }
+
+    public PluginSourceNotMatchException(Throwable cause)
+    {
+        super(cause);
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/PluginType.java

@@ -0,0 +1,47 @@
+package org.embulk.plugin;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+public class PluginType
+{
+    private final String name;
+
+    // TODO accept isObject()/ObjectNode for complex PluginSource
+    @JsonCreator
+    public PluginType(String name)
+    {
+        if (name == null) {
+            throw new NullPointerException("name must not be null");
+        }
+        this.name = name;
+    }
+
+    @JsonValue
+    public String getName()
+    {
+        return name;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return name.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object other)
+    {
+        if (!(other instanceof PluginType)) {
+            return false;
+        }
+        PluginType o = (PluginType) other;
+        return name.equals(o.name);
+    }
+
+    @Override
+    public String toString()
+    {
+        return name;
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/SetThreadContextClassLoader.java

@@ -0,0 +1,19 @@
+package org.embulk.plugin;
+
+public class SetThreadContextClassLoader
+        implements AutoCloseable
+{
+    private final ClassLoader original;
+
+    public SetThreadContextClassLoader(ClassLoader classLoader)
+    {
+        this.original = Thread.currentThread().getContextClassLoader();
+        Thread.currentThread().setContextClassLoader(classLoader);
+    }
+
+    @Override
+    public void close()
+    {
+        Thread.currentThread().setContextClassLoader(original);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/Buffer.java

@@ -0,0 +1,113 @@
+package org.embulk.spi;
+
+import java.util.Arrays;
+
+public class Buffer
+{
+    public static final Buffer EMPTY = Buffer.allocate(0);
+
+    private byte[] array;
+    private int offset;
+    private int filled;
+    private final int capacity;
+
+    protected Buffer(byte[] wrap, int offset, int capacity)
+    {
+        this.array = wrap;
+        this.offset = offset;
+        this.capacity = capacity;
+        this.filled = offset;
+        if (array.length < offset + capacity) {
+            // TODO
+            throw new IllegalStateException("capacity out of bound");
+        }
+    }
+
+    public static Buffer allocate(int length)
+    {
+        return new Buffer(new byte[length], 0, length);
+    }
+
+    public static Buffer copyOf(byte[] src)
+    {
+        return copyOf(src, 0, src.length);
+    }
+
+    public static Buffer copyOf(byte[] src, int index, int length)
+    {
+        return wrap(Arrays.copyOfRange(src, index, length));
+    }
+
+    public static Buffer wrap(byte[] src)
+    {
+        return wrap(src, 0, src.length);
+    }
+
+    public static Buffer wrap(byte[] src, int offset, int size)
+    {
+        return new Buffer(src, offset, size).limit(size);
+    }
+
+    public byte[] array()
+    {
+        return array;
+    }
+
+    public int offset()
+    {
+        return offset;
+    }
+
+    public Buffer offset(int offset)
+    {
+        this.offset = offset;
+        return this;
+    }
+
+    public int limit()
+    {
+        return filled - offset;
+    }
+
+    public Buffer limit(int limit)
+    {
+        if (capacity < limit) {
+            // TODO
+            throw new IllegalStateException("limit index out of bound: capacity="+capacity+" limit="+limit);
+        }
+        this.filled = offset + limit;
+        return this;
+    }
+
+    public int capacity()
+    {
+        return capacity;
+    }
+
+    public void setBytes(int index, byte[] source, int sourceIndex, int length)
+    {
+        System.arraycopy(source, sourceIndex, array, offset + index, length);
+    }
+
+    public void setBytes(int index, Buffer source, int sourceIndex, int length)
+    {
+        setBytes(index, source.array(), source.offset() + sourceIndex, length);
+    }
+
+    public void getBytes(int index, byte[] dest, int destIndex, int length)
+    {
+        System.arraycopy(array, offset + index, dest, destIndex, length);
+    }
+
+    public void getBytes(int index, Buffer dest, int destIndex, int length)
+    {
+        getBytes(index, dest.array(), dest.offset() + destIndex, length);
+    }
+
+    public void release()
+    {
+    }
+
+    // TODO equals
+    // TODO hashCode
+}