embulk 0.6.21 → 0.6.22

data/embulk-core/src/main/java/org/embulk/config/ConfigLoader.java

@@ -26,6 +26,11 @@ public class ConfigLoader
         this.model = model;
     }
 
+    public ConfigSource newConfigSource()
+    {
+        return new DataSourceImpl(model);
+    }
+
     public ConfigSource fromJsonFile(File file) throws IOException
     {
         try (FileInputStream is = new FileInputStream(file)) {

data/embulk-core/src/main/java/org/embulk/config/DataSourceImpl.java

@@ -123,7 +123,7 @@ public class DataSourceImpl
         if (v == null) {
             remove(attrName);
         } else {
-            data.put(attrName, model.writeObjectAsJsonNode(v));
+            data.set(attrName, model.writeObjectAsJsonNode(v));
         }
         return this;
     }
@@ -131,7 +131,7 @@ public class DataSourceImpl
     @Override
     public DataSourceImpl setNested(String attrName, DataSource v)
     {
-        data.put(attrName, v.getObjectNode());
+        data.set(attrName, v.getObjectNode());
         return this;
     }
 
@@ -139,7 +139,7 @@ public class DataSourceImpl
     public DataSourceImpl setAll(DataSource other)
     {
         for (Map.Entry<String, JsonNode> field : other.getAttributes()) {
-            data.put(field.getKey(), field.getValue());
+            data.set(field.getKey(), field.getValue());
         }
         return this;
     }

data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java

@@ -27,7 +27,7 @@ public class ExecModule
     {
         Preconditions.checkNotNull(binder, "binder is null.");
 
-        binder.bind(ILoggerFactory.class).toProvider(LoggerProvider.class);
+        binder.bind(ILoggerFactory.class).toProvider(LoggerProvider.class).in(Scopes.SINGLETON);
         binder.bind(ModelManager.class).in(Scopes.SINGLETON);
         binder.bind(BufferAllocator.class).to(PooledBufferAllocator.class).in(Scopes.SINGLETON);
         binder.bind(TempFileAllocator.class).in(Scopes.SINGLETON);

data/embulk-core/src/main/java/org/embulk/exec/PooledBufferAllocator.java

@@ -3,6 +3,8 @@ package org.embulk.exec;
 import io.netty.buffer.PooledByteBufAllocator;
 import io.netty.buffer.ByteBuf;
 import io.netty.util.ResourceLeakDetector;
+import io.netty.util.internal.logging.InternalLoggerFactory;
+import io.netty.util.internal.logging.JdkLoggerFactory;
 import org.embulk.spi.Buffer;
 import org.embulk.spi.BufferAllocator;
 import com.google.inject.Inject;
@@ -18,7 +20,7 @@ public class PooledBufferAllocator
     private final int pageSize;
 
     @Inject
-    public PooledBufferAllocator(@ForSystemConfig ConfigSource systemConfig)
+    public PooledBufferAllocator(@ForSystemConfig ConfigSource systemConfig, org.slf4j.ILoggerFactory factory)
     {
         this.nettyBuffer = new PooledByteBufAllocator(false);
         this.pageSize = systemConfig.get(ByteSize.class, "page_size", new ByteSize(DEFAULT_PAGE_SIZE)).getBytesInt();

data/embulk-core/src/main/java/org/embulk/guice/Bootstrap.java

@@ -0,0 +1,150 @@
+/*
+ * Copyright 2015 Sadayuki Furuhashi
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.embulk.guice;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.annotations.Beta;
+
+import com.google.inject.Binder;
+import com.google.inject.Guice;
+import com.google.inject.Module;
+import com.google.inject.Stage;
+import com.google.inject.Injector;
+import java.util.Arrays;
+import java.util.List;
+
+@Beta
+public class Bootstrap
+{
+    private List<Module> modules;
+
+    private boolean requireExplicitBindings = true;
+
+    private boolean started;
+
+    public Bootstrap(Module... modules)
+    {
+        this(Arrays.asList(modules));
+    }
+
+    public Bootstrap(Iterable<? extends Module> modules)
+    {
+        this.modules = ImmutableList.copyOf(modules);
+    }
+
+    public Bootstrap requireExplicitBindings(boolean requireExplicitBindings)
+    {
+        this.requireExplicitBindings = requireExplicitBindings;
+        return this;
+    }
+
+    public Bootstrap addModules(Module... additionalModules)
+    {
+        return addModules(Arrays.asList(additionalModules));
+    }
+
+    public Bootstrap addModules(Iterable<? extends Module> additionalModules)
+    {
+        modules = ImmutableList.copyOf(Iterables.concat(modules, additionalModules));
+        return this;
+    }
+
+    //public Bootstrap onPreDestroy()
+    //{
+    //}
+
+    //public Bootstrap onPreDestroyException()
+    //{
+    //}
+
+    //public Bootstrap onStop()
+    //{
+    //}
+
+    //public Bootstrap forEachModule(Consumer<? super Module> function)
+    //{
+    //    for (Module module : modules) {
+    //        function.accept(module);
+    //    }
+    //    return this;
+    //}
+
+    //public <T> Bootstrap forEachModule(Class<T> ifClass, Consumer<? super T> function)
+    //{
+    //    for (Module module : modules) {
+    //        if (ifClass.instance(module) {
+    //            function.accept(module);
+    //        }
+    //    }
+    //    return this;
+    //}
+
+    public Bootstrap overrideModules(Function<? super List<Module>, ? extends Iterable<? extends Module>> function)
+    {
+        modules = ImmutableList.copyOf(function.apply(modules));
+        return this;
+    }
+
+    public Injector initialize()
+    {
+        Injector injector = start();
+        injector.getInstance(LifeCycleManager.class).destroyOnShutdownHook();
+        return injector;
+    }
+
+    public CloseableInjector initializeCloseable()
+    {
+        Injector injector = start();
+        return new CloseableInjectorProxy(injector, injector.getInstance(LifeCycleManager.class));
+    }
+
+    private Injector start()
+    {
+        if (started) {
+            throw new IllegalStateException("System already initialized");
+        }
+        started = true;
+
+        ImmutableList.Builder<Module> moduleList = ImmutableList.builder();
+
+        moduleList.addAll(modules);
+
+        moduleList.add(new Module()
+        {
+            @Override
+            public void configure(Binder binder)
+            {
+                binder.disableCircularProxies();
+                if (requireExplicitBindings) {
+                    binder.requireExplicitBindings();
+                }
+            }
+        });
+
+        moduleList.add(new LifeCycleModule());
+
+        Injector injector = Guice.createInjector(Stage.PRODUCTION, moduleList.build());
+
+        LifeCycleManager lifeCycleManager = injector.getInstance(LifeCycleManager.class);
+        if (lifeCycleManager.size() > 0) {
+            lifeCycleManager.start();
+        }
+
+        return injector;
+    }
+}

data/embulk-core/src/main/java/org/embulk/guice/CloseableInjector.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2015 Sadayuki Furuhashi
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.embulk.guice;
+
+import com.google.inject.Injector;
+
+public interface CloseableInjector
+        extends Injector, AutoCloseable
+{ }

data/embulk-core/src/main/java/org/embulk/guice/CloseableInjectorProxy.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2015 Sadayuki Furuhashi
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.embulk.guice;
+
+import com.google.inject.Injector;
+
+class CloseableInjectorProxy
+        extends InjectorProxy
+        implements CloseableInjector
+{
+    private final Injector injector;
+    private final LifeCycleManager lifeCycleManager;
+
+    public CloseableInjectorProxy(Injector injector, LifeCycleManager lifeCycleManager)
+    {
+        this.injector = injector;
+        this.lifeCycleManager = lifeCycleManager;
+    }
+
+    @Override
+    protected synchronized Injector injector()
+    {
+        if (lifeCycleManager.isDestroyed()) {
+            throw new IllegalStateException("Injector already closed");
+        }
+        return injector;
+    }
+
+    @Override
+    public synchronized void close() throws Exception
+    {
+        lifeCycleManager.destroy();  // reentrant
+    }
+}

data/embulk-core/src/main/java/org/embulk/guice/InjectorProxy.java

@@ -0,0 +1,145 @@
+/*
+ * Copyright 2015 Sadayuki Furuhashi
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.embulk.guice;
+
+import com.google.inject.Binding;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.MembersInjector;
+import com.google.inject.Module;
+import com.google.inject.Provider;
+import com.google.inject.Scope;
+import com.google.inject.TypeLiteral;
+import com.google.inject.spi.TypeConverterBinding;
+
+import java.lang.annotation.Annotation;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+abstract class InjectorProxy
+        implements Injector
+{
+    protected abstract Injector injector();
+
+    @Override
+    public void injectMembers(Object instance)
+    {
+        injector().injectMembers(instance);
+    }
+
+    @Override
+    public <T> MembersInjector<T> getMembersInjector(TypeLiteral<T> typeLiteral)
+    {
+        return injector().getMembersInjector(typeLiteral);
+    }
+
+    @Override
+    public <T> MembersInjector<T> getMembersInjector(Class<T> type)
+    {
+        return injector().getMembersInjector(type);
+    }
+
+    @Override
+    public Map<Key<?>, Binding<?>> getBindings()
+    {
+        return injector().getBindings();
+    }
+
+    @Override
+    public Map<Key<?>, Binding<?>> getAllBindings()
+    {
+        return injector().getAllBindings();
+    }
+
+    @Override
+    public <T> Binding<T> getBinding(Key<T> key)
+    {
+        return injector().getBinding(key);
+    }
+
+    @Override
+    public <T> Binding<T> getBinding(Class<T> type)
+    {
+        return injector().getBinding(type);
+    }
+
+    @Override
+    public <T> Binding<T> getExistingBinding(Key<T> key)
+    {
+        return injector().getExistingBinding(key);
+    }
+
+    @Override
+    public <T> List<Binding<T>> findBindingsByType(TypeLiteral<T> type)
+    {
+        return injector().findBindingsByType(type);
+    }
+
+    @Override
+    public <T> Provider<T> getProvider(Key<T> key)
+    {
+        return injector().getProvider(key);
+    }
+
+    @Override
+    public <T> Provider<T> getProvider(Class<T> type)
+    {
+        return injector().getProvider(type);
+    }
+
+    @Override
+    public <T> T getInstance(Key<T> key)
+    {
+        return injector().getInstance(key);
+    }
+
+    @Override
+    public <T> T getInstance(Class<T> type)
+    {
+        return injector().getInstance(type);
+    }
+
+    @Override
+    public Injector getParent()
+    {
+        return injector().getParent();
+    }
+
+    @Override
+    public Injector createChildInjector(Iterable<? extends Module> modules)
+    {
+        return injector().createChildInjector(modules);
+    }
+
+    @Override
+    public Injector createChildInjector(Module... modules)
+    {
+        return injector().createChildInjector(modules);
+    }
+
+    @Override
+    public Map<Class<? extends Annotation>, Scope> getScopeBindings()
+    {
+        return injector().getScopeBindings();
+    }
+
+    @Override
+    public Set<TypeConverterBinding> getTypeConverterBindings()
+    {
+        return injector().getTypeConverterBindings();
+    }
+}

data/embulk-core/src/main/java/org/embulk/guice/LifeCycleManager.java

@@ -0,0 +1,187 @@
+/*
+ * Copyright 2010 Proofpoint, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * Copyright 2015 Sadayuki Furuhashi
+ */
+package org.embulk.guice;
+
+import com.google.common.collect.Lists;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Collections;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Manages PostConstruct and PreDestroy life cycles
+ */
+public final class LifeCycleManager
+{
+    private final AtomicReference<State> state = new AtomicReference<State>(State.LATENT);
+    private final Queue<Object> managedInstances = new ConcurrentLinkedQueue<Object>();
+    private final LifeCycleMethodsMap methodsMap;
+
+    private enum State
+    {
+        LATENT,
+        STARTING,
+        STARTED,
+        STOPPING,
+        STOPPED
+    }
+
+    /**
+     * @param managedInstances list of objects that have life cycle annotations
+     * @param methodsMap existing or new methods map
+     * @throws Exception exceptions starting instances (depending on mode)
+     */
+    public LifeCycleManager(List<Object> managedInstances, LifeCycleMethodsMap methodsMap)
+            throws Exception
+    {
+        this.methodsMap = (methodsMap != null) ? methodsMap : new LifeCycleMethodsMap();
+        for (Object instance : managedInstances) {
+            addInstance(instance);
+        }
+    }
+
+    /**
+     * Returns the number of managed instances
+     *
+     * @return qty
+     */
+    public int size()
+    {
+        return managedInstances.size();
+    }
+
+    /**
+     * Start the life cycle - all instances will have their {@link javax.annotation.PostConstruct} method(s) called
+     *
+     * @throws Exception errors
+     */
+    public void start()
+    {
+        if (!state.compareAndSet(State.LATENT, State.STARTING)) {
+            throw new IllegalStateException("System already starting");
+        }
+        //log.info("Life cycle starting...");
+
+        for (Object obj : managedInstances) {
+            LifeCycleMethods methods = methodsMap.get(obj.getClass());
+            if (!methods.hasFor(PreDestroy.class)) {
+                managedInstances.remove(obj);   // remove reference to instances that aren't needed anymore
+            }
+        }
+
+        state.set(State.STARTED);
+        //log.info("Life cycle startup complete. System ready.");
+    }
+
+    /**
+     * Add a shutdown hook that calls {@link destroy} method
+     */
+    public void destroyOnShutdownHook()
+    {
+        Runtime.getRuntime().addShutdownHook(new Thread()
+        {
+            @Override
+            public void run()
+            {
+                try {
+                    LifeCycleManager.this.destroy();
+                }
+                catch (Exception e) {
+                    //log.error(e, "Trying to shut down");
+                }
+            }
+        });
+    }
+
+    /**
+     * Stop the life cycle - all instances will have their {@link javax.annotation.PreDestroy} method(s) called
+     *
+     * @throws Exception errors
+     */
+    public void destroy()
+            throws Exception
+    {
+        if (!state.compareAndSet(State.STARTED, State.STOPPING)) {
+            return;
+        }
+
+        //log.info("Life cycle stopping...");
+
+        List<Object> reversedInstances = Lists.newArrayList(managedInstances);
+        Collections.reverse(reversedInstances);
+
+        for (Object obj : reversedInstances) {
+            //log.debug("Stopping %s", obj.getClass().getName());
+            LifeCycleMethods methods = methodsMap.get(obj.getClass());
+            for (Method preDestroy : methods.methodsFor(PreDestroy.class)) {
+                //log.debug("\t%s()", preDestroy.getName());
+                preDestroy.invoke(obj);
+            }
+        }
+
+        state.set(State.STOPPED);
+        //log.info("Life cycle stopped.");
+    }
+
+    /**
+     * Return true if state is {@link destroy} is called
+     */
+    public boolean isDestroyed()
+    {
+        State currentState = state.get();
+        return currentState == State.STOPPING || currentState == State.STOPPED;
+    }
+
+    /**
+     * Add an additional managed instance
+     *
+     * @param instance instance to add
+     * @throws Exception errors
+     */
+    public void addInstance(Object instance)
+            throws Exception
+    {
+        if (isDestroyed()) {
+            throw new IllegalStateException("System already stopped");
+        }
+        else {
+            startInstance(instance);
+            if (methodsMap.get(instance.getClass()).hasFor(PreDestroy.class)) {
+                managedInstances.add(instance);
+            }
+        }
+    }
+
+    private void startInstance(Object obj)
+            throws IllegalAccessException, InvocationTargetException
+    {
+        //log.debug("Starting %s", obj.getClass().getName());
+        LifeCycleMethods methods = methodsMap.get(obj.getClass());
+        for (Method postConstruct : methods.methodsFor(PostConstruct.class)) {
+            //log.debug("\t%s()", postConstruct.getName());
+            postConstruct.invoke(obj);
+        }
+    }
+}