embulk 0.7.0-java

data/embulk-core/src/main/resources/embulk/logback-color.xml

@@ -0,0 +1,72 @@
+<configuration>
+  <appender name="console-error" class="ch.qos.logback.core.ConsoleAppender">
+    <filter class="ch.qos.logback.classic.filter.LevelFilter">
+      <level>ERROR</level>
+      <onMatch>ACCEPT</onMatch>
+      <onMismatch>DENY</onMismatch>
+    </filter>
+    <withJansi>true</withJansi>
+    <encoder>
+      <pattern>%magenta(%d{yyyy-MM-dd HH:mm:ss.SSS Z} [%level] (%thread\): %m%n)</pattern>
+    </encoder>
+  </appender>
+
+  <appender name="console-warn" class="ch.qos.logback.core.ConsoleAppender">
+    <filter class="ch.qos.logback.classic.filter.LevelFilter">
+      <level>WARN</level>
+      <onMatch>ACCEPT</onMatch>
+      <onMismatch>DENY</onMismatch>
+    </filter>
+    <withJansi>true</withJansi>
+    <encoder>
+      <pattern>%yellow(%d{yyyy-MM-dd HH:mm:ss.SSS Z} [%level] (%thread\): %m%n)</pattern>
+    </encoder>
+  </appender>
+
+  <appender name="console-info" class="ch.qos.logback.core.ConsoleAppender">
+    <filter class="ch.qos.logback.classic.filter.LevelFilter">
+      <level>INFO</level>
+      <onMatch>ACCEPT</onMatch>
+      <onMismatch>DENY</onMismatch>
+    </filter>
+    <withJansi>true</withJansi>
+    <encoder>
+      <pattern>%green(%d{yyyy-MM-dd HH:mm:ss.SSS Z} [%level] (%thread\): %m%n)</pattern>
+    </encoder>
+  </appender>
+
+  <appender name="console-debug" class="ch.qos.logback.core.ConsoleAppender">
+    <filter class="ch.qos.logback.classic.filter.LevelFilter">
+      <level>DEBUG</level>
+      <onMatch>ACCEPT</onMatch>
+      <onMismatch>DENY</onMismatch>
+    </filter>
+    <withJansi>true</withJansi>
+    <encoder>
+      <pattern>%white(%d{yyyy-MM-dd HH:mm:ss.SSS Z} [%level] (%thread\): %m%n)</pattern>
+    </encoder>
+  </appender>
+
+  <appender name="console-trace" class="ch.qos.logback.core.ConsoleAppender">
+    <filter class="ch.qos.logback.classic.filter.LevelFilter">
+      <level>TRACE</level>
+      <onMatch>ACCEPT</onMatch>
+      <onMismatch>DENY</onMismatch>
+    </filter>
+    <withJansi>true</withJansi>
+    <encoder>
+      <pattern>%blue(%d{yyyy-MM-dd HH:mm:ss.SSS Z} [%level] (%thread\): %m%n)</pattern>
+    </encoder>
+  </appender>
+
+  <logger name="io.netty.util" level="INFO"/>
+  <logger name="io.netty.buffer" level="INFO"/>
+
+  <root>
+    <appender-ref ref="console-error"/>
+    <appender-ref ref="console-warn"/>
+    <appender-ref ref="console-info"/>
+    <appender-ref ref="console-debug"/>
+    <appender-ref ref="console-trace"/>
+  </root>
+</configuration>

data/embulk-core/src/main/resources/embulk/logback-console.xml

@@ -0,0 +1,14 @@
+<configuration>
+  <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder>
+      <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS Z} [%level] (%thread\): %m%n</pattern>
+    </encoder>
+  </appender>
+
+  <logger name="io.netty.util" level="INFO"/>
+  <logger name="io.netty.buffer" level="INFO"/>
+
+  <root>
+    <appender-ref ref="console"/>
+  </root>
+</configuration>

data/embulk-core/src/main/resources/embulk/logback-file.xml

@@ -0,0 +1,25 @@
+<configuration>
+  <property name="embulk.logFile" value="embulk.log" />
+
+  <appender name="file" class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>${embulk.logFile}</file>
+    <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+      <maxFileSize>5kB</maxFileSize>
+    </triggeringPolicy>
+    <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+      <fileNamePattern>${embulk.logFile}.%i</fileNamePattern>
+      <minIndex>1</minIndex>
+      <maxIndex>5</maxIndex>
+    </rollingPolicy>
+    <encoder>
+      <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS Z} [%level] (%thread\): %m%n</pattern>
+    </encoder>
+  </appender>
+
+  <logger name="io.netty.util" level="INFO"/>
+  <logger name="io.netty.buffer" level="INFO"/>
+
+  <root>
+    <appender-ref ref="file"/>
+  </root>
+</configuration>

data/embulk-core/src/main/resources/embulk/parent_first_packages.properties

@@ -0,0 +1,70 @@
+# generated by './gradlew updateResources' at 2015-06-30 23:52:09 UTC
+ch.qos.logback.classic
+ch.qos.logback.core
+com.fasterxml.jackson.annotation
+com.fasterxml.jackson.core
+com.fasterxml.jackson.databind
+com.fasterxml.jackson.datatype.guava
+com.fasterxml.jackson.datatype.joda
+com.fasterxml.jackson.module.guice
+com.google.common.annotations
+com.google.common.base
+com.google.common.cache
+com.google.common.collect
+com.google.common.escape
+com.google.common.eventbus
+com.google.common.hash
+com.google.common.html
+com.google.common.io
+com.google.common.math
+com.google.common.net
+com.google.common.primitives
+com.google.common.reflect
+com.google.common.util.concurrent
+com.google.common.xml
+com.google.inject
+com.google.thirdparty.publicsuffix
+com.headius.invokebinder
+com.headius.options
+com.ibm.icu.impl
+com.ibm.icu.lang
+com.ibm.icu.math
+com.ibm.icu.text
+com.ibm.icu.util
+com.jcraft.jzlib
+com.kenai.constantine
+com.kenai.jffi
+com.kenai.jnr.x86asm
+com.martiansoftware.nailgun
+edu.umd.cs.findbugs.annotations
+io.airlift.slice
+io.netty.buffer
+io.netty.util
+javax.annotation
+javax.inject
+javax.validation
+jnr.constants
+jnr.enxio.channels
+jnr.enxio.example
+jnr.ffi
+jnr.netdb
+jnr.posix
+jnr.unixsocket
+jnr.x86asm
+net.jcip.annotations
+org.aopalliance.aop
+org.aopalliance.intercept
+org.apache.bval
+org.apache.commons.beanutils
+org.apache.commons.collections
+org.apache.commons.lang3
+org.embulk
+org.fusesource.hawtjni.runtime
+org.fusesource.jansi
+org.jcodings
+org.joda.time
+org.joni
+org.jruby
+org.slf4j
+org.yaml.snakeyaml
+org.yecht

data/embulk-core/src/main/resources/embulk/parent_first_resources.properties

@@ -0,0 +1,28 @@
+# generated by './gradlew updateResources' at 2015-06-30 23:52:09 UTC
+ch/qos/logback/classic/boolex
+ch/qos/logback/classic/db/script
+com/ibm/icu
+com/martiansoftware/nailgun/builtins
+edu/umd/cs/findbugs/annotations
+embulk
+javax/annotation
+jni/Darwin
+jni/arm-Linux
+jni/i386-Linux
+jni/i386-SunOS
+jni/i386-Windows
+jni/ppc-AIX
+jni/sparcv9-SunOS
+jni/x86_64-FreeBSD
+jni/x86_64-Linux
+jni/x86_64-SunOS
+jni/x86_64-Windows
+jruby
+net/jcip/annotations
+okay
+org/apache/bval/jsr303
+org/joda/time/format
+org/joda/time/tz/data
+tables
+templates
+yaml

data/embulk-core/src/test/java/org/embulk/EmbulkTestRuntime.java

@@ -0,0 +1,114 @@
+package org.embulk;
+
+import java.util.Random;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+import com.google.inject.Injector;
+import com.google.inject.Binder;
+import com.google.inject.Module;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.DataSourceImpl;
+import org.embulk.config.ModelManager;
+import org.embulk.exec.SystemConfigModule;
+import org.embulk.exec.ExecModule;
+import org.embulk.exec.ExtensionServiceLoaderModule;
+import org.embulk.plugin.BuiltinPluginSourceModule;
+import org.embulk.jruby.JRubyScriptingModule;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.Exec;
+import org.embulk.spi.ExecAction;
+import org.embulk.spi.ExecSession;
+
+public class EmbulkTestRuntime
+        extends GuiceBinder
+{
+    private static ConfigSource getSystemConfig()
+    {
+        // TODO set some default values
+        return new DataSourceImpl(null);
+    }
+
+    public static class TestRuntimeModule
+            implements Module
+    {
+        @Override
+        public void configure(Binder binder)
+        {
+            ConfigSource systemConfig = getSystemConfig();
+            new SystemConfigModule(systemConfig).configure(binder);
+            new ExecModule().configure(binder);
+            new ExtensionServiceLoaderModule(systemConfig).configure(binder);
+            new BuiltinPluginSourceModule().configure(binder);
+            new JRubyScriptingModule(systemConfig).configure(binder);
+            new TestUtilityModule().configure(binder);
+            new TestPluginSourceModule().configure(binder);
+        }
+    }
+
+    private ExecSession exec;
+
+    public EmbulkTestRuntime()
+    {
+        super(new TestRuntimeModule());
+        Injector injector = getInjector();
+        ConfigSource execConfig = new DataSourceImpl(injector.getInstance(ModelManager.class));
+        this.exec = ExecSession.builder(injector).fromExecConfig(execConfig).build();
+    }
+
+    public ExecSession getExec()
+    {
+        return exec;
+    }
+
+    public BufferAllocator getBufferAllocator()
+    {
+        return getInstance(BufferAllocator.class);
+    }
+
+    public ModelManager getModelManager()
+    {
+        return getInstance(ModelManager.class);
+    }
+
+    public Random getRandom()
+    {
+        return getInstance(RandomManager.class).getRandom();
+    }
+
+    @Override
+    public Statement apply(Statement base, Description description)
+    {
+        final Statement superStatement = EmbulkTestRuntime.super.apply(base, description);
+        return new Statement() {
+            public void evaluate() throws Throwable
+            {
+                try {
+                    Exec.doWith(exec, new ExecAction<Void>() {
+                        public Void run()
+                        {
+                            try {
+                                superStatement.evaluate();
+                            } catch (Throwable ex) {
+                                throw new RuntimeExecutionException(ex);
+                            }
+                            return null;
+                        }
+                    });
+                } catch (RuntimeException ex) {
+                    throw ex.getCause();
+                } finally {
+                    exec.cleanup();
+                }
+            }
+        };
+    }
+
+    private static class RuntimeExecutionException
+            extends RuntimeException
+    {
+        public RuntimeExecutionException(Throwable cause)
+        {
+            super(cause);
+        }
+    }
+}

data/embulk-core/src/test/java/org/embulk/GuiceBinder.java

@@ -0,0 +1,72 @@
+package org.embulk;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.rules.TestRule;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+
+public class GuiceBinder
+        implements TestRule
+{
+    private final List<Module> baseModules;
+    private List<Module> extraModules;
+    private Injector injector;
+
+    public GuiceBinder(Module... baseModules)
+    {
+        this.baseModules = ImmutableList.copyOf(baseModules);
+        reset();
+    }
+
+    private void reset()
+    {
+        extraModules = new ArrayList<Module>();
+        injector = null;
+    }
+
+    public synchronized void addModule(Module module)
+    {
+        if (injector != null) {
+            throw new IllegalStateException("Injector is already initialized. Call addModule before getInjector or getInstance");
+        }
+        extraModules.add(module);
+    }
+
+    public synchronized Injector getInjector()
+    {
+        if (injector == null) {
+            ImmutableList.Builder<Module> modules = ImmutableList.builder();
+            modules.addAll(baseModules);
+            modules.addAll(extraModules);
+            injector = Guice.createInjector(modules.build());
+        }
+        return injector;
+    }
+
+    public <T> T getInstance(Class<T> klass)
+    {
+        return getInjector().getInstance(klass);
+    }
+
+    @Override
+    public Statement apply(Statement base, Description description)
+    {
+        return new GuceBinderWatcher().apply(base, description);
+    }
+
+    private class GuceBinderWatcher
+            extends TestWatcher
+    {
+        @Override
+        protected void starting(Description description)
+        {
+            reset();
+        }
+    }
+}

data/embulk-core/src/test/java/org/embulk/RandomManager.java

@@ -0,0 +1,53 @@
+package org.embulk;
+
+import java.util.Map;
+import java.util.Random;
+
+public class RandomManager
+{
+    protected long seed;
+    protected Random random;
+
+    public RandomManager()
+    {
+        this(getDefaultSeed());
+    }
+
+    public RandomManager(long seed)
+    {
+        this.seed = seed;
+        this.random = new Random(seed);
+        System.out.println(" Random seed: 0x"+Long.toHexString(seed)+"L");
+    }
+
+    public long getRandomSeed()
+    {
+        return seed;
+    }
+
+    public void setRandomSeed(long seed)
+    {
+        random.setSeed(seed);
+        this.seed = seed;
+        System.out.println(" Set random seed: 0x"+Long.toHexString(this.seed)+"L");
+    }
+
+    public Random getRandom()
+    {
+        return random;
+    }
+
+    private static long getDefaultSeed() {
+        Map<String, String> env = System.getenv();
+        String s = env.get("RANDOM_SEED");
+        try {
+            if(s != null) {
+                return Long.parseLong(s);
+            }
+        } catch (NumberFormatException e) {
+            System.out.println("RANDOM_SEED variable is wrong: "+e);
+        }
+
+        return new Random().nextLong();
+    }
+}