embulk-executor-mapreduce 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d683908af5b1f5e06ea035f9ecd4e7a89dbd4eab
-  data.tar.gz: f8a5a7db822f6241a40211dd1782e4fda83df073
+  metadata.gz: 3c12d59b65314dd94cf12e29b7caf8e83fd947bd
+  data.tar.gz: 478ea3a4e77c7a9e5395a4c4942ac9697772a10d
 SHA512:
-  metadata.gz: 48b955f8b5a70864dd745347079cc83545fdb6953091812917beeb28e6aa9f66726c68e8b85b0aa3c8da40f419fc0b1efa78aab90bd899e42bb4c1a0c14edaae
-  data.tar.gz: 99199b98f4f56e2d5d74c7f12be41b19bd31aa3f496144a940b16c6c53abf3980c181ae186f40ba3bf307106abb53dc00683c54157ea000652d24d66cff12cd5
+  metadata.gz: e8c955d1e7a4e0b318bc21a36c51410045b4b4c9e62b954e8848f66693d26fe580a79afb8c8e1c1910f9ecd1731f78772b6b9af7f64e817b9666776da26b126c
+  data.tar.gz: 14fb4a34dbaf1b59b8d37ceac1356d4ab66014a559943f16da087d9f17800e46c80e9b98ae1583be5568ee8909034ba708bfa4ca3dab27ddde90b5c8e2ab8f59

data/src/main/java/org/embulk/executor/mapreduce/MapReduceExecutor.java

@@ -1,5 +1,6 @@
 package org.embulk.executor.mapreduce;
 
+import java.security.PrivilegedAction;
 import java.util.List;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -16,6 +17,8 @@ import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.net.MalformedURLException;
+
+import org.apache.hadoop.security.UserGroupInformation;
 import org.slf4j.Logger;
 import org.joda.time.format.DateTimeFormat;
 import com.google.inject.Inject;
@@ -183,12 +186,12 @@ public class MapReduceExecutor
         }
     }
 
-    void run(MapReduceExecutorTask task,
-            int mapTaskCount, int reduceTaskCount, ProcessState state)
+    void run(final MapReduceExecutorTask task,
+            int mapTaskCount, final int reduceTaskCount, final ProcessState state)
     {
-        ModelManager modelManager = task.getModelManager();
+        final ModelManager modelManager = task.getModelManager();
 
-        Configuration conf = new Configuration();
+        final Configuration conf = new Configuration();
         // don't call conf.setQuietMode(false). Configuraiton has invalid resource names by default
         for (String path : task.getConfigFiles()) {
             File file = new File(path);
@@ -204,31 +207,56 @@ public class MapReduceExecutor
         }
 
         String uniqueTransactionName = getTransactionUniqueName(Exec.session());
-        Path stateDir = new Path(new Path(task.getStatePath()), uniqueTransactionName);
-
-        Job job;
-        try {
-            job = Job.getInstance(conf);
-        } catch (IOException e) {
-            throw Throwables.propagate(e);
-        }
-        job.setJobName(task.getJobName());
+        final Path stateDir = new Path(new Path(task.getStatePath()), uniqueTransactionName);
 
         // create a dedicated classloader for this yarn application.
         // allow task.getConfig to overwrite this parameter
-        job.getConfiguration().set(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, "true");  // mapreduce.job.classloader
-        job.getConfiguration().set(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES, "java.,org.apache.hadoop.");  // mapreduce.job.classloader.system.classes
+        conf.set(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, "true");  // mapreduce.job.classloader
+        conf.set(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES, "java.,org.apache.hadoop.");  // mapreduce.job.classloader.system.classes
 
         // extra config
         for (Map.Entry<String, String> pair : task.getConfig().entrySet()) {
-            job.getConfiguration().set(pair.getKey(), pair.getValue());
+            conf.set(pair.getKey(), pair.getValue());
         }
 
         // framework config
-        EmbulkMapReduce.setSystemConfig(job.getConfiguration(), modelManager, systemConfig);
-        EmbulkMapReduce.setExecutorTask(job.getConfiguration(), modelManager, task);
-        EmbulkMapReduce.setMapTaskCount(job.getConfiguration(), mapTaskCount);  // used by EmbulkInputFormat
-        EmbulkMapReduce.setStateDirectoryPath(job.getConfiguration(), stateDir);
+        EmbulkMapReduce.setSystemConfig(conf, modelManager, systemConfig);
+        EmbulkMapReduce.setExecutorTask(conf, modelManager, task);
+        EmbulkMapReduce.setMapTaskCount(conf, mapTaskCount);  // used by EmbulkInputFormat
+        EmbulkMapReduce.setStateDirectoryPath(conf, stateDir);
+
+        // jar files
+        List<Path> jars = collectJars(task.getLibjars(), task.getExcludeJars());
+        conf.set("tmpjars", StringUtils.join(",", jars));
+
+        String remoteUser = conf.get(MRJobConfig.USER_NAME);  // mapreduce.job.user.name
+        if (remoteUser != null) {
+            UserGroupInformation.createRemoteUser(remoteUser).doAs(
+                    new PrivilegedAction<Void>()
+                    {
+                        @Override
+                        public Void run()
+                        {
+                            runJob(task, modelManager, reduceTaskCount, state, stateDir, conf);
+                            return null;
+                        }
+                    }
+            );
+        } else {
+            runJob(task, modelManager, reduceTaskCount, state, stateDir, conf);
+        }
+    }
+
+    void runJob(MapReduceExecutorTask task, ModelManager modelManager,
+            int reduceTaskCount, ProcessState state, Path stateDir, Configuration conf)
+    {
+        Job job;
+        try {
+            job = Job.getInstance(conf);
+        } catch (IOException e) {
+            throw Throwables.propagate(e);
+        }
+        job.setJobName(task.getJobName());
 
         // archive plugins (also create state dir)
         PluginArchive archive = new PluginArchive.Builder()
@@ -240,10 +268,6 @@ public class MapReduceExecutor
             throw new RuntimeException(ex);
         }
 
-        // jar files
-        List<Path> jars = collectJars(task.getLibjars(), task.getExcludeJars());
-        job.getConfiguration().set("tmpjars", StringUtils.join(",", jars));
-
         job.setInputFormatClass(EmbulkInputFormat.class);
 
         if (reduceTaskCount > 0) {

data/src/main/java/org/embulk/executor/mapreduce/TimestampPartitioning.java

@@ -1,5 +1,6 @@
 package org.embulk.executor.mapreduce;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.joda.time.DateTimeZone;
 import com.google.common.base.Optional;
 import org.embulk.config.Config;
@@ -40,7 +41,8 @@ public class TimestampPartitioning
         public void setTargetColumn(Column column);
     }
 
-    private static enum Unit
+    @VisibleForTesting
+    static enum Unit
     {
         HOUR(60*60),
         DAY(24*60*60);
@@ -70,12 +72,13 @@ public class TimestampPartitioning
             //case "year": return YEAR;
             default:
                 throw new ConfigException(
-                        String.format("Unknown unit '%s'. Supported units are hour and day"));
+                        String.format("Unknown unit '%s'. Supported units are hour and day", s));
             }
         }
     }
 
-    private static enum UnixTimestampUnit
+    @VisibleForTesting
+    static enum UnixTimestampUnit
     {
         SEC(1),
         MILLI(1000),
@@ -103,7 +106,7 @@ public class TimestampPartitioning
             case "nano": return NANO;
             default:
                 throw new ConfigException(
-                        String.format("Unknown unix_timestamp_unit '%s'. Supported units are sec, milli, micro, and nano"));
+                        String.format("Unknown unix_timestamp_unit '%s'. Supported units are sec, milli, micro, and nano", s));
             }
         }
     }
@@ -253,7 +256,8 @@ public class TimestampPartitioning
         }
     }
 
-    private static class TimestampPartitioner
+    @VisibleForTesting
+    static class TimestampPartitioner
             extends AbstractTimestampPartitioner
     {
         public TimestampPartitioner(Column column, Unit unit)
@@ -269,7 +273,8 @@ public class TimestampPartitioning
         }
     }
 
-    private static class LongUnixTimestampPartitioner
+    @VisibleForTesting
+    static class LongUnixTimestampPartitioner
             extends AbstractTimestampPartitioner
     {
         private final UnixTimestampUnit unixTimestampUnit;

data/src/test/java/org/embulk/executor/mapreduce/MapReduceExecutorTestRuntime.java

@@ -0,0 +1,130 @@
+package org.embulk.executor.mapreduce;
+
+import java.util.Random;
+
+import com.google.inject.util.Modules;
+import org.embulk.GuiceBinder;
+import org.embulk.RandomManager;
+import org.embulk.TestPluginSourceModule;
+import org.embulk.TestUtilityModule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+import com.google.inject.Injector;
+import com.google.inject.Binder;
+import com.google.inject.Module;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.DataSourceImpl;
+import org.embulk.config.ModelManager;
+import org.embulk.exec.SystemConfigModule;
+import org.embulk.exec.ExecModule;
+import org.embulk.exec.ExtensionServiceLoaderModule;
+import org.embulk.plugin.BuiltinPluginSourceModule;
+import org.embulk.jruby.JRubyScriptingModule;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.Exec;
+import org.embulk.spi.ExecAction;
+import org.embulk.spi.ExecSession;
+
+// TODO This class should be merged into EmbulkTestRuntime class. Because EmbulkTestRuntime doesn't have module overriding feature.
+public class MapReduceExecutorTestRuntime
+        extends GuiceBinder
+{
+    private static ConfigSource getSystemConfig()
+    {
+        // TODO set some default values
+        return new DataSourceImpl(null);
+    }
+
+    public static class TestRuntimeModule
+            implements Module
+    {
+        @Override
+        public void configure(Binder binder)
+        {
+            ConfigSource systemConfig = getSystemConfig();
+
+            new SystemConfigModule(systemConfig).configure(binder);
+            new ExecModule().configure(binder);
+            new ExtensionServiceLoaderModule(systemConfig).configure(binder);
+            new BuiltinPluginSourceModule().configure(binder);
+            new JRubyScriptingModule(systemConfig).configure(binder);
+            new TestUtilityModule().configure(binder);
+            new TestPluginSourceModule().configure(binder);
+
+        }
+    }
+
+    private ExecSession exec;
+
+    public MapReduceExecutorTestRuntime()
+    {
+        super(Modules.override(new TestRuntimeModule()).with(new Module() {
+            @Override
+            public void configure(Binder binder)
+            {
+                new TestMapReduceExecutor.ExecutorPluginApplyModule().configure(binder);
+                new TestMapReduceExecutor.LoggerOverrideModule().configure(binder);
+            }
+        }));
+        Injector injector = getInjector();
+        ConfigSource execConfig = new DataSourceImpl(injector.getInstance(ModelManager.class));
+        this.exec = ExecSession.builder(injector).fromExecConfig(execConfig).build();
+    }
+
+    public ExecSession getExec()
+    {
+        return exec;
+    }
+
+    public BufferAllocator getBufferAllocator()
+    {
+        return getInstance(BufferAllocator.class);
+    }
+
+    public ModelManager getModelManager()
+    {
+        return getInstance(ModelManager.class);
+    }
+
+    public Random getRandom()
+    {
+        return getInstance(RandomManager.class).getRandom();
+    }
+
+    @Override
+    public Statement apply(Statement base, Description description)
+    {
+        final Statement superStatement = MapReduceExecutorTestRuntime.super.apply(base, description);
+        return new Statement() {
+            public void evaluate() throws Throwable
+            {
+                try {
+                    Exec.doWith(exec, new ExecAction<Void>() {
+                        public Void run()
+                        {
+                            try {
+                                superStatement.evaluate();
+                            } catch (Throwable ex) {
+                                throw new RuntimeExecutionException(ex);
+                            }
+                            return null;
+                        }
+                    });
+                } catch (RuntimeException ex) {
+                    throw ex.getCause();
+                } finally {
+                    exec.cleanup();
+                }
+            }
+        };
+    }
+
+    private static class RuntimeExecutionException
+            extends RuntimeException
+    {
+        public RuntimeExecutionException(Throwable cause)
+        {
+            super(cause);
+        }
+    }
+}

data/src/test/java/org/embulk/executor/mapreduce/TestAttemptState.java

@@ -0,0 +1,58 @@
+package org.embulk.executor.mapreduce;
+
+import com.google.common.base.Optional;
+
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestAttemptState
+{
+    @Rule
+    public MapReduceExecutorTestRuntime runtime = new MapReduceExecutorTestRuntime();
+
+    @Test
+    public void readAndWrite()
+            throws IOException {
+        TaskAttemptID attemptId = TaskAttemptID.forName("attempt_200707121733_0003_m_000005_0");
+        int inputTaskIndex = 1;
+        int outputTaskIndex = 2;
+        Exception ex = new Exception();
+
+        AttemptState attemptState = new AttemptState(attemptId, Optional.of(inputTaskIndex), Optional.of(outputTaskIndex));
+        attemptState.setException(ex);
+        attemptState.setInputTaskReport(runtime.getExec().newTaskReport());
+        attemptState.setOutputTaskReport(runtime.getExec().newTaskReport());
+
+        try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+            attemptState.writeTo(out, runtime.getModelManager());
+
+            try (ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray())) {
+                assertAttemptStateEquals(attemptState, AttemptState.readFrom(in, runtime.getModelManager()));
+            }
+        }
+    }
+
+    private static void assertAttemptStateEquals(AttemptState s1, AttemptState s2)
+    {
+        assertEquals(s1.getAttemptId(), s2.getAttemptId());
+        assertEquals(s1.getInputTaskIndex(), s2.getInputTaskIndex());
+        assertEquals(s1.getOutputTaskIndex(), s2.getOutputTaskIndex());
+        assertEquals(s1.getException(), s2.getException());
+        assertEquals(s1.getInputTaskReport(), s2.getInputTaskReport());
+        assertEquals(s1.getOutputTaskReport(), s2.getOutputTaskReport());
+    }
+
+    @Test
+    public void throwEOFIfInvalidJsonString()
+            throws IOException {
+        String json = "{\"key\":\"va";
+        // TODO
+    }
+}

data/src/test/java/org/embulk/executor/mapreduce/TestEmbulkInputFormat.java

@@ -0,0 +1,54 @@
+package org.embulk.executor.mapreduce;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestEmbulkInputFormat
+{
+    @Rule
+    public MapReduceExecutorTestRuntime runtime = new MapReduceExecutorTestRuntime();
+
+    private Configuration conf;
+    private EmbulkInputFormat format;
+
+    @Before
+    public void createResources()
+    {
+        conf = new Configuration();
+        format = new EmbulkInputFormat();
+    }
+
+    @Test
+    public void getSplits()
+            throws Exception
+    {
+        checkNumOfSplits(0);
+
+        for (int i = 0; i < 10; i++) {
+
+            int split = runtime.getRandom().nextInt(10000);
+            checkNumOfSplits(split);
+        }
+    }
+
+    private void checkNumOfSplits(int split)
+            throws Exception
+    {
+        conf.set("embulk.mapreduce.taskCount", Integer.toString(split));
+        JobContext jobContext = newJobContext(conf);
+        assertEquals(split, format.getSplits(jobContext).size());
+    }
+
+    private JobContext newJobContext(Configuration conf)
+    {
+        JobID jobID = new JobID("test", runtime.getRandom().nextInt());
+        return new JobContextImpl(conf, jobID);
+    }
+}

data/src/test/java/org/embulk/executor/mapreduce/TestEmbulkInputSplit.java

@@ -0,0 +1,46 @@
+package org.embulk.executor.mapreduce;
+
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class TestEmbulkInputSplit
+{
+    @Test
+    public void readAndWrite()
+            throws IOException
+    {
+        readAndWrite(new EmbulkInputSplit());
+        readAndWrite(new EmbulkInputSplit(new int[] {0, 1, 2, 3}));
+    }
+
+    private void readAndWrite(EmbulkInputSplit is) throws IOException
+    {
+        try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+            try (DataOutputStream dout = new DataOutputStream(out)) {
+                is.write(dout);
+                dout.flush();
+
+                try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(out.toByteArray()))) {
+                    EmbulkInputSplit newIs = new EmbulkInputSplit();
+                    newIs.readFields(in);
+                    assertEmbulkInputSplitEquals(is, newIs);
+                }
+            }
+        }
+    }
+
+    private static void assertEmbulkInputSplitEquals(EmbulkInputSplit is1, EmbulkInputSplit is2)
+    {
+        assertArrayEquals(is1.getTaskIndexes(), is2.getTaskIndexes());
+        assertEquals(is1.getLength(), is2.getLength());
+        assertArrayEquals(is1.getLocations(), is2.getLocations());
+    }
+}