embulk-executor-mapreduce 0.2.2 → 0.2.3

data/src/test/java/org/embulk/executor/mapreduce/TestEmbulkRecordReader.java

@@ -0,0 +1,25 @@
+package org.embulk.executor.mapreduce;
+
+import org.apache.hadoop.io.NullWritable;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestEmbulkRecordReader
+{
+    @Test
+    public void simpleTest()
+    {
+        int[] taskIndexes = new int[] {0, 1, 4, 6, 7};
+        try (EmbulkRecordReader r = new EmbulkRecordReader(new EmbulkInputSplit(taskIndexes))) {
+            int i = 0;
+            while (r.nextKeyValue()) {
+                assertEquals(taskIndexes[i], r.getCurrentKey().get());
+                assertTrue(r.getCurrentValue() instanceof NullWritable);
+                i++;
+            }
+            assertEquals(taskIndexes.length, i);
+        }
+    }
+}

data/src/test/java/org/embulk/executor/mapreduce/TestMapReduceExecutor.java

@@ -0,0 +1,251 @@
+package org.embulk.executor.mapreduce;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.inject.Binder;
+import com.google.inject.Module;
+import com.google.inject.Provider;
+import com.google.inject.util.Modules;
+import org.embulk.EmbulkEmbed;
+import org.embulk.RandomManager;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.UserDataExceptions;
+import org.embulk.exec.PartialExecutionException;
+import org.embulk.spi.ExecutorPlugin;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.ILoggerFactory;
+import org.slf4j.impl.Log4jLoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.List;
+import java.util.Random;
+
+import static org.embulk.plugin.InjectedPluginSource.registerPluginTo;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+// this tests use Hadoop's standalone mode
+public class TestMapReduceExecutor
+{
+    private EmbulkEmbed embulk;
+    private Random random = new RandomManager(System.currentTimeMillis()).getRandom();
+
+    @Before
+    public void createResources()
+    {
+        EmbulkEmbed.Bootstrap bootstrap = new EmbulkEmbed.Bootstrap();
+
+        ConfigSource systemConfig = bootstrap.getSystemConfigLoader().newConfigSource();
+
+        if (random.nextBoolean()) {
+            systemConfig.set("embulk_factory_class", MapReduceEmbulkFactory.class.getName());
+        } else {
+            systemConfig.set("embulk_factory_class", MapReduceEmbulkFactory2.class.getName());
+        }
+
+        bootstrap.setSystemConfig(systemConfig);
+        bootstrap.overrideModules(getModuleOverrides(systemConfig));
+        embulk = bootstrap.initialize();
+    }
+
+    @Test
+    public void testEmbulkMapper()
+            throws Exception
+    {
+        ConfigSource config = loadConfigSource(embulk.newConfigLoader(), "config/embulk_mapred_config.yml");
+        embulk.run(config);
+        // TODO compare input and output
+    }
+
+    @Test
+    public void testEmbulkPartitioningMapperReducer()
+            throws Exception
+    {
+        ConfigSource config = loadConfigSource(embulk.newConfigLoader(), "config/embulk_mapred_partitioning_config.yml");
+        embulk.run(config);
+        // TODO compare input and output
+    }
+
+    @Test
+    public void testInvalidConfigFiles()
+            throws Exception
+    {
+        try {
+            ConfigSource config = loadConfigSource(embulk.newConfigLoader(), "config/embulk_mapred_invalid_config_files_config.yml");
+            embulk.run(config);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof ConfigException);
+        }
+    }
+
+    @Test
+    public void testInvalidPartitioning()
+            throws Exception
+    {
+        try {
+            ConfigSource config = loadConfigSource(embulk.newConfigLoader(), "config/embulk_mapred_invalid_partitioning_config.yml");
+            embulk.run(config);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof ConfigException);
+        }
+    }
+
+    @Test
+    public void testInvalidReducers()
+            throws Exception
+    {
+        try {
+            ConfigSource config = loadConfigSource(embulk.newConfigLoader(), "config/embulk_mapred_invalid_reducers_config.yml");
+            embulk.run(config);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof ConfigException);
+        }
+    }
+
+    @Test
+    public void testInvalidLibjars()
+            throws Exception
+    {
+        try {
+            ConfigSource config = loadConfigSource(embulk.newConfigLoader(), "config/embulk_mapred_invalid_libjars_config.yml");
+            embulk.run(config);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t.getCause() instanceof FileNotFoundException);
+        }
+    }
+
+    @Test
+    public void testStopOnInvalidRecord()
+            throws Exception
+    {
+        try {
+            ConfigSource config = loadConfigSource(embulk.newConfigLoader(), "config/embulk_mapred_stop_on_invalid_record_config.yml");
+            embulk.run(config);
+            fail();
+        }
+        catch (Throwable t) {
+            t.printStackTrace();
+            assertTrue(t instanceof PartialExecutionException);
+            assertTrue(UserDataExceptions.isUserDataException(t.getCause()));
+        }
+    }
+
+    private static ConfigSource loadConfigSource(ConfigLoader configLoader, String yamlFile)
+            throws IOException
+    {
+        return configLoader.fromYaml(TestMapReduceExecutor.class.getClassLoader().getResourceAsStream(yamlFile));
+    }
+
+    private static Function<List<Module>, List<Module>> getModuleOverrides(final ConfigSource systemConfig)
+    {
+        return new Function<List<Module>, List<Module>>()
+        {
+            public List<Module> apply(List<Module> modules)
+            {
+                return overrideModules(modules, systemConfig);
+            }
+        };
+    }
+
+    private static List<Module> overrideModules(List<Module> modules, ConfigSource systemConfig)
+    {
+        return ImmutableList.of(Modules.override(Iterables.concat(modules, getAdditionalModules(systemConfig)))
+                .with(getOverrideModules(systemConfig)));
+    }
+
+    private static List<Module> getAdditionalModules(ConfigSource systemConfig)
+    {
+        return ImmutableList.<Module>of(new ExecutorPluginApplyModule());
+    }
+
+    private static List<Module> getOverrideModules(ConfigSource systemConfig)
+    {
+        return ImmutableList.<Module>of(new LoggerOverrideModule());
+    }
+
+    static class ExecutorPluginApplyModule
+            implements Module
+    {
+        @Override
+        public void configure(Binder binder)
+        {
+            registerPluginTo(binder, ExecutorPlugin.class, "mapreduce", MapReduceExecutor.class);
+        }
+    }
+
+    static class LoggerOverrideModule
+            implements Module
+    {
+        @Override
+        public void configure(Binder binder)
+        {
+            binder.bind(ILoggerFactory.class).toProvider(new Provider<ILoggerFactory>()
+            {
+                @Override
+                public ILoggerFactory get()
+                {
+                    return new Log4jLoggerFactory(); // YARN has used log4j.
+                }
+            });
+        }
+    }
+
+    public static class MapReduceEmbulkFactory
+    {
+        public EmbulkEmbed.Bootstrap bootstrap(final ConfigSource systemConfig)
+        {
+            EmbulkEmbed.Bootstrap bootstrap = new EmbulkEmbed.Bootstrap();
+            bootstrap.setSystemConfig(systemConfig);
+
+            // add modules
+            //bootstrap.addModules(ImmutableList.<Module>of());
+
+            // override modules
+            bootstrap.overrideModules(new Function<List<Module>, List<Module>>()
+            {
+                public List<Module> apply(List<Module> modules)
+                {
+                    return ImmutableList.of(Modules.override(modules).with(new LoggerOverrideModule()));
+                }
+            });
+
+            return bootstrap;
+        }
+    }
+
+    public static class MapReduceEmbulkFactory2
+    {
+        public EmbulkEmbed.Bootstrap bootstrap(final ConfigSource systemConfig, final ConfigSource executorParams)
+        {
+            EmbulkEmbed.Bootstrap bootstrap = new EmbulkEmbed.Bootstrap();
+            bootstrap.setSystemConfig(systemConfig);
+
+            // add modules
+            //bootstrap.addModules(ImmutableList.<Module>of());
+
+            // override modules
+            bootstrap.overrideModules(new Function<List<Module>, List<Module>>()
+            {
+                public List<Module> apply(List<Module> modules)
+                {
+                    return ImmutableList.of(Modules.override(modules).with(new LoggerOverrideModule()));
+                }
+            });
+
+            return bootstrap;
+        }
+    }
+}

data/src/test/java/org/embulk/executor/mapreduce/TestPageBufferWritable.java

@@ -0,0 +1,84 @@
+package org.embulk.executor.mapreduce;
+
+import org.embulk.spi.Buffer;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageTestUtils;
+import org.embulk.spi.Schema;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.type.Types;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestPageBufferWritable
+{
+    @Rule
+    public MapReduceExecutorTestRuntime runtime = new MapReduceExecutorTestRuntime();
+
+    @Test
+    public void writeAndRead() throws IOException
+    {
+        Schema schema = Schema.builder()
+                .add("c0", Types.BOOLEAN)
+                .add("c1", Types.LONG)
+                .add("c2", Types.DOUBLE)
+                .add("c3", Types.STRING)
+                .add("c4", Types.TIMESTAMP)
+                .build();
+
+        for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), schema,
+                true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L),
+                true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L))) {
+
+            try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+                try (DataOutputStream dout = new DataOutputStream(out)) {
+                    PageWritable pw1 = new PageWritable();
+                    pw1.set(page);
+                    pw1.write(dout);
+
+                    BufferWritable bw1 = new BufferWritable();
+                    bw1.set(page.buffer());
+                    bw1.write(dout);
+                    dout.flush();
+
+                    try (DataInputStream din = new DataInputStream(new ByteArrayInputStream(out.toByteArray()))) {
+                        PageWritable pw2 = new PageWritable();
+                        pw2.readFields(din);
+
+                        BufferWritable bw2 = new BufferWritable();
+                        bw2.readFields(din);
+
+                        assertPageWritableEquals(pw1, pw2);
+                        assertBufferWritableEquals(bw1, bw2);
+                    }
+                }
+            }
+        }
+    }
+
+    static void assertPageWritableEquals(PageWritable pw1, PageWritable pw2)
+    {
+        Page p1 = pw1.get();
+        Page p2 = pw2.get();
+
+        assertEquals(p1.getStringReferences(), p2.getStringReferences());
+        assertBufferEquals(p1.buffer(), p2.buffer());
+    }
+
+    static void assertBufferWritableEquals(BufferWritable bw1, BufferWritable bw2)
+    {
+        assertBufferEquals(bw1.get(), bw2.get());
+    }
+
+    static void assertBufferEquals(Buffer b1, Buffer b2)
+    {
+        assertEquals(b1, b2);
+    }
+}

data/src/test/java/org/embulk/executor/mapreduce/TestTimestampPartitioning.java

@@ -0,0 +1,222 @@
+package org.embulk.executor.mapreduce;
+
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.PageTestUtils;
+import org.embulk.spi.Schema;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.type.Types;
+import org.embulk.executor.mapreduce.TimestampPartitioning.LongUnixTimestampPartitioner;
+import org.embulk.executor.mapreduce.TimestampPartitioning.TimestampPartitioner;
+import org.embulk.executor.mapreduce.TimestampPartitioning.Unit;
+import org.embulk.executor.mapreduce.TimestampPartitioning.UnixTimestampUnit;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+public class TestTimestampPartitioning
+{
+    @Rule
+    public MapReduceExecutorTestRuntime runtime = new MapReduceExecutorTestRuntime();
+
+    private TimestampPartitioning tp;
+
+    @Before
+    public void createTimestampPartitioning()
+    {
+        tp = new TimestampPartitioning();
+    }
+
+    @Test
+    public void validateConfigSource()
+            throws IOException
+    {
+        { // specified column is not included in schema
+            ConfigSource config = runtime.getExec().newConfigSource()
+                    .set("column", "_c0").set("unit", "hour").set("timezone", "UTC");
+            Schema schema = Schema.builder().add("not_included", Types.TIMESTAMP).build();
+
+            try {
+                tp.configure(config, schema, 0);
+                fail();
+            } catch (Throwable t) {
+                assertTrue(t instanceof ConfigException);
+            }
+        }
+
+        { // only UTC is supported now
+            ConfigSource config = runtime.getExec().newConfigSource()
+                    .set("column", "_c0").set("unit", "hour").set("timezone", "PDT");
+            Schema schema = Schema.builder().add("_c0", Types.TIMESTAMP).build();
+
+            try {
+                tp.configure(config, schema, 0);
+                fail();
+            } catch (Throwable t) {
+                assertTrue(t instanceof ConfigException);
+            }
+        }
+
+        { // the unit is only 'hour' or 'day'
+            ConfigSource config = runtime.getExec().newConfigSource()
+                    .set("column", "_c0").set("unit", "invalid").set("timezone", "UTC");
+            Schema schema = Schema.builder().add("_c0", Types.TIMESTAMP).build();
+
+            try {
+                tp.configure(config, schema, 0);
+                fail();
+            } catch (Throwable t) {
+                assertTrue(t instanceof ConfigException);
+            }
+        }
+
+        { // the column type is only timestamp or long
+            ConfigSource config = runtime.getExec().newConfigSource()
+                    .set("column", "_c0").set("unit", "hour").set("timezone", "UTC");
+            Schema schema = Schema.builder().add("_c0", Types.STRING).build();
+
+            try {
+                tp.configure(config, schema, 0);
+                fail();
+            } catch (Throwable t) {
+                assertTrue(t instanceof ConfigException);
+            }
+        }
+
+        { // if the column type is long, unix_timestamp_unit is required
+            ConfigSource config = runtime.getExec().newConfigSource()
+                    .set("column", "_c0").set("unit", "hour").set("timezone", "UTC").set("unix_timestamp_unit", "invalid");
+            Schema schema = Schema.builder().add("_c0", Types.LONG).build();
+
+            try {
+                tp.configure(config, schema, 0);
+                fail();
+            } catch (Throwable t) {
+                assertTrue(t instanceof ConfigException);
+            }
+        }
+    }
+
+    @Test
+    public void comparePartitionKeys()
+            throws Exception
+    {
+        List<PartitionKey> pks = new ArrayList<>();
+
+        Column c0 = new Column(0, "c0", Types.LONG);
+        Column c1 = new Column(1, "c1", Types.TIMESTAMP);
+        Schema schema = new Schema(Arrays.asList(c0, c1));
+
+        LongUnixTimestampPartitioner lp = new LongUnixTimestampPartitioner(c0, Unit.HOUR, UnixTimestampUnit.SEC);
+        TimestampPartitioner tp = new TimestampPartitioner(c1, Unit.HOUR);
+
+        long timeWindow = System.currentTimeMillis()/1000/3600*3600;
+        PageReader r = new PageReader(schema);
+        for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), schema,
+                timeWindow, Timestamp.ofEpochSecond(timeWindow),
+                timeWindow+1, Timestamp.ofEpochSecond(timeWindow+1),
+                timeWindow+3600, Timestamp.ofEpochSecond(timeWindow+3600),
+                timeWindow+3600+1, Timestamp.ofEpochSecond(timeWindow+3600+1),
+                timeWindow+2*3600, Timestamp.ofEpochSecond(timeWindow+2*3600),
+                timeWindow+2*3600+1, Timestamp.ofEpochSecond(timeWindow+2*3600+1)
+        )){
+            r.setPage(page);
+            while (r.nextRecord()) {
+                pks.add(lp.updateKey(r).clone());
+                pks.add(tp.updateKey(r).clone());
+            }
+        }
+
+        for (int i = 0; i < pks.size(); i += 2) {
+            assertTrue(pks.get(i).equals(pks.get(i+1))); // long(tw) == timestamp(tw)
+        }
+        for (int i = 0; i < pks.size() - 4; i += 4) {
+            assertTrue(pks.get(i).equals(pks.get(i+2))); // long(tw) == long (tw+1)
+        }
+        for (int i = 0; i < pks.size() - 4; i += 4) {
+            assertFalse(pks.get(i).equals(pks.get(i+4))); // long(tw) != long (tw+3600)
+        }
+    }
+
+    @Test
+    public void checkUnit()
+    {
+        long hourlyTimeWindow = System.currentTimeMillis() / 1000 / 3600 * 3600;
+        long dailyTimeWindow = System.currentTimeMillis() / 1000 / 86400 * 86600;
+
+        // hour
+        {
+            assertEquals(Unit.HOUR, Unit.of("hour"));
+            assertTrue(Unit.HOUR.utcPartition(hourlyTimeWindow) == Unit.HOUR.utcPartition(hourlyTimeWindow + 1));
+            assertTrue(Unit.HOUR.utcPartition(hourlyTimeWindow) != Unit.HOUR.utcPartition(hourlyTimeWindow + 3600));
+        }
+
+        // day
+        {
+            assertEquals(Unit.DAY, Unit.of("day"));
+            assertTrue(Unit.DAY.utcPartition(dailyTimeWindow) == Unit.DAY.utcPartition(dailyTimeWindow + 1));
+            assertTrue(Unit.DAY.utcPartition(dailyTimeWindow) != Unit.DAY.utcPartition(dailyTimeWindow + 86400));
+        }
+
+        // invalid_unit
+        {
+            try {
+                Unit.of("invalid_unit");
+                fail();
+            } catch (Exception e) {
+                assertTrue(e instanceof ConfigException);
+            }
+        }
+    }
+
+    @Test
+    public void checkUnixTimestampUnit()
+    {
+        long currentNano = System.nanoTime();
+        long currentSec = currentNano / 1000000000;
+
+        // sec
+        {
+            assertEquals(UnixTimestampUnit.SEC, UnixTimestampUnit.of("sec"));
+            assertEquals(currentSec, UnixTimestampUnit.SEC.toSeconds(currentNano / 1000000000));
+        }
+
+        // milli
+        {
+            assertEquals(UnixTimestampUnit.MILLI, UnixTimestampUnit.of("milli"));
+            assertEquals(currentSec, UnixTimestampUnit.MILLI.toSeconds(currentNano / 1000000));
+        }
+
+        // micro
+        {
+            assertEquals(UnixTimestampUnit.MICRO, UnixTimestampUnit.of("micro"));
+            assertEquals(currentSec, UnixTimestampUnit.MICRO.toSeconds(currentNano / 1000));
+        }
+
+        // nano
+        {
+            assertEquals(UnixTimestampUnit.NANO, UnixTimestampUnit.of("nano"));
+            assertEquals(currentSec, UnixTimestampUnit.NANO.toSeconds(currentNano));
+        }
+
+        // invalid_unit
+        {
+            try {
+                UnixTimestampUnit.of("invalid_unit");
+                fail();
+            } catch (Exception e) {
+                assertTrue(e instanceof ConfigException);
+            }
+        }
+    }
+}