embulk-executor-mapreduce 0.1.0

data/src/main/java/org/embulk/executor/mapreduce/PluginArchive.java

@@ -0,0 +1,189 @@
+package org.embulk.executor.mapreduce;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.io.File;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Files;
+import java.nio.file.DirectoryStream;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.NotDirectoryException;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+import java.util.zip.ZipInputStream;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.ByteStreams;
+import org.jruby.embed.ScriptingContainer;
+import org.jruby.embed.InvokeFailedException;
+
+public class PluginArchive
+{
+    public static class GemSpec
+    {
+        private final String name;
+        private final List<String> requirePaths;
+
+        @JsonCreator
+        public GemSpec(
+                @JsonProperty("name") String name,
+                @JsonProperty("requirePaths") List<String> requirePaths)
+        {
+            this.name = name;
+            this.requirePaths = requirePaths;
+        }
+
+        @JsonProperty("name")
+        public String getName()
+        {
+            return name;
+        }
+
+        @JsonProperty("requirePaths")
+        public List<String> getRequirePaths()
+        {
+            return requirePaths;
+        }
+    }
+
+    private static class LocalGem
+            extends GemSpec
+    {
+        private final File localPath;
+
+        public LocalGem(File localPath, String name, List<String> requirePaths)
+        {
+            super(name, requirePaths);
+            this.localPath = localPath;
+        }
+
+        public File getLocalPath()
+        {
+            return localPath;
+        }
+    }
+
+    public static class Builder
+    {
+        private final ImmutableList.Builder<LocalGem> localGems = ImmutableList.builder();
+
+        @SuppressWarnings("unchecked")
+        public Builder addLoadedRubyGems(ScriptingContainer jruby)
+        {
+            List<List<String>> tuples = (List<List<String>>) jruby.runScriptlet("Gem.loaded_specs.map {|k,v| [k, v.full_gem_path, v.require_paths].flatten }");
+            for (List<String> tuple : tuples) {
+                String name = tuple.remove(0);
+                String fullGemPath = tuple.remove(0);
+                List<String> requirePaths = ImmutableList.copyOf(tuple);
+                addSpec(new File(fullGemPath), name, requirePaths);
+            }
+            return this;
+        }
+
+        public Builder addSpec(File localPath, String name, List<String> requirePaths)
+        {
+            localGems.add(new LocalGem(localPath, name, requirePaths));
+            return this;
+        }
+
+        public PluginArchive build()
+        {
+            return new PluginArchive(localGems.build());
+        }
+    }
+
+    private final List<LocalGem> localGems;
+
+    private PluginArchive(List<LocalGem> localGems)
+    {
+        this.localGems = localGems;
+    }
+
+    @SuppressWarnings("unchecked")
+    public void restoreLoadPathsTo(ScriptingContainer jruby)
+    {
+        List<String> loadPaths = (List<String>) jruby.runScriptlet("$LOAD_PATH");
+        for (LocalGem localGem : localGems) {
+            Path localGemPath = localGem.getLocalPath().toPath();
+            for (String requirePath : localGem.getRequirePaths()) {
+                loadPaths.add(localGemPath.resolve(requirePath).toString());
+            }
+        }
+        jruby.setLoadPaths(loadPaths);
+    }
+
+    public List<GemSpec> dump(OutputStream out)
+        throws IOException
+    {
+        ImmutableList.Builder<GemSpec> builder = ImmutableList.builder();
+        try (ZipOutputStream zip = new ZipOutputStream(out)) {
+            for (LocalGem localGem : localGems) {
+                zipDirectory(zip, localGem.getLocalPath().toPath(), localGem.getName() + "/");
+                builder.add(new GemSpec(localGem.getName(), localGem.getRequirePaths()));
+            }
+        }
+        return builder.build();
+    }
+
+    private static void zipDirectory(ZipOutputStream zip, Path directory, String name)
+        throws IOException
+    {
+        try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(directory)) {
+            for (Path path : dirStream) {
+                if (Files.isDirectory(path)) {
+                    zipDirectory(zip, path, name + path.getFileName() + "/");
+                } else {
+                    zip.putNextEntry(new ZipEntry(name + path.getFileName()));
+                    try (InputStream in = Files.newInputStream(path)) {
+                        ByteStreams.copy(in, zip);
+                    }
+                    zip.closeEntry();
+                }
+            }
+        } catch (NoSuchFileException | NotDirectoryException ex) {
+            // ignore
+        }
+    }
+
+    public static PluginArchive load(File localDirectory, List<GemSpec> gemSpecs,
+            InputStream in) throws IOException
+    {
+        try (ZipInputStream zip = new ZipInputStream(in)) {
+            unzipDirectory(zip, localDirectory.toPath());
+        }
+
+        ImmutableList.Builder<LocalGem> builder = ImmutableList.builder();
+        for (GemSpec gemSpec : gemSpecs) {
+            builder.add(new LocalGem(
+                        new File(localDirectory, gemSpec.getName()),
+                        gemSpec.getName(),
+                        gemSpec.getRequirePaths()));
+        }
+        return new PluginArchive(builder.build());
+    }
+
+    private static void unzipDirectory(ZipInputStream zip, Path directory)
+        throws IOException
+    {
+        while (true) {
+            ZipEntry entry = zip.getNextEntry();
+            if (entry == null) {
+                break;
+            }
+            Path path = directory.resolve(entry.getName());
+            if (entry.getName().endsWith("/")) {
+                Files.createDirectories(path);
+            } else {
+                Files.createDirectories(path.getParent());
+                try (OutputStream out = Files.newOutputStream(path)) {
+                    ByteStreams.copy(zip, out);
+                }
+            }
+        }
+    }
+}

data/src/main/java/org/embulk/executor/mapreduce/RemoteTaskFailedException.java

@@ -0,0 +1,10 @@
+package org.embulk.executor.mapreduce;
+
+public class RemoteTaskFailedException
+        extends RuntimeException
+{
+    public RemoteTaskFailedException(String message)
+    {
+        super(message);
+    }
+}

data/src/main/java/org/embulk/executor/mapreduce/SetContextClassLoader.java

@@ -0,0 +1,19 @@
+package org.embulk.executor.mapreduce;
+
+public class SetContextClassLoader
+        implements AutoCloseable
+{
+    private final ClassLoader original;
+
+    public SetContextClassLoader(ClassLoader classLoader)
+    {
+        this.original = Thread.currentThread().getContextClassLoader();
+        Thread.currentThread().setContextClassLoader(classLoader);
+    }
+
+    @Override
+    public void close()
+    {
+        Thread.currentThread().setContextClassLoader(original);
+    }
+}

data/src/main/java/org/embulk/executor/mapreduce/TimestampPartitioning.java

@@ -0,0 +1,291 @@
+package org.embulk.executor.mapreduce;
+
+import org.joda.time.DateTimeZone;
+import com.google.common.base.Optional;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigException;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.type.TimestampType;
+import org.embulk.spi.type.LongType;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+import org.embulk.spi.Buffer;
+
+public class TimestampPartitioning
+        implements Partitioning
+{
+    public interface PartitioningTask
+            extends Task
+    {
+        @Config("column")
+        public String getColumn();
+
+        @Config("unit")
+        public String getUnit();
+
+        @Config("timezone")
+        @ConfigDefault("\"UTC\"")
+        public DateTimeZone getTimeZone();
+
+        @Config("unix_timestamp_unit")
+        @ConfigDefault("\"sec\"")
+        public String getUnixTimestamp();
+
+        public Column getTargetColumn();
+        public void setTargetColumn(Column column);
+    }
+
+    private static enum Unit
+    {
+        HOUR(60*60),
+        DAY(24*60*60);
+        //WEEK
+        //MONTH,
+        //YEAR;
+
+        private final int unit;
+
+        private Unit(int unit)
+        {
+            this.unit = unit;
+        }
+
+        public long utcPartition(long seconds)
+        {
+            return seconds / unit;
+        }
+
+        public static Unit of(String s)
+        {
+            switch (s) {
+            case "hour": return HOUR;
+            case "day": return DAY;
+            //case "week": return WEEK;
+            //case "month": return MONTH;
+            //case "year": return YEAR;
+            default:
+                throw new ConfigException(
+                        String.format("Unknown unit '%s'. Supported units are hour and day"));
+            }
+        }
+    }
+
+    private static enum UnixTimestampUnit
+    {
+        SEC(1),
+        MILLI(1000),
+        MICRO(1000000),
+        NANO(1000000000);
+
+        private final int unit;
+
+        private UnixTimestampUnit(int unit)
+        {
+            this.unit = unit;
+        }
+
+        public long toSeconds(long v)
+        {
+            return v / unit;
+        }
+
+        public static UnixTimestampUnit of(String s)
+        {
+            switch (s) {
+            case "sec": return SEC;
+            case "milli": return MILLI;
+            case "micro": return MICRO;
+            case "nano": return NANO;
+            default:
+                throw new ConfigException(
+                        String.format("Unknown unix_timestamp_unit '%s'. Supported units are sec, milli, micro, and nano"));
+            }
+        }
+    }
+
+    @Override
+    public TaskSource configure(ConfigSource config, Schema schema, int outputTaskCount)
+    {
+        PartitioningTask task = config.loadConfig(PartitioningTask.class);
+        Column column = findColumnByName(schema, task.getColumn());
+
+        if (!task.getTimeZone().equals(DateTimeZone.UTC)) {
+            // TODO
+            throw new ConfigException("Timestamp partitioner supports only UTC time zone for now");
+        }
+
+        // validate unit
+        Unit.of(task.getUnit());
+
+        // validate type
+        if (column.getType() instanceof TimestampType) {
+            // ok
+        } else if (column.getType() instanceof LongType) {
+            // validate unix_timestamp_unit
+            UnixTimestampUnit.of(task.getUnixTimestamp());
+        } else {
+            throw new ConfigException(
+                    String.format("Partitioning column '%s' must be timestamp or long but got '%s'", column.getName(), column.getType()));
+        }
+
+        task.setTargetColumn(column);
+
+        return task.dump();
+    }
+
+    private static Column findColumnByName(Schema schema, String columnName)
+    {
+        for (Column column : schema.getColumns()) {
+            if (column.getName().equals(columnName)) {
+                return column;
+            }
+        }
+        throw new ConfigException(
+                String.format("Column '%s' is not found in schema", columnName));
+    }
+
+    @Override
+    public Partitioner newPartitioner(TaskSource taskSource)
+    {
+        PartitioningTask task = taskSource.loadTask(PartitioningTask.class);
+
+        Column column = task.getTargetColumn();
+        if (column.getType() instanceof TimestampType) {
+            return new TimestampPartitioner(column, Unit.of(task.getUnit()));
+        } else if (column.getType() instanceof LongType) {
+            return new LongUnixTimestampPartitioner(column, Unit.of(task.getUnit()), UnixTimestampUnit.of(task.getUnixTimestamp()));
+        } else {
+            throw new AssertionError();
+        }
+    }
+
+    private static class LongPartitionKey
+            implements PartitionKey
+    {
+        public static Buffer newKeyBuffer()
+        {
+            Buffer buffer = Buffer.allocate(8);
+            buffer.limit(8);
+            return buffer;
+        }
+
+        private long value;
+
+        public LongPartitionKey()
+        { }
+
+        private LongPartitionKey(long value)
+        {
+            this.value = value;
+        }
+
+        public void set(long value)
+        {
+            this.value = value;
+        }
+
+        @Override
+        public void dump(Buffer buffer)
+        {
+            // TODO optimize
+            buffer.array()[0] = (byte) (((int) (value >>>  0)) & 0xff);
+            buffer.array()[1] = (byte) (((int) (value >>>  4)) & 0xff);
+            buffer.array()[2] = (byte) (((int) (value >>>  8)) & 0xff);
+            buffer.array()[3] = (byte) (((int) (value >>> 12)) & 0xff);
+            buffer.array()[4] = (byte) (((int) (value >>> 16)) & 0xff);
+            buffer.array()[5] = (byte) (((int) (value >>> 20)) & 0xff);
+            buffer.array()[6] = (byte) (((int) (value >>> 24)) & 0xff);
+            buffer.array()[7] = (byte) (((int) (value >>> 28)) & 0xff);
+        }
+
+        @Override
+        public LongPartitionKey clone()
+        {
+            return new LongPartitionKey(value);
+        }
+
+        @Override
+        public boolean equals(Object other)
+        {
+            if (!(other instanceof LongPartitionKey)) {
+                return false;
+            }
+            LongPartitionKey o = (LongPartitionKey) other;
+            return value == o.value;
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return (int) (value ^ (value >>> 32));
+        }
+    }
+
+    private static abstract class AbstractTimestampPartitioner
+            implements Partitioner
+    {
+        protected final Column column;
+        protected final Unit unit;
+        private final LongPartitionKey key;
+
+        public AbstractTimestampPartitioner(Column column, Unit unit)
+        {
+            this.column = column;
+            this.unit = unit;
+            this.key = new LongPartitionKey();
+        }
+
+        @Override
+        public Buffer newKeyBuffer()
+        {
+            return LongPartitionKey.newKeyBuffer();
+        }
+
+        protected LongPartitionKey updateKey(long v)
+        {
+            key.set(v);
+            return key;
+        }
+    }
+
+    private static class TimestampPartitioner
+            extends AbstractTimestampPartitioner
+    {
+        public TimestampPartitioner(Column column, Unit unit)
+        {
+            super(column, unit);
+        }
+
+        @Override
+        public PartitionKey updateKey(PageReader record)
+        {
+            Timestamp v = record.getTimestamp(column);
+            return super.updateKey(unit.utcPartition(v.getEpochSecond()));
+        }
+    }
+
+    private static class LongUnixTimestampPartitioner
+            extends AbstractTimestampPartitioner
+    {
+        private final UnixTimestampUnit unixTimestampUnit;
+
+        public LongUnixTimestampPartitioner(Column column, Unit unit,
+                UnixTimestampUnit unixTimestampUnit)
+        {
+            super(column, unit);
+            this.unixTimestampUnit = unixTimestampUnit;
+        }
+
+        @Override
+        public PartitionKey updateKey(PageReader record)
+        {
+            long v = record.getLong(column);
+            return super.updateKey(unit.utcPartition(unixTimestampUnit.toSeconds(v)));
+        }
+    }
+}

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: embulk-executor-mapreduce
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Sadayuki Furuhashi
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-04-08 00:00:00.000000000 Z
+dependencies: []
+description: Executes tasks on Hadoop.
+email:
+- frsyuki@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- build.gradle
+- lib/embulk/executor/mapreduce.rb
+- src/main/java/org/embulk/executor/mapreduce/AttemptState.java
+- src/main/java/org/embulk/executor/mapreduce/BufferWritable.java
+- src/main/java/org/embulk/executor/mapreduce/BufferedPagePartitioner.java
+- src/main/java/org/embulk/executor/mapreduce/EmbulkInputFormat.java
+- src/main/java/org/embulk/executor/mapreduce/EmbulkInputSplit.java
+- src/main/java/org/embulk/executor/mapreduce/EmbulkMapReduce.java
+- src/main/java/org/embulk/executor/mapreduce/EmbulkPartitioningMapReduce.java
+- src/main/java/org/embulk/executor/mapreduce/EmbulkRecordReader.java
+- src/main/java/org/embulk/executor/mapreduce/MapReduceExecutor.java
+- src/main/java/org/embulk/executor/mapreduce/MapReduceExecutorTask.java
+- src/main/java/org/embulk/executor/mapreduce/PageWritable.java
+- src/main/java/org/embulk/executor/mapreduce/PartitionKey.java
+- src/main/java/org/embulk/executor/mapreduce/Partitioner.java
+- src/main/java/org/embulk/executor/mapreduce/Partitioning.java
+- src/main/java/org/embulk/executor/mapreduce/PluginArchive.java
+- src/main/java/org/embulk/executor/mapreduce/RemoteTaskFailedException.java
+- src/main/java/org/embulk/executor/mapreduce/SetContextClassLoader.java
+- src/main/java/org/embulk/executor/mapreduce/TimestampPartitioning.java
+- classpath/activation-1.1.jar
+- classpath/apacheds-i18n-2.0.0-M15.jar
+- classpath/apacheds-kerberos-codec-2.0.0-M15.jar
+- classpath/api-asn1-api-1.0.0-M20.jar
+- classpath/api-util-1.0.0-M20.jar
+- classpath/avro-1.7.4.jar
+- classpath/commons-beanutils-1.7.0.jar
+- classpath/commons-cli-1.2.jar
+- classpath/commons-codec-1.6.jar
+- classpath/commons-collections-3.2.1.jar
+- classpath/commons-compress-1.4.1.jar
+- classpath/commons-configuration-1.6.jar
+- classpath/commons-digester-1.8.jar
+- classpath/commons-httpclient-3.1.jar
+- classpath/commons-io-2.4.jar
+- classpath/commons-lang-2.6.jar
+- classpath/commons-logging-1.1.3.jar
+- classpath/commons-math3-3.1.1.jar
+- classpath/commons-net-3.1.jar
+- classpath/curator-client-2.6.0.jar
+- classpath/curator-framework-2.6.0.jar
+- classpath/curator-recipes-2.6.0.jar
+- classpath/embulk-executor-mapreduce-0.1.0.jar
+- classpath/gson-2.2.4.jar
+- classpath/hadoop-annotations-2.6.0.jar
+- classpath/hadoop-auth-2.6.0.jar
+- classpath/hadoop-client-2.6.0.jar
+- classpath/hadoop-common-2.6.0.jar
+- classpath/hadoop-hdfs-2.6.0.jar
+- classpath/hadoop-mapreduce-client-app-2.6.0.jar
+- classpath/hadoop-mapreduce-client-common-2.6.0.jar
+- classpath/hadoop-mapreduce-client-core-2.6.0.jar
+- classpath/hadoop-mapreduce-client-jobclient-2.6.0.jar
+- classpath/hadoop-mapreduce-client-shuffle-2.6.0.jar
+- classpath/hadoop-yarn-api-2.6.0.jar
+- classpath/hadoop-yarn-client-2.6.0.jar
+- classpath/hadoop-yarn-common-2.6.0.jar
+- classpath/hadoop-yarn-server-common-2.6.0.jar
+- classpath/hadoop-yarn-server-nodemanager-2.6.0.jar
+- classpath/htrace-core-3.0.4.jar
+- classpath/httpclient-4.2.5.jar
+- classpath/httpcore-4.2.4.jar
+- classpath/jackson-core-asl-1.9.13.jar
+- classpath/jackson-jaxrs-1.9.13.jar
+- classpath/jackson-mapper-asl-1.9.13.jar
+- classpath/jackson-xc-1.9.13.jar
+- classpath/jaxb-api-2.2.2.jar
+- classpath/jaxb-impl-2.2.3-1.jar
+- classpath/jersey-client-1.9.jar
+- classpath/jersey-core-1.9.jar
+- classpath/jersey-guice-1.9.jar
+- classpath/jersey-json-1.9.jar
+- classpath/jersey-server-1.9.jar
+- classpath/jettison-1.1.jar
+- classpath/jetty-util-6.1.26.jar
+- classpath/jline-0.9.94.jar
+- classpath/jsr305-1.3.9.jar
+- classpath/leveldbjni-all-1.8.jar
+- classpath/netty-3.7.0.Final.jar
+- classpath/paranamer-2.3.jar
+- classpath/protobuf-java-2.5.0.jar
+- classpath/servlet-api-2.5.jar
+- classpath/snappy-java-1.0.4.1.jar
+- classpath/stax-api-1.0-2.jar
+- classpath/xmlenc-0.52.jar
+- classpath/xz-1.0.jar
+- classpath/zookeeper-3.4.6.jar
+homepage: https://github.com/embulk/embulk-executor-mapreduce
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: MapReduce executor plugin for Embulk
+test_files: []