embulk-executor-mapreduce 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 995cd8754f48d4d4e0d222bbad7d39dfdad6648f
-  data.tar.gz: 901b6a9298e76ed45ab7840ca4e4135bb68a9d8d
+  metadata.gz: 5c531e6955469a01f0e2ed716a65fdf228ae95ba
+  data.tar.gz: d8724a7abcaedd7549a397d2b14df6edb2832d52
 SHA512:
-  metadata.gz: 02cbb013f7b83f51c787e649d1efd0e69b726af56636b2487b5d1b277bb285521db44a5b4b1ee45c59d7e5c0a71515bc9b590cdb5a3f90f5fd871581e4d23286
-  data.tar.gz: f66984fd6801ee826b6c4a46db1c326aa699584497224daed575e11c6f83348f5cbf2ee469c26a856bf4d9479d57e609613abdc7fec69f3f2ccbde8ac612340c
+  metadata.gz: 9983809cd453596cf3fc683f75a22b43dbcca0abf202c7ad7bdc9cbb640c673e95be706f6d49ad04bb3881f8668bc9b8e3946115d54e2a14e90dfb163fbd9b0f
+  data.tar.gz: 7b272911c78f9bac87fa867f0e3978a96f31c8fb348b150410a6991e1f5f425782450c9422ca4022e2cb15abc61037caaf080fb196f48bd0a441e3e81cc31b15

data/src/main/java/org/embulk/executor/mapreduce/AttemptState.java

@@ -16,7 +16,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonValue;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.embulk.config.ModelManager;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 
 public class AttemptState
 {
@@ -24,8 +24,8 @@ public class AttemptState
     private final Optional<Integer> inputTaskIndex;
     private final Optional<Integer> outputTaskIndex;
     private Optional<String> exception;
-    private Optional<CommitReport> inputCommitReport;
-    private Optional<CommitReport> outputCommitReport;
+    private Optional<TaskReport> inputTaskReport;
+    private Optional<TaskReport> outputTaskReport;
 
     public AttemptState(TaskAttemptID attemptId, Optional<Integer> inputTaskIndex, Optional<Integer> outputTaskIndex)
     {
@@ -40,12 +40,12 @@ public class AttemptState
             @JsonProperty("inputTaskIndex") Optional<Integer> inputTaskIndex,
             @JsonProperty("outputTaskIndex") Optional<Integer> outputTaskIndex,
             @JsonProperty("exception") Optional<String> exception,
-            @JsonProperty("inputCommitReport") Optional<CommitReport> inputCommitReport,
-            @JsonProperty("outputCommitReport") Optional<CommitReport> outputCommitReport)
+            @JsonProperty("inputTaskReport") Optional<TaskReport> inputTaskReport,
+            @JsonProperty("outputTaskReport") Optional<TaskReport> outputTaskReport)
     {
         this(TaskAttemptID.forName(attemptId),
                 inputTaskIndex, outputTaskIndex, exception,
-                inputCommitReport, outputCommitReport);
+                inputTaskReport, outputTaskReport);
     }
 
     public AttemptState(
@@ -53,15 +53,15 @@ public class AttemptState
             Optional<Integer> inputTaskIndex,
             Optional<Integer> outputTaskIndex,
             Optional<String> exception,
-            Optional<CommitReport> inputCommitReport,
-            Optional<CommitReport> outputCommitReport)
+            Optional<TaskReport> inputTaskReport,
+            Optional<TaskReport> outputTaskReport)
     {
         this.attemptId = attemptId;
         this.inputTaskIndex = inputTaskIndex;
         this.outputTaskIndex = outputTaskIndex;
         this.exception = exception;
-        this.inputCommitReport = inputCommitReport;
-        this.outputCommitReport = outputCommitReport;
+        this.inputTaskReport = inputTaskReport;
+        this.outputTaskReport = outputTaskReport;
     }
 
     @JsonIgnore
@@ -112,28 +112,28 @@ public class AttemptState
         return exception;
     }
 
-    @JsonProperty("inputCommitReport")
-    public Optional<CommitReport> getInputCommitReport()
+    @JsonProperty("inputTaskReport")
+    public Optional<TaskReport> getInputTaskReport()
     {
-        return inputCommitReport;
+        return inputTaskReport;
     }
 
-    @JsonProperty("outputCommitReport")
-    public Optional<CommitReport> getOutputCommitReport()
+    @JsonProperty("outputTaskReport")
+    public Optional<TaskReport> getOutputTaskReport()
     {
-        return outputCommitReport;
+        return outputTaskReport;
     }
 
     @JsonIgnore
-    public void setInputCommitReport(CommitReport inputCommitReport)
+    public void setInputTaskReport(TaskReport inputTaskReport)
     {
-        this.inputCommitReport = Optional.of(inputCommitReport);
+        this.inputTaskReport = Optional.of(inputTaskReport);
     }
 
     @JsonIgnore
-    public void setOutputCommitReport(CommitReport outputCommitReport)
+    public void setOutputTaskReport(TaskReport outputTaskReport)
     {
-        this.outputCommitReport = Optional.of(outputCommitReport);
+        this.outputTaskReport = Optional.of(outputTaskReport);
     }
 
     public void writeTo(OutputStream out, ModelManager modelManager) throws IOException

data/src/main/java/org/embulk/executor/mapreduce/DefaultEmbulkFactory.java

@@ -0,0 +1,13 @@
+package org.embulk.executor.mapreduce;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.EmbulkEmbed;
+
+public class DefaultEmbulkFactory
+{
+    public EmbulkEmbed.Bootstrap bootstrap(ConfigSource systemConfig)
+    {
+        return new EmbulkEmbed.Bootstrap()
+            .setSystemConfig(systemConfig);
+    }
+}

data/src/main/java/org/embulk/executor/mapreduce/EmbulkMapReduce.java

@@ -2,6 +2,7 @@ package org.embulk.executor.mapreduce;
 
 import java.util.List;
 import java.util.ArrayList;
+import java.util.Map;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.io.File;
@@ -10,6 +11,7 @@ import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.EOFException;
 import java.io.InterruptedIOException;
+import java.lang.reflect.Method;
 import java.lang.reflect.InvocationTargetException;
 import com.google.inject.Injector;
 import com.google.common.base.Optional;
@@ -35,7 +37,8 @@ import org.apache.hadoop.mapreduce.MRConfig;
 import org.embulk.config.ModelManager;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigLoader;
-import org.embulk.config.CommitReport;
+import org.embulk.config.DataSourceImpl;
+import org.embulk.config.TaskReport;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Exec;
 import org.embulk.spi.ExecAction;
@@ -44,7 +47,7 @@ import org.embulk.spi.ProcessTask;
 import org.embulk.spi.util.Executors;
 import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
-import org.embulk.EmbulkService;
+import org.embulk.EmbulkEmbed;
 import org.slf4j.Logger;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -52,6 +55,7 @@ import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 public class EmbulkMapReduce
 {
+    private static final String EMBULK_FACTORY_CLASS = "embulk_factory_class";
     private static final String SYSTEM_CONFIG_SERVICE_CLASS = "mapreduce_service_class";
 
     private static final String CK_SYSTEM_CONFIG = "embulk.mapreduce.systemConfig";
@@ -111,29 +115,41 @@ public class EmbulkMapReduce
                 config.get(CK_TASK));
     }
 
-    public static Injector newEmbulkInstance(Configuration config)
+    public static EmbulkEmbed.Bootstrap newEmbulkBootstrap(Configuration config)
     {
         ConfigSource systemConfig = getSystemConfig(config);
-        String serviceClassName = systemConfig.get(String.class, SYSTEM_CONFIG_SERVICE_CLASS, "org.embulk.EmbulkService");
+
+        // for warnings of old versions
+        if (!systemConfig.get(String.class, SYSTEM_CONFIG_SERVICE_CLASS, "org.embulk.EmbulkService").equals("org.embulk.EmbulkService")) {
+            throw new RuntimeException("System config 'mapreduce_service_class' is not supported any more. Please use 'embulk_factory_class' instead");
+        }
+
+        String factoryClassName = systemConfig.get(String.class, EMBULK_FACTORY_CLASS, DefaultEmbulkFactory.class.getName());
 
         try {
-            Object obj;
-            if (serviceClassName.equals("org.embulk.EmbulkService")) {
-                obj = new EmbulkService(systemConfig);
-            } else {
-                Class<?> serviceClass = Class.forName(serviceClassName);
-                obj = serviceClass.getConstructor(ConfigSource.class).newInstance(systemConfig);
-            }
+            Class<?> factoryClass = Class.forName(factoryClassName);
+            Object factory = factoryClass.newInstance();
 
-            if (obj instanceof EmbulkService) {
-                return ((EmbulkService) obj).getInjector();
-            } else {
-                return (Injector) obj.getClass().getMethod("getInjector").invoke(obj);
+            Object bootstrap;
+            try {
+                // factory.bootstrap(ConfigSource masterSystemConfig, ConfigSource executorParams)
+                Method method = factoryClass.getMethod("bootstrap", ConfigSource.class, ConfigSource.class);
+                Map<String, String> hadoopConfig = config.getValByRegex("");
+                ConfigSource executorParams = new DataSourceImpl(new ModelManager(null, new ObjectMapper())).set("hadoopConfig", hadoopConfig).getNested("hadoopConfig");  // TODO add a method to embulk that creates an empty DataSource instance
+                bootstrap = method.invoke(factory, systemConfig, executorParams);
             }
+            catch (NoSuchMethodException ex) {
+                // factory.bootstrap(ConfigSource masterSystemConfig)
+                bootstrap = factoryClass.getMethod("bootstrap", ConfigSource.class).invoke(factory, systemConfig);
+            }
+
+            return (EmbulkEmbed.Bootstrap) bootstrap;
 
-        } catch (InvocationTargetException ex) {
+        }
+        catch (InvocationTargetException ex) {
             throw Throwables.propagate(ex.getCause());
-        } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | IllegalAccessException | IllegalArgumentException ex) {
+        }
+        catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | IllegalAccessException | IllegalArgumentException ex) {
             throw Throwables.propagate(ex);
         }
     }
@@ -377,8 +393,7 @@ public class EmbulkMapReduce
     public static class SessionRunner
     {
         private final Configuration config;
-        private final Injector injector;
-        private final ModelManager modelManager;
+        private final EmbulkEmbed embed;
         private final MapReduceExecutorTask task;
         private final ExecSession session;
         private final File localGemPath;
@@ -386,10 +401,9 @@ public class EmbulkMapReduce
         public SessionRunner(TaskAttemptContext context)
         {
             this.config = context.getConfiguration();
-            this.injector = newEmbulkInstance(context.getConfiguration());
-            this.modelManager = injector.getInstance(ModelManager.class);
-            this.task = getExecutorTask(injector, context.getConfiguration());
-            this.session = ExecSession.builder(injector).fromExecConfig(task.getExecConfig()).build();
+            this.embed = newEmbulkBootstrap(context.getConfiguration()).initialize();  // TODO use initializeCloseable?
+            this.task = getExecutorTask(embed.getInjector(), context.getConfiguration());
+            this.session = ExecSession.builder(embed.getInjector()).fromExecConfig(task.getExecConfig()).build();
 
             try {
                 LocalDirAllocator localDirAllocator = new LocalDirAllocator(MRConfig.LOCAL_DIR);
@@ -403,7 +417,7 @@ public class EmbulkMapReduce
         public PluginArchive readPluginArchive() throws IOException
         {
             localGemPath.mkdirs();
-            return EmbulkMapReduce.readPluginArchive(localGemPath, config, getStateDirectoryPath(config), modelManager);
+            return EmbulkMapReduce.readPluginArchive(localGemPath, config, getStateDirectoryPath(config), embed.getModelManager());
         }
 
         public Configuration getConfiguration()
@@ -413,17 +427,17 @@ public class EmbulkMapReduce
 
         public ModelManager getModelManager()
         {
-            return modelManager;
+            return embed.getModelManager();
         }
 
         public BufferAllocator getBufferAllocator()
         {
-            return injector.getInstance(BufferAllocator.class);
+            return embed.getBufferAllocator();
         }
 
         public ScriptingContainer getScriptingContainer()
         {
-            return injector.getInstance(ScriptingContainer.class);
+            return embed.getInjector().getInstance(ScriptingContainer.class);
         }
 
         public MapReduceExecutorTask getMapReduceExecutorTask()
@@ -480,9 +494,9 @@ public class EmbulkMapReduce
         }
 
         @Override
-        public void inputCommitted(CommitReport report)
+        public void inputCommitted(TaskReport report)
         {
-            state.setInputCommitReport(report);
+            state.setInputTaskReport(report);
             try {
                 writeAttemptStateFile(config, stateDir, state, modelManager);
             } catch (IOException e) {
@@ -491,9 +505,9 @@ public class EmbulkMapReduce
         }
 
         @Override
-        public void outputCommitted(CommitReport report)
+        public void outputCommitted(TaskReport report)
         {
-            state.setOutputCommitReport(report);
+            state.setOutputTaskReport(report);
             try {
                 writeAttemptStateFile(config, stateDir, state, modelManager);
             } catch (IOException e) {

data/src/main/java/org/embulk/executor/mapreduce/EmbulkPartitioningMapReduce.java

@@ -12,7 +12,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.embulk.config.ModelManager;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
@@ -218,7 +218,7 @@ public class EmbulkPartitioningMapReduce
                     try {
                         if (!failed) {
                             output.finish();
-                            CommitReport report = output.commit();
+                            TaskReport report = output.commit();
                             handler.outputCommitted(report);
                         }
                     } finally {
@@ -266,7 +266,7 @@ public class EmbulkPartitioningMapReduce
 
         public void cleanup(TaskSource taskSource,
                 Schema schema, int taskCount,
-                List<CommitReport> successCommitReports)
+                List<TaskReport> successTaskReports)
         {
             // won't be called
             throw new RuntimeException("");
@@ -301,9 +301,9 @@ public class EmbulkPartitioningMapReduce
                 public void abort()
                 { }
 
-                public CommitReport commit()
+                public TaskReport commit()
                 {
-                    return Exec.newCommitReport();
+                    return Exec.newTaskReport();
                 }
             };
         }

data/src/main/java/org/embulk/executor/mapreduce/MapReduceExecutor.java

@@ -40,7 +40,7 @@ import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
 import org.embulk.exec.ForSystemConfig;
 import org.embulk.config.ConfigSource;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 import org.embulk.config.ConfigException;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ModelManager;
@@ -78,7 +78,7 @@ public class MapReduceExecutor
         final int outputTaskCount;
         final int reduceTaskCount;
 
-        if (task.getPartitioning().isPresent()) {
+        if (task.getPartitioning().isPresent() && inputTaskCount > 0) { // here can disable partitioning and force set reduceTaskCount and outputTaskCount to 0 if inputTaskCount is 0
             reduceTaskCount = task.getReducers().or(inputTaskCount);
             if (reduceTaskCount <= 0) {
                 throw new ConfigException("Reducers must be larger than 1 if partition: is set");
@@ -381,15 +381,15 @@ public class MapReduceExecutor
     private static void updateTaskState(TaskState state, AttemptState attempt, boolean isInput)
     {
         state.start();
-        Optional<CommitReport> commitReport = isInput ? attempt.getInputCommitReport() : attempt.getOutputCommitReport();
-        boolean committed = commitReport.isPresent();
+        Optional<TaskReport> taskReport = isInput ? attempt.getInputTaskReport() : attempt.getOutputTaskReport();
+        boolean committed = taskReport.isPresent();
         if (attempt.getException().isPresent()) {
             if (!state.isCommitted()) {
                 state.setException(new RemoteTaskFailedException(attempt.getException().get()));
             }
         }
-        if (commitReport.isPresent()) {
-            state.setCommitReport(commitReport.get());
+        if (taskReport.isPresent()) {
+            state.setTaskReport(taskReport.get());
             state.finish();
         }
     }
@@ -436,12 +436,12 @@ public class MapReduceExecutor
 
         public boolean isInputCommitted()
         {
-            return attemptState != null && attemptState.getInputCommitReport().isPresent();
+            return attemptState != null && attemptState.getInputTaskReport().isPresent();
         }
 
         public boolean isOutputCommitted()
         {
-            return attemptState != null && attemptState.getOutputCommitReport().isPresent();
+            return attemptState != null && attemptState.getOutputTaskReport().isPresent();
         }
 
         public TaskAttemptID getTaskAttempId()

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-executor-mapreduce
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-28 00:00:00.000000000 Z
+date: 2015-08-19 00:00:00.000000000 Z
 dependencies: []
 description: Executes tasks on Hadoop.
 email:
@@ -22,6 +22,7 @@ files:
 - src/main/java/org/embulk/executor/mapreduce/AttemptState.java
 - src/main/java/org/embulk/executor/mapreduce/BufferWritable.java
 - src/main/java/org/embulk/executor/mapreduce/BufferedPagePartitioner.java
+- src/main/java/org/embulk/executor/mapreduce/DefaultEmbulkFactory.java
 - src/main/java/org/embulk/executor/mapreduce/EmbulkInputFormat.java
 - src/main/java/org/embulk/executor/mapreduce/EmbulkInputSplit.java
 - src/main/java/org/embulk/executor/mapreduce/EmbulkMapReduce.java
@@ -60,7 +61,7 @@ files:
 - classpath/curator-client-2.6.0.jar
 - classpath/curator-framework-2.6.0.jar
 - classpath/curator-recipes-2.6.0.jar
-- classpath/embulk-executor-mapreduce-0.1.5.jar
+- classpath/embulk-executor-mapreduce-0.2.0.jar
 - classpath/gson-2.2.4.jar
 - classpath/hadoop-annotations-2.6.0.jar
 - classpath/hadoop-auth-2.6.0.jar