embulk-executor-mapreduce 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 44794b114e21e1c5ced89169aa74b7d86a882d4d
-  data.tar.gz: 44ce570ea8b23c6d0c430598057022a1c36e73b2
+  metadata.gz: e8a1f6914bd6836006726de5d2f8badef02c614b
+  data.tar.gz: 97b14a2720664e78424dd8974865c97bbb4165de
 SHA512:
-  metadata.gz: cc9a2d3da2e8b89c0fb630652209897efcbf1f4ef7aebc71f82c6f3913bb6cb812bd2b583a84d7cac98b6dfd50d8380c92beac0a8406b0d69497400c3b40ee6d
-  data.tar.gz: 1f81b7fabaae57386f33c924bfd797e9fbe5f684e2a14cf87ccf68c2bc231a22b7aa5844f1ffeac9bee287fa082b34aaa45dbe6711445273ddf5635f0013a766
+  metadata.gz: 0b9c87ea48d10b8cab86e60aa251f0ce4d10720350cbe57c8ad229cb5a5e00d98ba05cb8e68e63218da9c619a34c100ef9da3e71bfd895a97b6d09e2ac816db4
+  data.tar.gz: 996dd45f438a2e420649627130021d20a295ce540611974bb50f79dc961b0f341adcce1ca944bd6facfd751bbfc430e944861e03f91fb57f7b785cf03c098735

data/src/main/java/org/embulk/executor/mapreduce/AttemptState.java

@@ -144,6 +144,7 @@ public class AttemptState
 
     public static AttemptState readFrom(InputStream in, ModelManager modelManager) throws IOException
     {
+        // If InputStream contains partial JSON (like '{"key":"va'), this method throws EOFException
         Scanner s = new Scanner(in, "UTF-8").useDelimiter("\\A");  // TODO
         if (s.hasNext()) {
             return modelManager.readObject(AttemptState.class, s.next());

data/src/main/java/org/embulk/executor/mapreduce/EmbulkMapReduce.java

@@ -1,5 +1,7 @@
 package org.embulk.executor.mapreduce;
 
+import java.io.EOFException;
+import java.io.InterruptedIOException;
 import java.util.List;
 import java.util.ArrayList;
 import java.util.concurrent.ExecutionException;
@@ -37,7 +39,12 @@ import org.embulk.spi.ExecAction;
 import org.embulk.spi.ExecSession;
 import org.embulk.spi.ProcessTask;
 import org.embulk.spi.util.Executors;
+import org.embulk.spi.util.RetryExecutor.Retryable;
+import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
 import org.embulk.EmbulkService;
+import org.slf4j.Logger;
+
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 public class EmbulkMapReduce
 {
@@ -141,12 +148,52 @@ public class EmbulkMapReduce
         }
     }
 
-    public static AttemptState readAttemptStateFile(Configuration config,
-            Path stateDir, TaskAttemptID id, ModelManager modelManager) throws IOException
+    public static AttemptState readAttemptStateFile(final Configuration config,
+            Path stateDir, TaskAttemptID id, final ModelManager modelManager) throws IOException
     {
-        Path path = new Path(stateDir, id.toString());
-        try (FSDataInputStream in = path.getFileSystem(config).open(path)) {
-            return AttemptState.readFrom(in, modelManager);
+        final Logger log = Exec.getLogger(EmbulkMapReduce.class);
+        final Path path = new Path(stateDir, id.toString());
+        try {
+            return retryExecutor()
+                    .withRetryLimit(5)
+                    .withInitialRetryWait(2 * 1000)
+                    .withMaxRetryWait(20 * 1000)
+                    .runInterruptible(new Retryable<AttemptState>() {
+                        @Override
+                        public AttemptState call() throws IOException {
+                            try (FSDataInputStream in = path.getFileSystem(config).open(path)) {
+                                return AttemptState.readFrom(in, modelManager);
+                            }
+                        }
+
+                        @Override
+                        public boolean isRetryableException(Exception exception) {
+                            // AttemptState.readFrom throws 2 types of exceptions:
+                            //   a) EOFException: race between readFrom and writeTo. See comments on AttemptState.readFrom.
+                            //   b) IOException "Cannot obtain block length for LocatedBlock": HDFS-1058. See https://github.com/embulk/embulk-executor-mapreduce/pull/3
+                            //   c) other IOException: FileSystem is not working
+                            //
+                            // a) and b) are temporary problem which is not critical. c) could be temporary problem and it is critical.
+                            // Here retries regardless of the exception type because we can't distinguish b) from c).
+                            return true;
+                        }
+
+                        @Override
+                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                                throws RetryGiveupException {
+                            log.warn("Retrying opening state file " + path.getName() + " error: " + exception);
+                        }
+
+                        @Override
+                        public void onGiveup(Exception firstException, Exception lastException)
+                                throws RetryGiveupException {
+                        }
+                    });
+        } catch (RetryGiveupException e) {
+            Throwables.propagateIfInstanceOf(e.getCause(), IOException.class);
+            throw Throwables.propagate(e.getCause());
+        } catch (InterruptedException e) {
+            throw new InterruptedIOException();
         }
     }
 
@@ -154,6 +201,7 @@ public class EmbulkMapReduce
             Path stateDir, AttemptState state, ModelManager modelManager) throws IOException
     {
         Path path = new Path(stateDir, state.getAttemptId().toString());
+        // TODO retry file create and write
         try (FSDataOutputStream out = path.getFileSystem(config).create(path, true)) {
             state.writeTo(out, modelManager);
         }

data/src/main/java/org/embulk/executor/mapreduce/MapReduceExecutor.java

@@ -221,12 +221,14 @@ public class MapReduceExecutor
                             job.mapProgress() * 100, job.reduceProgress() * 100));
                 Thread.sleep(interval);
 
-                updateProcessState(job, mapTaskCount, stateDir, state, modelManager);
+                updateProcessState(job, mapTaskCount, stateDir, state, modelManager, true);
             }
 
+            // Here sets skipUnavailable=false to updateProcessState method because race
+            // condition of AttemptReport.readFrom and .writeTo does not happen here.
             log.info(String.format("map %.1f%% reduce %.1f%%",
                         job.mapProgress() * 100, job.reduceProgress() * 100));
-            updateProcessState(job, mapTaskCount, stateDir, state, modelManager);
+            updateProcessState(job, mapTaskCount, stateDir, state, modelManager, false);
 
             Counters counters = job.getCounters();
             if (counters != null) {
@@ -291,7 +293,7 @@ public class MapReduceExecutor
     }
 
     private void updateProcessState(Job job, int mapTaskCount, Path stateDir,
-            ProcessState state, ModelManager modelManager) throws IOException
+            ProcessState state, ModelManager modelManager, boolean skipUnavailable) throws IOException
     {
         List<AttemptReport> reports = getAttemptReports(job.getConfiguration(), stateDir, modelManager);
 
@@ -299,8 +301,12 @@ public class MapReduceExecutor
             if (report == null) {
                 continue;
             }
-            if (!report.isStarted()) {
-                continue;
+            if (!report.isAvailable()) {
+                if (skipUnavailable) {
+                    continue;
+                } else {
+                    throw report.getUnavailableException();
+                }
             }
             AttemptState attempt = report.getAttemptState();
             if (attempt.getInputTaskIndex().isPresent()) {
@@ -338,23 +344,32 @@ public class MapReduceExecutor
     {
         private final TaskAttemptID attemptId;
         private final AttemptState attemptState;
+        private final IOException unavailableException;
 
-        public AttemptReport(TaskAttemptID attemptId)
+        public AttemptReport(TaskAttemptID attemptId, AttemptState attemptState)
         {
-            this(attemptId, null);
+            this.attemptId = attemptId;
+            this.attemptState = attemptState;
+            this.unavailableException = null;
         }
 
-        public AttemptReport(TaskAttemptID attemptId, AttemptState attemptState)
+        public AttemptReport(TaskAttemptID attemptId, IOException unavailableException)
         {
             this.attemptId = attemptId;
-            this.attemptState = attemptState;
+            this.attemptState = null;
+            this.unavailableException = unavailableException;
         }
 
-        public boolean isStarted()
+        public boolean isAvailable()
         {
             return attemptState != null;
         }
 
+        public IOException getUnavailableException()
+        {
+            return unavailableException;
+        }
+
         public boolean isInputCommitted()
         {
             return attemptState != null && attemptState.getInputCommitReport().isPresent();
@@ -382,8 +397,12 @@ public class MapReduceExecutor
                 AttemptState state = EmbulkMapReduce.readAttemptStateFile(config,
                         stateDir, aid, modelManager);
                 builder.add(new AttemptReport(aid, state));
-            } catch (EOFException ex) {  // plus Not Found exception
-                builder.add(new AttemptReport(aid, null));
+            } catch (IOException ex) {
+                // Either of:
+                //   * race condition of AttemptReport.writeTo and .readFrom
+                //   * FileSystem is not working
+                // See also comments on MapReduceExecutor.readAttemptStateFile.isRetryableException.
+                builder.add(new AttemptReport(aid, ex));
             }
         }
         return builder.build();

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-executor-mapreduce
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-04-08 00:00:00.000000000 Z
+date: 2015-06-22 00:00:00.000000000 Z
 dependencies: []
 description: Executes tasks on Hadoop.
 email:
@@ -42,6 +42,7 @@ files:
 - classpath/apacheds-kerberos-codec-2.0.0-M15.jar
 - classpath/api-asn1-api-1.0.0-M20.jar
 - classpath/api-util-1.0.0-M20.jar
+- classpath/asm-3.1.jar
 - classpath/avro-1.7.4.jar
 - classpath/commons-beanutils-1.7.0.jar
 - classpath/commons-cli-1.2.jar
@@ -59,7 +60,7 @@ files:
 - classpath/curator-client-2.6.0.jar
 - classpath/curator-framework-2.6.0.jar
 - classpath/curator-recipes-2.6.0.jar
-- classpath/embulk-executor-mapreduce-0.1.0.jar
+- classpath/embulk-executor-mapreduce-0.1.1.jar
 - classpath/gson-2.2.4.jar
 - classpath/hadoop-annotations-2.6.0.jar
 - classpath/hadoop-auth-2.6.0.jar