embulk-executor-mapreduce 0.2.5 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4461eebeecc53f99b9b9683d7553a585a87e1a1f
-  data.tar.gz: a019cd9224918ae2721482a9cf92c9c8148a05a6
+  metadata.gz: 0f276edacaa4ab8219234ec69e521edaee7c5104
+  data.tar.gz: 1bff944c1f3ab1b3a406fd871ec43888eeff43a1
 SHA512:
-  metadata.gz: 91e107ce10160fc097930b139f07b59dcb80b1201dde0723cc302fd1e142a283ad8a817d7518ac7684f0b066ed55537c20ae0af5446230eb0f63026d9bf7e21d
-  data.tar.gz: bc045316fedf83de62e34bbf9304152680d90e46ec6fa885fc054aca50d9a967b2d8f8eb3fa86e7c685badba7e3f15834f985fee923c5bf54c9f5197f43a68fb
+  metadata.gz: f49485f369fde98696410fc98cd38ff4e27f83a1286f1e892bc2bd06345cfb2cc28049a7ce8751f84af37663c1633345942431555c97ef4190d55f0c376724d8
+  data.tar.gz: 7f4460a9690a2462f87b179398bc2380f699fed06112034bfa10168e1547152fc6afc45816ea69f2261559aba0c9dccc47d633d4cac26a3de571afc52685f039

data/src/main/java/org/embulk/executor/mapreduce/EmbulkMapReduce.java

@@ -196,7 +196,7 @@ public class EmbulkMapReduce
 
     public static JobStatus getJobStatus(final Job job) throws IOException
     {
-        return hadoopOperationWithRetry("getting job status", new Callable<JobStatus>() {
+        return hadoopOperationWithRetry("Getting job status", new Callable<JobStatus>() {
             public JobStatus call() throws IOException
             {
                 return new JobStatus(job.isComplete(), job.mapProgress(), job.reduceProgress());
@@ -206,7 +206,7 @@ public class EmbulkMapReduce
 
     public static Counters getJobCounters(final Job job) throws IOException
     {
-        return hadoopOperationWithRetry("getting job counters", new Callable<Counters>() {
+        return hadoopOperationWithRetry("Getting job counters", new Callable<Counters>() {
             public Counters call() throws IOException
             {
                 return job.getCounters();
@@ -217,7 +217,7 @@ public class EmbulkMapReduce
     public static List<TaskAttemptID> listAttempts(final Configuration config,
             final Path stateDir) throws IOException
     {
-        return hadoopOperationWithRetry("getting list of attempt state files on "+stateDir, new Callable<List<TaskAttemptID>>() {
+        return hadoopOperationWithRetry("Getting list of attempt state files on "+stateDir, new Callable<List<TaskAttemptID>>() {
             public List<TaskAttemptID> call() throws IOException
             {
                 FileStatus[] stats = stateDir.getFileSystem(config).listStatus(stateDir);
@@ -244,7 +244,7 @@ public class EmbulkMapReduce
             final PluginArchive archive, final ModelManager modelManager) throws IOException
     {
         final Path path = new Path(stateDir, PLUGIN_ARCHIVE_FILE_NAME);
-        hadoopOperationWithRetry("writing plugin archive to "+path, new Callable<Void>() {
+        hadoopOperationWithRetry("Writing plugin archive to "+path, new Callable<Void>() {
             public Void call() throws IOException
             {
                 stateDir.getFileSystem(config).mkdirs(stateDir);
@@ -264,7 +264,7 @@ public class EmbulkMapReduce
             Path stateDir, final ModelManager modelManager) throws IOException
     {
         final Path path = new Path(stateDir, PLUGIN_ARCHIVE_FILE_NAME);
-        return hadoopOperationWithRetry("reading plugin archive file from "+path, new Callable<PluginArchive>() {
+        return hadoopOperationWithRetry("Reading plugin archive file from "+path, new Callable<PluginArchive>() {
                 public PluginArchive call() throws IOException
                 {
                     List<PluginArchive.GemSpec> specs = modelManager.readObject(
@@ -281,7 +281,7 @@ public class EmbulkMapReduce
             Path stateDir, final AttemptState state, final ModelManager modelManager) throws IOException
     {
         final Path path = new Path(stateDir, state.getAttemptId().toString());
-        hadoopOperationWithRetry("writing attempt state file to "+path, new Callable<Void>() {
+        hadoopOperationWithRetry("Writing attempt state file to "+path, new Callable<Void>() {
             public Void call() throws IOException
             {
                 try (FSDataOutputStream out = path.getFileSystem(config).create(path, true)) {
@@ -326,7 +326,8 @@ public class EmbulkMapReduce
                             //      e) EOFException: file exists but its format is invalid because this task is retried and last job/attempt left corrupted files (such as empty, partially written, etc)
                             //      f) IOException: FileSystem is not working
                             //
-                            if (exception instanceof EOFException && !concurrentWriteIsPossible) {
+                            if (exception instanceof EOFException) {
+                                // a) and b) don't need retrying. See MapReduceExecutor.getAttemptReports that ignores EOFException.
                                 // e) is not recoverable.
                                 return false;
                             }
@@ -337,8 +338,9 @@ public class EmbulkMapReduce
                         public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
                                 throws RetryGiveupException
                         {
-                            log.warn("Retrying opening state file {} ({}/{}) error: {}",
-                                    path, retryCount, retryLimit, exception);
+                            log.warn("Reading a state file failed. Retrying {}/{} after {} seconds. Message: {}",
+                                    retryCount, retryLimit, retryWait, exception.getMessage(),
+                                    retryCount % 3 == 0 ? exception : null);
                         }
 
                         @Override
@@ -384,8 +386,9 @@ public class EmbulkMapReduce
                         public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
                                 throws RetryGiveupException
                         {
-                            log.warn("Retrying {} ({}/{}) error: {}",
-                                    message, retryCount, retryLimit, exception);
+                            log.warn("{} failed. Retrying {}/{} after {} seconds. Message: {}",
+                                    message, retryCount, retryLimit, retryWait, exception.getMessage(),
+                                    retryCount % 3 == 0 ? exception : null);
                         }
 
                         @Override

data/src/main/java/org/embulk/executor/mapreduce/MapReduceExecutor.java

@@ -6,7 +6,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Set;
 import java.util.Map;
-import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.HashMap;
 import java.io.File;
 import java.io.IOException;
@@ -333,7 +333,7 @@ public class MapReduceExecutor
 
     private List<Path> collectJars(List<String> extraJars, List<String> excludeJars)
     {
-        Set<Path> set = new HashSet<Path>();
+        Set<Path> set = new LinkedHashSet<Path>();
 
         collectURLClassLoaderJars(set, Exec.class.getClassLoader());
         collectURLClassLoaderJars(set, MapReduceExecutor.class.getClassLoader());

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-executor-mapreduce
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.2.6
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-02-09 00:00:00.000000000 Z
+date: 2016-04-12 00:00:00.000000000 Z
 dependencies: []
 description: Executes tasks on Hadoop.
 email:
@@ -84,7 +84,7 @@ files:
 - classpath/curator-client-2.6.0.jar
 - classpath/curator-framework-2.6.0.jar
 - classpath/curator-recipes-2.6.0.jar
-- classpath/embulk-executor-mapreduce-0.2.5.jar
+- classpath/embulk-executor-mapreduce-0.2.6.jar
 - classpath/gson-2.2.4.jar
 - classpath/hadoop-annotations-2.6.0.jar
 - classpath/hadoop-auth-2.6.0.jar