embulk-executor-mapreduce 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5c531e6955469a01f0e2ed716a65fdf228ae95ba
-  data.tar.gz: d8724a7abcaedd7549a397d2b14df6edb2832d52
+  metadata.gz: 5ac810376cbc4895b8745c82dcd642ad8302bf4f
+  data.tar.gz: 34b4f9a372be43b2915878b35954d75c25f7e8d8
 SHA512:
-  metadata.gz: 9983809cd453596cf3fc683f75a22b43dbcca0abf202c7ad7bdc9cbb640c673e95be706f6d49ad04bb3881f8668bc9b8e3946115d54e2a14e90dfb163fbd9b0f
-  data.tar.gz: 7b272911c78f9bac87fa867f0e3978a96f31c8fb348b150410a6991e1f5f425782450c9422ca4022e2cb15abc61037caaf080fb196f48bd0a441e3e81cc31b15
+  metadata.gz: 96fd33f8766a7510776f8253702c04291e675ea62c6ed05cca05958c905383770afc63d4056df1aaa8bac8be4d0175ba5dd4ada150d86a31b6375e2d1a7c222f
+  data.tar.gz: 01c46e09dd99360ce835ba419819da22d2a711b8a4cebf47382cb19092730e1608f76ae80bf79f498cc363fdbda0f0fd1fc4932a592e76f0b264b7b1e055d7a4

data/src/main/java/org/embulk/executor/mapreduce/MapReduceExecutor.java

@@ -1,6 +1,7 @@
 package org.embulk.executor.mapreduce;
 
 import java.util.List;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Set;
 import java.util.Map;
@@ -9,6 +10,7 @@ import java.util.HashMap;
 import java.io.File;
 import java.io.IOException;
 import java.io.EOFException;
+import java.nio.file.FileSystems;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
@@ -239,7 +241,7 @@ public class MapReduceExecutor
         }
 
         // jar files
-        Iterable<Path> jars = collectJars(task.getLibjars());
+        List<Path> jars = collectJars(task.getLibjars(), task.getExcludeJars());
         job.getConfiguration().set("tmpjars", StringUtils.join(",", jars));
 
         job.setInputFormatClass(EmbulkInputFormat.class);
@@ -304,7 +306,7 @@ public class MapReduceExecutor
         }
     }
 
-    private static Iterable<Path> collectJars(List<String> extraJars)
+    private List<Path> collectJars(List<String> extraJars, List<String> excludeJars)
     {
         Set<Path> set = new HashSet<Path>();
 
@@ -325,7 +327,29 @@ public class MapReduceExecutor
             }
         }
 
-        return set;
+        // validate jar files
+        List<Path> uses = new ArrayList<>(set.size());
+        for (Path path : set) {
+            String fileName = path.getName();
+            if (globMatchesWithAnyOf(excludeJars, fileName)) {
+                log.debug("Excluding jar '"+path+"'");
+            }
+            else {
+                uses.add(path);
+            }
+        }
+
+        return uses;
+    }
+
+    private static boolean globMatchesWithAnyOf(List<String> excludeJars, String fileName)
+    {
+        for (String pattern : excludeJars) {
+            if (FileSystems.getDefault().getPathMatcher("glob:"+pattern).matches(java.nio.file.Paths.get(fileName))) {
+                return true;
+            }
+        }
+        return false;
     }
 
     private static void collectURLClassLoaderJars(Set<Path> set, ClassLoader cl)

data/src/main/java/org/embulk/executor/mapreduce/MapReduceExecutorTask.java

@@ -31,6 +31,10 @@ public interface MapReduceExecutorTask
     @ConfigDefault("[]")
     public List<String> getLibjars();
 
+    @Config("exclude_jars")
+    @ConfigDefault("[]")
+    public List<String> getExcludeJars();
+
     @Config("state_path")
     @ConfigDefault("\"/tmp/embulk\"")
     public String getStatePath();

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-executor-mapreduce
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-08-19 00:00:00.000000000 Z
+date: 2015-10-26 00:00:00.000000000 Z
 dependencies: []
 description: Executes tasks on Hadoop.
 email:
@@ -61,7 +61,7 @@ files:
 - classpath/curator-client-2.6.0.jar
 - classpath/curator-framework-2.6.0.jar
 - classpath/curator-recipes-2.6.0.jar
-- classpath/embulk-executor-mapreduce-0.2.0.jar
+- classpath/embulk-executor-mapreduce-0.2.1.jar
 - classpath/gson-2.2.4.jar
 - classpath/hadoop-annotations-2.6.0.jar
 - classpath/hadoop-auth-2.6.0.jar