embulk-executor-mapreduce 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dc83806412506bc567037cdf24a1c247a99abf13
-  data.tar.gz: 10a3dd696c3729f58a0c4e2b0ec5b0217cbdfcc1
+  metadata.gz: 3f93e6df3aba9f8ca575092d4a3878abe484a452
+  data.tar.gz: d42c206f46d13885733f6e048eb4249b22ed7d29
 SHA512:
-  metadata.gz: f4ef4b1809a3acf01d0cf449efd4bc026fc77e60bd3aba8708102b0118c06a4b5a7bfd06ce70f497d0121da86ddfe9012dc20ab152c0c192ba0dad1eb80065be
-  data.tar.gz: 0999ab7bc7eb9fa1e71e61212c6be680c9e5fa232ea1cd4d57f45a8e7228e41ea4d4ee8c1e794de64b5caf77232264126979560fe09988bb55d264bfa2839e70
+  metadata.gz: 0f250d3cd401e52c149eac498a3549d25240c1f5091cca170aab53d95432a8463d77aacc1c9b33a3a3bd95005970186b511a7c60bbf03a1b0cf8c62ca54b7ab6
+  data.tar.gz: 43f8de24c547fdf4eb004e15a4bee6164401d7a4abefa79a1fa5f62ec7696279b9fe7ae80eab1d25f39104d1331d64c58ade74d19117dc6fa06f421843062eaa

data/src/main/java/org/embulk/executor/mapreduce/EmbulkMapReduce.java

@@ -5,6 +5,8 @@ import java.util.ArrayList;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.io.File;
+import java.io.InputStream;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.EOFException;
 import java.io.InterruptedIOException;
@@ -48,6 +50,7 @@ import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
 import org.embulk.EmbulkService;
 import org.slf4j.Logger;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 public class EmbulkMapReduce
@@ -71,8 +74,10 @@ public class EmbulkMapReduce
     {
         try {
             ModelManager bootstrapModelManager = new ModelManager(null, new ObjectMapper());
-            return new ConfigLoader(bootstrapModelManager).fromJson(
-                    new JsonFactory().createParser(config.get(CK_SYSTEM_CONFIG)));  // TODO add fromJson(String)
+            String json = config.get(CK_SYSTEM_CONFIG);
+            try (InputStream in = new ByteArrayInputStream(json.getBytes(UTF_8))) {
+                return new ConfigLoader(bootstrapModelManager).fromJson(in);
+            }
         } catch (IOException e) {
             throw Throwables.propagate(e);
         }
@@ -324,7 +329,12 @@ public class EmbulkMapReduce
 
     private static <T> T hadoopOperationWithRetry(final String message, final Callable<T> callable) throws IOException
     {
-        final Logger log = Exec.getLogger(EmbulkMapReduce.class);
+        return hadoopOperationWithRetry(Exec.getLogger(EmbulkMapReduce.class), message, callable);
+    }
+
+    private static <T> T hadoopOperationWithRetry(final Logger log,
+            final String message, final Callable<T> callable) throws IOException
+    {
         try {
             return retryExecutor()
                     .withRetryLimit(5)
@@ -379,7 +389,7 @@ public class EmbulkMapReduce
             this.injector = newEmbulkInstance(context.getConfiguration());
             this.modelManager = injector.getInstance(ModelManager.class);
             this.task = getExecutorTask(injector, context.getConfiguration());
-            this.session = new ExecSession(injector, task.getExecConfig());
+            this.session = ExecSession.builder(injector).fromExecConfig(task.getExecConfig()).build();
 
             try {
                 LocalDirAllocator localDirAllocator = new LocalDirAllocator(MRConfig.LOCAL_DIR);
@@ -505,11 +515,18 @@ public class EmbulkMapReduce
         private SessionRunner runner;
 
         @Override
-        public void setup(Context context) throws IOException
+        public void setup(Context context) throws IOException, InterruptedException
         {
             this.context = context;
             this.runner = new SessionRunner(context);
-            runner.readPluginArchive().restoreLoadPathsTo(runner.getScriptingContainer());
+
+            runner.execSession(new ExecAction<Void>() {  // for Exec.getLogger
+                public Void run() throws IOException
+                {
+                    runner.readPluginArchive().restoreLoadPathsTo(runner.getScriptingContainer());
+                    return null;
+                }
+            });
         }
 
         @Override

data/src/main/java/org/embulk/executor/mapreduce/EmbulkPartitioningMapReduce.java

@@ -45,11 +45,18 @@ public class EmbulkPartitioningMapReduce
         private SessionRunner runner;
 
         @Override
-        public void setup(Context context) throws IOException
+        public void setup(Context context) throws IOException, InterruptedException
         {
             this.context = context;
             this.runner = new SessionRunner(context);
-            runner.readPluginArchive().restoreLoadPathsTo(runner.getScriptingContainer());
+
+            runner.execSession(new ExecAction<Void>() {  // for Exec.getLogger
+                public Void run() throws IOException
+                {
+                    runner.readPluginArchive().restoreLoadPathsTo(runner.getScriptingContainer());
+                    return null;
+                }
+            });
         }
 
         @Override

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-executor-mapreduce
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-02 00:00:00.000000000 Z
+date: 2015-07-23 00:00:00.000000000 Z
 dependencies: []
 description: Executes tasks on Hadoop.
 email:
@@ -60,7 +60,7 @@ files:
 - classpath/curator-client-2.6.0.jar
 - classpath/curator-framework-2.6.0.jar
 - classpath/curator-recipes-2.6.0.jar
-- classpath/embulk-executor-mapreduce-0.1.3.jar
+- classpath/embulk-executor-mapreduce-0.1.4.jar
 - classpath/gson-2.2.4.jar
 - classpath/hadoop-annotations-2.6.0.jar
 - classpath/hadoop-auth-2.6.0.jar
@@ -96,10 +96,12 @@ files:
 - classpath/jline-0.9.94.jar
 - classpath/jsr305-1.3.9.jar
 - classpath/leveldbjni-all-1.8.jar
+- classpath/log4j-1.2.17.jar
 - classpath/netty-3.7.0.Final.jar
 - classpath/paranamer-2.3.jar
 - classpath/protobuf-java-2.5.0.jar
 - classpath/servlet-api-2.5.jar
+- classpath/slf4j-log4j12-1.7.5.jar
 - classpath/snappy-java-1.0.4.1.jar
 - classpath/stax-api-1.0-2.jar
 - classpath/xmlenc-0.52.jar