embulk-executor-mapreduce 0.2.7 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/classpath/embulk-executor-mapreduce-0.2.8.jar +0 -0
- data/src/main/java/org/embulk/executor/mapreduce/MapReduceExecutor.java +11 -0
- data/src/main/java/org/embulk/executor/mapreduce/MapReduceExecutorTask.java +7 -0
- metadata +3 -3
- data/classpath/embulk-executor-mapreduce-0.2.7.jar +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3c9be3db8d15de03f4be776254db9befaddf88d8
|
|
4
|
+
data.tar.gz: e766710c37b33da703005b74bded6823c61258d7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 964ca288f42c9b9ba7999aa4566cf5e795ab2d1ab0924de0d4671b87a9782d7b3fc7b3de12011c98aebac622cc514adae5bd9eac1f7fd972c2d32eac29b0548f
|
|
7
|
+
data.tar.gz: a91266af60be1200b3d4e828bab1ca8c13cbae2093cebb2ceb8dce4f26d7f00d9c19011b9b14b09caa319e089e316bc9cc29cf8af7b843f6d0f2ae7989728b94
|
|
Binary file
|
|
@@ -19,6 +19,7 @@ import java.net.URLClassLoader;
|
|
|
19
19
|
import java.net.MalformedURLException;
|
|
20
20
|
|
|
21
21
|
import org.apache.hadoop.security.UserGroupInformation;
|
|
22
|
+
import org.embulk.plugin.PluginType;
|
|
22
23
|
import org.slf4j.Logger;
|
|
23
24
|
import org.joda.time.format.DateTimeFormat;
|
|
24
25
|
import com.google.inject.Inject;
|
|
@@ -78,6 +79,16 @@ public class MapReduceExecutor
|
|
|
78
79
|
ExecutorPlugin.Control control)
|
|
79
80
|
{
|
|
80
81
|
final MapReduceExecutorTask task = config.loadConfig(MapReduceExecutorTask.class);
|
|
82
|
+
|
|
83
|
+
if (inputTaskCount <= task.getLocalModeInputTasks()) {
|
|
84
|
+
// if task count is equal or less than local_mode_input_tasks option, those tasks are executed by local executor
|
|
85
|
+
// TODO if partitioning config is present, they are executed by hadoop local mode
|
|
86
|
+
log.info("Executing tasks on using local threads");
|
|
87
|
+
ExecutorPlugin local = Exec.newPlugin(ExecutorPlugin.class, new PluginType("local"));
|
|
88
|
+
local.transaction(config, outputSchema, inputTaskCount, control);
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
|
|
81
92
|
task.setExecConfig(config);
|
|
82
93
|
|
|
83
94
|
final int outputTaskCount;
|
|
@@ -12,6 +12,8 @@ import org.embulk.config.TaskSource;
|
|
|
12
12
|
import org.embulk.config.ModelManager;
|
|
13
13
|
import org.embulk.spi.ProcessTask;
|
|
14
14
|
|
|
15
|
+
import javax.validation.constraints.Min;
|
|
16
|
+
|
|
15
17
|
public interface MapReduceExecutorTask
|
|
16
18
|
extends Task
|
|
17
19
|
{
|
|
@@ -51,6 +53,11 @@ public interface MapReduceExecutorTask
|
|
|
51
53
|
@ConfigDefault("null")
|
|
52
54
|
public Optional<ConfigSource> getPartitioning();
|
|
53
55
|
|
|
56
|
+
@Config("local_mode_input_tasks")
|
|
57
|
+
@ConfigDefault("0")
|
|
58
|
+
@Min(0)
|
|
59
|
+
public Integer getLocalModeInputTasks();
|
|
60
|
+
|
|
54
61
|
@ConfigInject
|
|
55
62
|
public ModelManager getModelManager();
|
|
56
63
|
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-executor-mapreduce
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.8
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sadayuki Furuhashi
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-
|
|
11
|
+
date: 2016-06-16 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
13
|
description: Executes tasks on Hadoop.
|
|
14
14
|
email:
|
|
@@ -84,7 +84,7 @@ files:
|
|
|
84
84
|
- classpath/curator-client-2.6.0.jar
|
|
85
85
|
- classpath/curator-framework-2.6.0.jar
|
|
86
86
|
- classpath/curator-recipes-2.6.0.jar
|
|
87
|
-
- classpath/embulk-executor-mapreduce-0.2.
|
|
87
|
+
- classpath/embulk-executor-mapreduce-0.2.8.jar
|
|
88
88
|
- classpath/gson-2.2.4.jar
|
|
89
89
|
- classpath/hadoop-annotations-2.6.0.jar
|
|
90
90
|
- classpath/hadoop-auth-2.6.0.jar
|
|
Binary file
|