embulk-executor-mapreduce 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/classpath/{embulk-executor-mapreduce-0.1.4.jar → embulk-executor-mapreduce-0.1.5.jar} +0 -0
- data/src/main/java/org/embulk/executor/mapreduce/EmbulkMapReduce.java +4 -4
- data/src/main/java/org/embulk/executor/mapreduce/EmbulkPartitioningMapReduce.java +2 -1
- data/src/main/java/org/embulk/executor/mapreduce/MapReduceExecutor.java +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 995cd8754f48d4d4e0d222bbad7d39dfdad6648f
|
|
4
|
+
data.tar.gz: 901b6a9298e76ed45ab7840ca4e4135bb68a9d8d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 02cbb013f7b83f51c787e649d1efd0e69b726af56636b2487b5d1b277bb285521db44a5b4b1ee45c59d7e5c0a71515bc9b590cdb5a3f90f5fd871581e4d23286
|
|
7
|
+
data.tar.gz: f66984fd6801ee826b6c4a46db1c326aa699584497224daed575e11c6f83348f5cbf2ee469c26a856bf4d9479d57e609613abdc7fec69f3f2ccbde8ac612340c
|
|
Binary file
|
|
@@ -14,9 +14,7 @@ import java.lang.reflect.InvocationTargetException;
|
|
|
14
14
|
import com.google.inject.Injector;
|
|
15
15
|
import com.google.common.base.Optional;
|
|
16
16
|
import com.google.common.base.Throwables;
|
|
17
|
-
import com.google.common.base.Throwables;
|
|
18
17
|
import com.google.common.collect.ImmutableList;
|
|
19
|
-
import com.fasterxml.jackson.core.JsonFactory;
|
|
20
18
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
21
19
|
import org.jruby.embed.ScriptingContainer;
|
|
22
20
|
import org.apache.hadoop.fs.Path;
|
|
@@ -28,7 +26,6 @@ import org.apache.hadoop.io.IntWritable;
|
|
|
28
26
|
import org.apache.hadoop.io.NullWritable;
|
|
29
27
|
import org.apache.hadoop.conf.Configuration;
|
|
30
28
|
import org.apache.hadoop.mapreduce.Job;
|
|
31
|
-
import org.apache.hadoop.mapreduce.JobContext;
|
|
32
29
|
import org.apache.hadoop.mapreduce.Counters;
|
|
33
30
|
import org.apache.hadoop.mapreduce.TaskAttemptID;
|
|
34
31
|
import org.apache.hadoop.mapreduce.Mapper;
|
|
@@ -233,6 +230,9 @@ public class EmbulkMapReduce
|
|
|
233
230
|
});
|
|
234
231
|
}
|
|
235
232
|
|
|
233
|
+
public static class GemSpecListType extends ArrayList<PluginArchive.GemSpec>
|
|
234
|
+
{ }
|
|
235
|
+
|
|
236
236
|
public static PluginArchive readPluginArchive(final File localDirectory, final Configuration config,
|
|
237
237
|
Path stateDir, final ModelManager modelManager) throws IOException
|
|
238
238
|
{
|
|
@@ -241,7 +241,7 @@ public class EmbulkMapReduce
|
|
|
241
241
|
public PluginArchive call() throws IOException
|
|
242
242
|
{
|
|
243
243
|
List<PluginArchive.GemSpec> specs = modelManager.readObject(
|
|
244
|
-
|
|
244
|
+
GemSpecListType.class,
|
|
245
245
|
config.get(CK_PLUGIN_ARCHIVE_SPECS));
|
|
246
246
|
try (FSDataInputStream in = path.getFileSystem(config).open(path)) {
|
|
247
247
|
return PluginArchive.load(localDirectory, specs, in);
|
|
@@ -156,11 +156,12 @@ public class EmbulkPartitioningMapReduce
|
|
|
156
156
|
{
|
|
157
157
|
this.context = context;
|
|
158
158
|
this.runner = new SessionRunner(context);
|
|
159
|
-
runner.readPluginArchive().restoreLoadPathsTo(runner.getScriptingContainer());
|
|
160
159
|
|
|
161
160
|
runner.execSession(new ExecAction<Void>() {
|
|
162
161
|
public Void run() throws Exception
|
|
163
162
|
{
|
|
163
|
+
runner.readPluginArchive().restoreLoadPathsTo(runner.getScriptingContainer());
|
|
164
|
+
|
|
164
165
|
int taskIndex = context.getTaskAttemptID().getTaskID().getId();
|
|
165
166
|
|
|
166
167
|
ProcessTask task = runner.getMapReduceExecutorTask().getProcessTask();
|
|
@@ -374,7 +374,7 @@ public class MapReduceExecutor
|
|
|
374
374
|
}
|
|
375
375
|
|
|
376
376
|
for (AttemptReport report : reportSet.getLatestOutputAttemptReports()) {
|
|
377
|
-
updateTaskState(state.getOutputTaskState(report.getOutputTaskIndex().get()), report.getAttemptState(),
|
|
377
|
+
updateTaskState(state.getOutputTaskState(report.getOutputTaskIndex().get()), report.getAttemptState(), false);
|
|
378
378
|
}
|
|
379
379
|
}
|
|
380
380
|
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-executor-mapreduce
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.5
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sadayuki Furuhashi
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2015-07-
|
|
11
|
+
date: 2015-07-28 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
13
|
description: Executes tasks on Hadoop.
|
|
14
14
|
email:
|
|
@@ -60,7 +60,7 @@ files:
|
|
|
60
60
|
- classpath/curator-client-2.6.0.jar
|
|
61
61
|
- classpath/curator-framework-2.6.0.jar
|
|
62
62
|
- classpath/curator-recipes-2.6.0.jar
|
|
63
|
-
- classpath/embulk-executor-mapreduce-0.1.
|
|
63
|
+
- classpath/embulk-executor-mapreduce-0.1.5.jar
|
|
64
64
|
- classpath/gson-2.2.4.jar
|
|
65
65
|
- classpath/hadoop-annotations-2.6.0.jar
|
|
66
66
|
- classpath/hadoop-auth-2.6.0.jar
|