embulk-executor-mapreduce 0.1.4 → 0.1.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/classpath/{embulk-executor-mapreduce-0.1.4.jar → embulk-executor-mapreduce-0.1.5.jar} +0 -0
- data/src/main/java/org/embulk/executor/mapreduce/EmbulkMapReduce.java +4 -4
- data/src/main/java/org/embulk/executor/mapreduce/EmbulkPartitioningMapReduce.java +2 -1
- data/src/main/java/org/embulk/executor/mapreduce/MapReduceExecutor.java +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 995cd8754f48d4d4e0d222bbad7d39dfdad6648f
|
4
|
+
data.tar.gz: 901b6a9298e76ed45ab7840ca4e4135bb68a9d8d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 02cbb013f7b83f51c787e649d1efd0e69b726af56636b2487b5d1b277bb285521db44a5b4b1ee45c59d7e5c0a71515bc9b590cdb5a3f90f5fd871581e4d23286
|
7
|
+
data.tar.gz: f66984fd6801ee826b6c4a46db1c326aa699584497224daed575e11c6f83348f5cbf2ee469c26a856bf4d9479d57e609613abdc7fec69f3f2ccbde8ac612340c
|
Binary file
|
@@ -14,9 +14,7 @@ import java.lang.reflect.InvocationTargetException;
|
|
14
14
|
import com.google.inject.Injector;
|
15
15
|
import com.google.common.base.Optional;
|
16
16
|
import com.google.common.base.Throwables;
|
17
|
-
import com.google.common.base.Throwables;
|
18
17
|
import com.google.common.collect.ImmutableList;
|
19
|
-
import com.fasterxml.jackson.core.JsonFactory;
|
20
18
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
21
19
|
import org.jruby.embed.ScriptingContainer;
|
22
20
|
import org.apache.hadoop.fs.Path;
|
@@ -28,7 +26,6 @@ import org.apache.hadoop.io.IntWritable;
|
|
28
26
|
import org.apache.hadoop.io.NullWritable;
|
29
27
|
import org.apache.hadoop.conf.Configuration;
|
30
28
|
import org.apache.hadoop.mapreduce.Job;
|
31
|
-
import org.apache.hadoop.mapreduce.JobContext;
|
32
29
|
import org.apache.hadoop.mapreduce.Counters;
|
33
30
|
import org.apache.hadoop.mapreduce.TaskAttemptID;
|
34
31
|
import org.apache.hadoop.mapreduce.Mapper;
|
@@ -233,6 +230,9 @@ public class EmbulkMapReduce
|
|
233
230
|
});
|
234
231
|
}
|
235
232
|
|
233
|
+
public static class GemSpecListType extends ArrayList<PluginArchive.GemSpec>
|
234
|
+
{ }
|
235
|
+
|
236
236
|
public static PluginArchive readPluginArchive(final File localDirectory, final Configuration config,
|
237
237
|
Path stateDir, final ModelManager modelManager) throws IOException
|
238
238
|
{
|
@@ -241,7 +241,7 @@ public class EmbulkMapReduce
|
|
241
241
|
public PluginArchive call() throws IOException
|
242
242
|
{
|
243
243
|
List<PluginArchive.GemSpec> specs = modelManager.readObject(
|
244
|
-
|
244
|
+
GemSpecListType.class,
|
245
245
|
config.get(CK_PLUGIN_ARCHIVE_SPECS));
|
246
246
|
try (FSDataInputStream in = path.getFileSystem(config).open(path)) {
|
247
247
|
return PluginArchive.load(localDirectory, specs, in);
|
@@ -156,11 +156,12 @@ public class EmbulkPartitioningMapReduce
|
|
156
156
|
{
|
157
157
|
this.context = context;
|
158
158
|
this.runner = new SessionRunner(context);
|
159
|
-
runner.readPluginArchive().restoreLoadPathsTo(runner.getScriptingContainer());
|
160
159
|
|
161
160
|
runner.execSession(new ExecAction<Void>() {
|
162
161
|
public Void run() throws Exception
|
163
162
|
{
|
163
|
+
runner.readPluginArchive().restoreLoadPathsTo(runner.getScriptingContainer());
|
164
|
+
|
164
165
|
int taskIndex = context.getTaskAttemptID().getTaskID().getId();
|
165
166
|
|
166
167
|
ProcessTask task = runner.getMapReduceExecutorTask().getProcessTask();
|
@@ -374,7 +374,7 @@ public class MapReduceExecutor
|
|
374
374
|
}
|
375
375
|
|
376
376
|
for (AttemptReport report : reportSet.getLatestOutputAttemptReports()) {
|
377
|
-
updateTaskState(state.getOutputTaskState(report.getOutputTaskIndex().get()), report.getAttemptState(),
|
377
|
+
updateTaskState(state.getOutputTaskState(report.getOutputTaskIndex().get()), report.getAttemptState(), false);
|
378
378
|
}
|
379
379
|
}
|
380
380
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-executor-mapreduce
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-07-
|
11
|
+
date: 2015-07-28 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Executes tasks on Hadoop.
|
14
14
|
email:
|
@@ -60,7 +60,7 @@ files:
|
|
60
60
|
- classpath/curator-client-2.6.0.jar
|
61
61
|
- classpath/curator-framework-2.6.0.jar
|
62
62
|
- classpath/curator-recipes-2.6.0.jar
|
63
|
-
- classpath/embulk-executor-mapreduce-0.1.
|
63
|
+
- classpath/embulk-executor-mapreduce-0.1.5.jar
|
64
64
|
- classpath/gson-2.2.4.jar
|
65
65
|
- classpath/hadoop-annotations-2.6.0.jar
|
66
66
|
- classpath/hadoop-auth-2.6.0.jar
|