embulk-executor-mapreduce 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/classpath/{embulk-executor-mapreduce-0.2.1.jar → embulk-executor-mapreduce-0.2.2.jar} +0 -0
- data/src/main/java/org/embulk/executor/mapreduce/AttemptState.java +18 -3
- data/src/main/java/org/embulk/executor/mapreduce/MapReduceExecutor.java +7 -1
- data/src/main/java/org/embulk/executor/mapreduce/RemoteTaskFailedDataException.java +12 -0
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d683908af5b1f5e06ea035f9ecd4e7a89dbd4eab
|
4
|
+
data.tar.gz: f8a5a7db822f6241a40211dd1782e4fda83df073
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 48b955f8b5a70864dd745347079cc83545fdb6953091812917beeb28e6aa9f66726c68e8b85b0aa3c8da40f419fc0b1efa78aab90bd899e42bb4c1a0c14edaae
|
7
|
+
data.tar.gz: 99199b98f4f56e2d5d74c7f12be41b19bd31aa3f496144a940b16c6c53abf3980c181ae186f40ba3bf307106abb53dc00683c54157ea000652d24d66cff12cd5
|
Binary file
|
@@ -17,6 +17,7 @@ import com.fasterxml.jackson.annotation.JsonValue;
|
|
17
17
|
import org.apache.hadoop.mapreduce.TaskAttemptID;
|
18
18
|
import org.embulk.config.ModelManager;
|
19
19
|
import org.embulk.config.TaskReport;
|
20
|
+
import org.embulk.config.UserDataExceptions;
|
20
21
|
|
21
22
|
public class AttemptState
|
22
23
|
{
|
@@ -24,6 +25,7 @@ public class AttemptState
|
|
24
25
|
private final Optional<Integer> inputTaskIndex;
|
25
26
|
private final Optional<Integer> outputTaskIndex;
|
26
27
|
private Optional<String> exception;
|
28
|
+
private boolean userDataException;
|
27
29
|
private Optional<TaskReport> inputTaskReport;
|
28
30
|
private Optional<TaskReport> outputTaskReport;
|
29
31
|
|
@@ -40,11 +42,13 @@ public class AttemptState
|
|
40
42
|
@JsonProperty("inputTaskIndex") Optional<Integer> inputTaskIndex,
|
41
43
|
@JsonProperty("outputTaskIndex") Optional<Integer> outputTaskIndex,
|
42
44
|
@JsonProperty("exception") Optional<String> exception,
|
45
|
+
@JsonProperty("userDataException") boolean userDataException,
|
43
46
|
@JsonProperty("inputTaskReport") Optional<TaskReport> inputTaskReport,
|
44
47
|
@JsonProperty("outputTaskReport") Optional<TaskReport> outputTaskReport)
|
45
48
|
{
|
46
49
|
this(TaskAttemptID.forName(attemptId),
|
47
|
-
inputTaskIndex, outputTaskIndex,
|
50
|
+
inputTaskIndex, outputTaskIndex,
|
51
|
+
exception, userDataException,
|
48
52
|
inputTaskReport, outputTaskReport);
|
49
53
|
}
|
50
54
|
|
@@ -53,6 +57,7 @@ public class AttemptState
|
|
53
57
|
Optional<Integer> inputTaskIndex,
|
54
58
|
Optional<Integer> outputTaskIndex,
|
55
59
|
Optional<String> exception,
|
60
|
+
boolean userDataException,
|
56
61
|
Optional<TaskReport> inputTaskReport,
|
57
62
|
Optional<TaskReport> outputTaskReport)
|
58
63
|
{
|
@@ -60,6 +65,7 @@ public class AttemptState
|
|
60
65
|
this.inputTaskIndex = inputTaskIndex;
|
61
66
|
this.outputTaskIndex = outputTaskIndex;
|
62
67
|
this.exception = exception;
|
68
|
+
this.userDataException = userDataException;
|
63
69
|
this.inputTaskReport = inputTaskReport;
|
64
70
|
this.outputTaskReport = outputTaskReport;
|
65
71
|
}
|
@@ -97,13 +103,16 @@ public class AttemptState
|
|
97
103
|
} catch (UnsupportedEncodingException ex) {
|
98
104
|
throw new RuntimeException(ex);
|
99
105
|
}
|
100
|
-
setException(
|
106
|
+
setException(
|
107
|
+
new String(os.toByteArray(), StandardCharsets.UTF_8),
|
108
|
+
UserDataExceptions.isUserDataException(exception));
|
101
109
|
}
|
102
110
|
|
103
111
|
@JsonIgnore
|
104
|
-
public void setException(String exception)
|
112
|
+
public void setException(String exception, boolean userDataException)
|
105
113
|
{
|
106
114
|
this.exception = Optional.of(exception);
|
115
|
+
this.userDataException = userDataException;
|
107
116
|
}
|
108
117
|
|
109
118
|
@JsonProperty("exception")
|
@@ -112,6 +121,12 @@ public class AttemptState
|
|
112
121
|
return exception;
|
113
122
|
}
|
114
123
|
|
124
|
+
@JsonProperty("userDataException")
|
125
|
+
public boolean isUserDataException()
|
126
|
+
{
|
127
|
+
return userDataException;
|
128
|
+
}
|
129
|
+
|
115
130
|
@JsonProperty("inputTaskReport")
|
116
131
|
public Optional<TaskReport> getInputTaskReport()
|
117
132
|
{
|
@@ -409,7 +409,13 @@ public class MapReduceExecutor
|
|
409
409
|
boolean committed = taskReport.isPresent();
|
410
410
|
if (attempt.getException().isPresent()) {
|
411
411
|
if (!state.isCommitted()) {
|
412
|
-
|
412
|
+
Exception remoteException;
|
413
|
+
if (attempt.isUserDataException()) {
|
414
|
+
remoteException = new RemoteTaskFailedDataException(attempt.getException().get());
|
415
|
+
} else {
|
416
|
+
remoteException = new RemoteTaskFailedException(attempt.getException().get());
|
417
|
+
}
|
418
|
+
state.setException(remoteException);
|
413
419
|
}
|
414
420
|
}
|
415
421
|
if (taskReport.isPresent()) {
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-executor-mapreduce
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-10-
|
11
|
+
date: 2015-10-28 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Executes tasks on Hadoop.
|
14
14
|
email:
|
@@ -35,6 +35,7 @@ files:
|
|
35
35
|
- src/main/java/org/embulk/executor/mapreduce/Partitioner.java
|
36
36
|
- src/main/java/org/embulk/executor/mapreduce/Partitioning.java
|
37
37
|
- src/main/java/org/embulk/executor/mapreduce/PluginArchive.java
|
38
|
+
- src/main/java/org/embulk/executor/mapreduce/RemoteTaskFailedDataException.java
|
38
39
|
- src/main/java/org/embulk/executor/mapreduce/RemoteTaskFailedException.java
|
39
40
|
- src/main/java/org/embulk/executor/mapreduce/SetContextClassLoader.java
|
40
41
|
- src/main/java/org/embulk/executor/mapreduce/TimestampPartitioning.java
|
@@ -61,7 +62,7 @@ files:
|
|
61
62
|
- classpath/curator-client-2.6.0.jar
|
62
63
|
- classpath/curator-framework-2.6.0.jar
|
63
64
|
- classpath/curator-recipes-2.6.0.jar
|
64
|
-
- classpath/embulk-executor-mapreduce-0.2.
|
65
|
+
- classpath/embulk-executor-mapreduce-0.2.2.jar
|
65
66
|
- classpath/gson-2.2.4.jar
|
66
67
|
- classpath/hadoop-annotations-2.6.0.jar
|
67
68
|
- classpath/hadoop-auth-2.6.0.jar
|