embulk-output-multi 0.2.1 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +5 -5
- data/build.gradle +6 -4
- data/gradle.properties +1 -1
- data/gradle/dependency-locks/compileClasspath.lockfile +1 -11
- data/gradle/dependency-locks/testCompileClasspath.lockfile +5 -9
- data/src/main/java/org/embulk/output/multi/AsyncRunControl.java +9 -17
- data/src/main/java/org/embulk/output/multi/MultiOutputPlugin.java +32 -83
- data/src/main/java/org/embulk/output/multi/MultiTransactionalPageOutput.java +137 -0
- data/src/main/java/org/embulk/output/multi/OutputPluginDelegate.java +43 -41
- data/src/main/java/org/embulk/output/multi/TaskReports.java +6 -6
- data/src/test/java/org/embulk/output/multi/TestMultiOutputPlugin.java +3 -3
- metadata +4 -5
- data/src/main/java/org/embulk/output/multi/PluginExecutionException.java +0 -15
- data/src/main/java/org/embulk/output/multi/TransactionalPageOutputDelegate.java +0 -92
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d23fe45b21d16664727849db0d1af9015a180834
|
4
|
+
data.tar.gz: 5b3dbb7047049be9dd3784cafb93855e1288fc36
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0fe2a6a708a3602be1e8512eba3d2ab3d9e4fd789ce833a0f5872f7c9790adc4112463d0ed5f84e45224bff331ef2594c4cff9596288c91be8144e04ce7ad7a9
|
7
|
+
data.tar.gz: 6b66ca734eb9786edd92632ede8678c2152a22b4ffa668215ea12d3ebc9250364582c0399e30ab130ce7ab2ff8de4dd245b34b8b61e2c10ef30ed34656ece096
|
data/README.md
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
# Multi output plugin for Embulk
|
2
2
|
|
3
|
-
This plugin
|
3
|
+
This plugin copies an output to multiple destinations.
|
4
4
|
|
5
5
|
### Notes
|
6
6
|
- It's still very experimental version, so might change its configuration names or styles without notification.
|
7
|
-
-
|
8
|
-
- It might not
|
7
|
+
- As this plugin performs multiple output methods, it might have a performance issue with large records.
|
8
|
+
- It might not work on other executors than LocalExecutor.
|
9
9
|
- I would appreciate it if you use this and give me reports/feedback!
|
10
10
|
|
11
11
|
## Overview
|
@@ -27,13 +27,13 @@ out:
|
|
27
27
|
outputs:
|
28
28
|
# Output to stdout
|
29
29
|
- type: stdout
|
30
|
-
# Output to
|
30
|
+
# Output to files as CSV
|
31
31
|
- type: file
|
32
32
|
path_prefix: out_file_
|
33
33
|
file_ext: csv
|
34
34
|
formatter:
|
35
35
|
type: csv
|
36
|
-
# Output to
|
36
|
+
# Output to files as TSV
|
37
37
|
- type: file
|
38
38
|
path_prefix: out_file_
|
39
39
|
file_ext: tsv
|
data/build.gradle
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
plugins {
|
2
|
-
id "com.github.kamatama41.embulk" version "0.
|
2
|
+
id "com.github.kamatama41.embulk" version "0.5.2"
|
3
3
|
id "net.researchgate.release" version "2.8.0"
|
4
4
|
}
|
5
5
|
|
@@ -17,8 +17,10 @@ configurations {
|
|
17
17
|
}
|
18
18
|
|
19
19
|
dependencies {
|
20
|
-
testImplementation "org.embulk:embulk-
|
21
|
-
testImplementation "org.embulk:embulk-
|
20
|
+
testImplementation "org.embulk:embulk-deps-buffer:0.9.23"
|
21
|
+
testImplementation "org.embulk:embulk-deps-config:0.9.23"
|
22
|
+
testImplementation "org.embulk:embulk-standards:0.9.23"
|
23
|
+
testImplementation "org.embulk:embulk-test:0.9.23"
|
22
24
|
testImplementation "com.github.kamatama41:embulk-test-helpers:0.7.4"
|
23
25
|
testImplementation "org.junit.jupiter:junit-jupiter-api:5.3.2"
|
24
26
|
testImplementation "org.junit.jupiter:junit-jupiter-engine:5.3.2"
|
@@ -29,7 +31,7 @@ test {
|
|
29
31
|
}
|
30
32
|
|
31
33
|
embulk {
|
32
|
-
version = "0.9.
|
34
|
+
version = "0.9.23"
|
33
35
|
category = "output"
|
34
36
|
name = "multi"
|
35
37
|
authors = ["Shinichi ISHIMURA"]
|
data/gradle.properties
CHANGED
@@ -1 +1 @@
|
|
1
|
-
version=0.
|
1
|
+
version=0.5.0
|
@@ -9,27 +9,17 @@ com.fasterxml.jackson.datatype:jackson-datatype-guava:2.6.7
|
|
9
9
|
com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
|
10
10
|
com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
|
11
11
|
com.fasterxml.jackson.module:jackson-module-guice:2.6.7
|
12
|
-
com.google.code.findbugs:annotations:3.0.0
|
13
12
|
com.google.guava:guava:18.0
|
14
13
|
com.google.inject.extensions:guice-multibindings:4.0
|
15
14
|
com.google.inject:guice:4.0
|
16
|
-
com.ibm.icu:icu4j:54.1.1
|
17
15
|
commons-beanutils:commons-beanutils-core:1.8.3
|
18
|
-
commons-cli:commons-cli:1.3.1
|
19
|
-
commons-collections:commons-collections:3.2.1
|
20
|
-
commons-lang:commons-lang:2.4
|
21
|
-
io.airlift:slice:0.9
|
22
|
-
io.netty:netty-buffer:4.0.44.Final
|
23
|
-
io.netty:netty-common:4.0.44.Final
|
24
16
|
javax.inject:javax.inject:1
|
25
17
|
javax.validation:validation-api:1.1.0.Final
|
26
18
|
joda-time:joda-time:2.9.2
|
27
19
|
org.apache.bval:bval-core:0.5
|
28
20
|
org.apache.bval:bval-jsr303:0.5
|
29
21
|
org.apache.commons:commons-lang3:3.4
|
30
|
-
org.
|
31
|
-
org.embulk:embulk-core:0.9.15
|
22
|
+
org.embulk:embulk-core:0.9.23
|
32
23
|
org.jruby:jruby-complete:9.1.15.0
|
33
24
|
org.msgpack:msgpack-core:0.8.11
|
34
25
|
org.slf4j:slf4j-api:1.7.12
|
35
|
-
org.yaml:snakeyaml:1.18
|
@@ -10,15 +10,10 @@ com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
|
|
10
10
|
com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
|
11
11
|
com.fasterxml.jackson.module:jackson-module-guice:2.6.7
|
12
12
|
com.github.kamatama41:embulk-test-helpers:0.7.4
|
13
|
-
com.google.code.findbugs:annotations:3.0.0
|
14
13
|
com.google.guava:guava:18.0
|
15
14
|
com.google.inject.extensions:guice-multibindings:4.0
|
16
15
|
com.google.inject:guice:4.0
|
17
|
-
com.ibm.icu:icu4j:54.1.1
|
18
16
|
commons-beanutils:commons-beanutils-core:1.8.3
|
19
|
-
commons-cli:commons-cli:1.3.1
|
20
|
-
commons-collections:commons-collections:3.2.1
|
21
|
-
commons-lang:commons-lang:2.4
|
22
17
|
io.airlift:slice:0.9
|
23
18
|
io.netty:netty-buffer:4.0.44.Final
|
24
19
|
io.netty:netty-common:4.0.44.Final
|
@@ -30,11 +25,12 @@ org.apache.bval:bval-core:0.5
|
|
30
25
|
org.apache.bval:bval-jsr303:0.5
|
31
26
|
org.apache.commons:commons-compress:1.10
|
32
27
|
org.apache.commons:commons-lang3:3.4
|
33
|
-
org.apache.velocity:velocity:1.7
|
34
28
|
org.apiguardian:apiguardian-api:1.0.0
|
35
|
-
org.embulk:embulk-core:0.9.
|
36
|
-
org.embulk:embulk-
|
37
|
-
org.embulk:embulk-
|
29
|
+
org.embulk:embulk-core:0.9.23
|
30
|
+
org.embulk:embulk-deps-buffer:0.9.23
|
31
|
+
org.embulk:embulk-deps-config:0.9.23
|
32
|
+
org.embulk:embulk-standards:0.9.23
|
33
|
+
org.embulk:embulk-test:0.9.23
|
38
34
|
org.hamcrest:hamcrest-core:1.3
|
39
35
|
org.hamcrest:hamcrest-library:1.3
|
40
36
|
org.jruby:jruby-complete:9.1.15.0
|
@@ -5,45 +5,46 @@ import org.embulk.config.TaskReport;
|
|
5
5
|
import org.embulk.config.TaskSource;
|
6
6
|
import org.embulk.spi.OutputPlugin;
|
7
7
|
|
8
|
-
import java.util.ArrayList;
|
9
8
|
import java.util.List;
|
10
9
|
import java.util.concurrent.Callable;
|
10
|
+
import java.util.concurrent.ConcurrentHashMap;
|
11
|
+
import java.util.concurrent.ConcurrentMap;
|
11
12
|
import java.util.concurrent.CountDownLatch;
|
12
13
|
import java.util.concurrent.ExecutionException;
|
13
14
|
import java.util.concurrent.ExecutorService;
|
14
15
|
import java.util.concurrent.Executors;
|
15
16
|
import java.util.concurrent.Future;
|
16
|
-
import java.util.concurrent.atomic.AtomicReferenceArray;
|
17
17
|
|
18
18
|
class AsyncRunControl {
|
19
19
|
private static final String THREAD_NAME_FORMAT = "multi-run-control-%d";
|
20
20
|
private final MultiOutputPlugin.PluginTask task;
|
21
21
|
private final OutputPlugin.Control control;
|
22
22
|
private final CountDownLatch latch;
|
23
|
-
private final
|
23
|
+
private final ConcurrentMap<String, TaskSource> taskSources;
|
24
24
|
private final ExecutorService executorService;
|
25
|
-
private Future<List<TaskReport>> result;
|
25
|
+
private final Future<List<TaskReport>> result;
|
26
26
|
|
27
27
|
static AsyncRunControl start(MultiOutputPlugin.PluginTask task, OutputPlugin.Control control) {
|
28
|
-
return new AsyncRunControl(task, control)
|
28
|
+
return new AsyncRunControl(task, control);
|
29
29
|
}
|
30
30
|
|
31
31
|
private AsyncRunControl(MultiOutputPlugin.PluginTask task, OutputPlugin.Control control) {
|
32
32
|
this.task = task;
|
33
33
|
this.control = control;
|
34
34
|
this.latch = new CountDownLatch(task.getOutputConfigs().size());
|
35
|
-
this.taskSources = new
|
35
|
+
this.taskSources = new ConcurrentHashMap<>(task.getOutputConfigs().size());
|
36
36
|
this.executorService = Executors.newSingleThreadExecutor(
|
37
37
|
new ThreadFactoryBuilder().setNameFormat(THREAD_NAME_FORMAT).build()
|
38
38
|
);
|
39
|
+
this.result = executorService.submit(new RunControl());
|
39
40
|
}
|
40
41
|
|
41
42
|
void cancel() {
|
42
43
|
result.cancel(true);
|
43
44
|
}
|
44
45
|
|
45
|
-
void addTaskSource(
|
46
|
-
taskSources.
|
46
|
+
void addTaskSource(String tag, TaskSource taskSource) {
|
47
|
+
taskSources.putIfAbsent(tag, taskSource);
|
47
48
|
latch.countDown();
|
48
49
|
}
|
49
50
|
|
@@ -55,19 +56,10 @@ class AsyncRunControl {
|
|
55
56
|
}
|
56
57
|
}
|
57
58
|
|
58
|
-
private AsyncRunControl start() {
|
59
|
-
this.result = executorService.submit(new RunControl());
|
60
|
-
return this;
|
61
|
-
}
|
62
|
-
|
63
59
|
private class RunControl implements Callable<List<TaskReport>> {
|
64
60
|
@Override
|
65
61
|
public List<TaskReport> call() throws Exception {
|
66
62
|
latch.await();
|
67
|
-
List<TaskSource> taskSources = new ArrayList<>(AsyncRunControl.this.taskSources.length());
|
68
|
-
for (int i = 0; i < AsyncRunControl.this.taskSources.length(); i++) {
|
69
|
-
taskSources.add(AsyncRunControl.this.taskSources.get(i));
|
70
|
-
}
|
71
63
|
task.setTaskSources(taskSources);
|
72
64
|
return control.run(task.dump());
|
73
65
|
}
|
@@ -9,21 +9,21 @@ import org.embulk.config.Task;
|
|
9
9
|
import org.embulk.config.TaskReport;
|
10
10
|
import org.embulk.config.TaskSource;
|
11
11
|
import org.embulk.plugin.PluginType;
|
12
|
-
import org.embulk.spi.Buffer;
|
13
12
|
import org.embulk.spi.Exec;
|
14
13
|
import org.embulk.spi.ExecSession;
|
15
14
|
import org.embulk.spi.OutputPlugin;
|
16
|
-
import org.embulk.spi.Page;
|
17
15
|
import org.embulk.spi.Schema;
|
18
16
|
import org.embulk.spi.TransactionalPageOutput;
|
17
|
+
import org.slf4j.Logger;
|
18
|
+
import org.slf4j.LoggerFactory;
|
19
19
|
|
20
20
|
import java.util.ArrayList;
|
21
|
+
import java.util.HashMap;
|
21
22
|
import java.util.List;
|
23
|
+
import java.util.Map;
|
22
24
|
import java.util.Optional;
|
23
25
|
import java.util.concurrent.ExecutionException;
|
24
|
-
import java.util.concurrent.Future;
|
25
26
|
import java.util.function.Function;
|
26
|
-
import java.util.stream.Collectors;
|
27
27
|
|
28
28
|
public class MultiOutputPlugin implements OutputPlugin {
|
29
29
|
public interface PluginTask extends Task {
|
@@ -32,12 +32,13 @@ public class MultiOutputPlugin implements OutputPlugin {
|
|
32
32
|
|
33
33
|
@Config(CONFIG_NAME_OUTPUT_CONFIG_DIFFS)
|
34
34
|
@ConfigDefault("null")
|
35
|
-
Optional<
|
35
|
+
Optional<Map<String, ConfigDiff>> getOutputConfigDiffs();
|
36
36
|
|
37
|
-
|
38
|
-
void setTaskSources(
|
37
|
+
Map<String, TaskSource> getTaskSources();
|
38
|
+
void setTaskSources(Map<String, TaskSource> taskSources);
|
39
39
|
}
|
40
40
|
|
41
|
+
private static final Logger LOGGER = LoggerFactory.getLogger(MultiOutputPlugin.class);
|
41
42
|
private static final String CONFIG_NAME_OUTPUT_CONFIG_DIFFS = "output_config_diffs";
|
42
43
|
static final String CONFIG_NAME_OUTPUT_TASK_REPORTS = "output_task_reports";
|
43
44
|
|
@@ -45,7 +46,7 @@ public class MultiOutputPlugin implements OutputPlugin {
|
|
45
46
|
public ConfigDiff transaction(ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control) {
|
46
47
|
final PluginTask task = config.loadConfig(PluginTask.class);
|
47
48
|
if (task.getOutputConfigs().isEmpty()) {
|
48
|
-
throw new ConfigException("'outputs' must have more than
|
49
|
+
throw new ConfigException("'outputs' must have more than or equals to 1 element.");
|
49
50
|
}
|
50
51
|
final ExecSession session = Exec.session();
|
51
52
|
final AsyncRunControl runControl = AsyncRunControl.start(task, control);
|
@@ -78,78 +79,15 @@ public class MultiOutputPlugin implements OutputPlugin {
|
|
78
79
|
public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) {
|
79
80
|
final PluginTask task = taskSource.loadTask(PluginTask.class);
|
80
81
|
final ExecSession session = Exec.session();
|
81
|
-
|
82
|
-
new TransactionalPageOutputDelegate(taskIndex, delegate, delegate.open(schema, taskIndex))
|
83
|
-
);
|
84
|
-
|
85
|
-
return new TransactionalPageOutput() {
|
86
|
-
@Override
|
87
|
-
public void add(Page original) {
|
88
|
-
final Buffer originalBuffer = original.buffer();
|
89
|
-
for (TransactionalPageOutputDelegate output : delegates) {
|
90
|
-
final Buffer copiedBuffer = Buffer.wrap(originalBuffer.array());
|
91
|
-
copiedBuffer.offset(originalBuffer.offset());
|
92
|
-
copiedBuffer.limit(originalBuffer.limit());
|
93
|
-
|
94
|
-
final Page copiedPage = Page.wrap(copiedBuffer);
|
95
|
-
copiedPage.setStringReferences(new ArrayList<>(original.getStringReferences()));
|
96
|
-
copiedPage.setValueReferences(new ArrayList<>(original.getValueReferences()));
|
97
|
-
|
98
|
-
output.add(copiedPage);
|
99
|
-
}
|
100
|
-
}
|
101
|
-
|
102
|
-
@Override
|
103
|
-
public void finish() {
|
104
|
-
for (TransactionalPageOutputDelegate output : delegates) {
|
105
|
-
output.finish();
|
106
|
-
}
|
107
|
-
}
|
108
|
-
|
109
|
-
@Override
|
110
|
-
public void close() {
|
111
|
-
for (TransactionalPageOutputDelegate output : delegates) {
|
112
|
-
output.close();
|
113
|
-
}
|
114
|
-
}
|
115
|
-
|
116
|
-
@Override
|
117
|
-
public void abort() {
|
118
|
-
for (TransactionalPageOutputDelegate output : delegates) {
|
119
|
-
output.abort();
|
120
|
-
}
|
121
|
-
}
|
122
|
-
|
123
|
-
@Override
|
124
|
-
public TaskReport commit() {
|
125
|
-
final TaskReport report = Exec.newTaskReport();
|
126
|
-
final List<TaskReport> reports = new ArrayList<>();
|
127
|
-
final List<OutputPluginDelegate> errorPlugins = new ArrayList<>();
|
128
|
-
for (TransactionalPageOutputDelegate output : delegates) {
|
129
|
-
try {
|
130
|
-
reports.add(output.commit());
|
131
|
-
} catch (PluginExecutionException e) {
|
132
|
-
errorPlugins.add(e.getPlugin());
|
133
|
-
}
|
134
|
-
}
|
135
|
-
if (!errorPlugins.isEmpty()) {
|
136
|
-
throw new RuntimeException(
|
137
|
-
String.format("Following plugins failed to output [%s]",
|
138
|
-
errorPlugins.stream().map(OutputPluginDelegate::getPluginCode).collect(Collectors.joining(", "))
|
139
|
-
));
|
140
|
-
}
|
141
|
-
report.set(CONFIG_NAME_OUTPUT_TASK_REPORTS, new TaskReports(reports));
|
142
|
-
return report;
|
143
|
-
}
|
144
|
-
};
|
82
|
+
return MultiTransactionalPageOutput.open(schema, taskIndex, mapWithPluginDelegate(task, session, Function.identity()));
|
145
83
|
}
|
146
84
|
|
147
|
-
private static ConfigDiff buildConfigDiff(List<
|
85
|
+
private static ConfigDiff buildConfigDiff(List<OutputPluginDelegate.Transaction> transactions) {
|
148
86
|
final ConfigDiff configDiff = Exec.newConfigDiff();
|
149
|
-
|
150
|
-
for (
|
87
|
+
Map<String, ConfigDiff> configDiffs = new HashMap<>();
|
88
|
+
for (OutputPluginDelegate.Transaction transaction: transactions) {
|
151
89
|
try {
|
152
|
-
configDiffs.
|
90
|
+
configDiffs.put(transaction.getTag(), transaction.getResult());
|
153
91
|
} catch (InterruptedException e) {
|
154
92
|
Thread.currentThread().interrupt();
|
155
93
|
throw new RuntimeException(e);
|
@@ -163,18 +101,29 @@ public class MultiOutputPlugin implements OutputPlugin {
|
|
163
101
|
|
164
102
|
private <T> List<T> mapWithPluginDelegate(PluginTask task, ExecSession session, Function<OutputPluginDelegate, T> action) {
|
165
103
|
List<T> result = new ArrayList<>();
|
166
|
-
for (int
|
167
|
-
final ConfigSource config = task.getOutputConfigs().get(
|
168
|
-
if (task.getOutputConfigDiffs().isPresent()) {
|
169
|
-
config.merge(task.getOutputConfigDiffs().get().get(pluginIndex));
|
170
|
-
}
|
104
|
+
for (int i = 0; i < task.getOutputConfigs().size(); i++) {
|
105
|
+
final ConfigSource config = task.getOutputConfigs().get(i);
|
171
106
|
final PluginType pluginType = config.get(PluginType.class, "type");
|
172
107
|
final OutputPlugin outputPlugin = session.newPlugin(OutputPlugin.class, pluginType);
|
108
|
+
|
109
|
+
final String tag = String.format("%s_%d", pluginType.getName(), i);
|
110
|
+
|
111
|
+
// Merge ConfigDiff if exists
|
112
|
+
if (task.getOutputConfigDiffs().isPresent()) {
|
113
|
+
final ConfigDiff configDiff = task.getOutputConfigDiffs().get().get(tag);
|
114
|
+
if (configDiff != null) {
|
115
|
+
config.merge(configDiff);
|
116
|
+
} else {
|
117
|
+
LOGGER.debug("ConfigDiff for '{}' not found.", tag);
|
118
|
+
}
|
119
|
+
}
|
120
|
+
// Set TaskSource if exists
|
173
121
|
TaskSource taskSource = null;
|
174
122
|
if (task.getTaskSources() != null) {
|
175
|
-
taskSource = task.getTaskSources().get(
|
123
|
+
taskSource = task.getTaskSources().get(tag);
|
176
124
|
}
|
177
|
-
|
125
|
+
|
126
|
+
result.add(action.apply(new OutputPluginDelegate(tag, outputPlugin, config, taskSource)));
|
178
127
|
}
|
179
128
|
return result;
|
180
129
|
}
|
@@ -0,0 +1,137 @@
|
|
1
|
+
package org.embulk.output.multi;
|
2
|
+
|
3
|
+
import org.embulk.config.TaskReport;
|
4
|
+
import org.embulk.spi.Buffer;
|
5
|
+
import org.embulk.spi.Exec;
|
6
|
+
import org.embulk.spi.Page;
|
7
|
+
import org.embulk.spi.Schema;
|
8
|
+
import org.embulk.spi.TransactionalPageOutput;
|
9
|
+
import org.slf4j.Logger;
|
10
|
+
import org.slf4j.LoggerFactory;
|
11
|
+
|
12
|
+
import java.util.ArrayList;
|
13
|
+
import java.util.HashMap;
|
14
|
+
import java.util.List;
|
15
|
+
import java.util.Map;
|
16
|
+
import java.util.function.Consumer;
|
17
|
+
import java.util.stream.Collectors;
|
18
|
+
|
19
|
+
public class MultiTransactionalPageOutput implements TransactionalPageOutput {
|
20
|
+
private static final Logger LOGGER = LoggerFactory.getLogger(MultiTransactionalPageOutput.class);
|
21
|
+
private final int taskIndex;
|
22
|
+
private final List<Delegate> delegates;
|
23
|
+
|
24
|
+
static MultiTransactionalPageOutput open(Schema schema, int taskIndex, List<OutputPluginDelegate> plugins) {
|
25
|
+
return new MultiTransactionalPageOutput(
|
26
|
+
taskIndex,
|
27
|
+
plugins.stream()
|
28
|
+
.map(plugin -> new Delegate(plugin, plugin.open(schema, taskIndex)))
|
29
|
+
.collect(Collectors.toList())
|
30
|
+
);
|
31
|
+
}
|
32
|
+
|
33
|
+
private MultiTransactionalPageOutput(int taskIndex, List<Delegate> delegates) {
|
34
|
+
this.taskIndex = taskIndex;
|
35
|
+
this.delegates = delegates;
|
36
|
+
}
|
37
|
+
|
38
|
+
@Override
|
39
|
+
public void add(Page page) {
|
40
|
+
applyToAllPlugins(delegate -> delegate.add(copyPage(page)));
|
41
|
+
page.release();
|
42
|
+
}
|
43
|
+
|
44
|
+
@Override
|
45
|
+
public void finish() {
|
46
|
+
applyToAllPlugins(Delegate::finish);
|
47
|
+
}
|
48
|
+
|
49
|
+
@Override
|
50
|
+
public void close() {
|
51
|
+
applyToAllPlugins(Delegate::close);
|
52
|
+
}
|
53
|
+
|
54
|
+
@Override
|
55
|
+
public void abort() {
|
56
|
+
applyToAllPlugins(Delegate::abort);
|
57
|
+
}
|
58
|
+
|
59
|
+
@Override
|
60
|
+
public TaskReport commit() {
|
61
|
+
final Map<String, TaskReport> reports = new HashMap<>();
|
62
|
+
applyToAllPlugins(delegate -> reports.put(delegate.getTag(), delegate.commit()));
|
63
|
+
final TaskReport report = Exec.newTaskReport();
|
64
|
+
report.set(MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS, new TaskReports(reports));
|
65
|
+
return report;
|
66
|
+
}
|
67
|
+
|
68
|
+
private void applyToAllPlugins(Consumer<Delegate> command) {
|
69
|
+
final List<OutputPluginDelegate> errorPlugins = new ArrayList<>();
|
70
|
+
for (Delegate delegate : delegates) {
|
71
|
+
try {
|
72
|
+
command.accept(delegate);
|
73
|
+
} catch (Exception e) {
|
74
|
+
LOGGER.warn(String.format("Output for %s on index %d failed.", delegate.plugin.getTag(), taskIndex), e);
|
75
|
+
errorPlugins.add(delegate.plugin);
|
76
|
+
}
|
77
|
+
}
|
78
|
+
|
79
|
+
if (!errorPlugins.isEmpty()) {
|
80
|
+
throw new RuntimeException(
|
81
|
+
String.format("Following plugins failed to output [%s] on index %d",
|
82
|
+
errorPlugins.stream().map(OutputPluginDelegate::getTag).collect(Collectors.joining(", ")),
|
83
|
+
taskIndex
|
84
|
+
));
|
85
|
+
}
|
86
|
+
}
|
87
|
+
|
88
|
+
private static Page copyPage(Page original) {
|
89
|
+
final Buffer originalBuffer = original.buffer();
|
90
|
+
final Buffer copiedBuffer = Buffer.wrap(originalBuffer.array(), originalBuffer.offset(), originalBuffer.capacity());
|
91
|
+
copiedBuffer.limit(originalBuffer.limit());
|
92
|
+
|
93
|
+
final Page copiedPage = Page.wrap(copiedBuffer);
|
94
|
+
copiedPage.setStringReferences(new ArrayList<>(original.getStringReferences()));
|
95
|
+
copiedPage.setValueReferences(new ArrayList<>(original.getValueReferences()));
|
96
|
+
return copiedPage;
|
97
|
+
}
|
98
|
+
|
99
|
+
static class Delegate implements TransactionalPageOutput {
|
100
|
+
private final OutputPluginDelegate plugin;
|
101
|
+
private final TransactionalPageOutput output;
|
102
|
+
|
103
|
+
private Delegate(OutputPluginDelegate plugin, TransactionalPageOutput output) {
|
104
|
+
this.plugin = plugin;
|
105
|
+
this.output = output;
|
106
|
+
}
|
107
|
+
|
108
|
+
@Override
|
109
|
+
public void add(Page page) {
|
110
|
+
output.add(page);
|
111
|
+
}
|
112
|
+
|
113
|
+
@Override
|
114
|
+
public void finish() {
|
115
|
+
output.finish();
|
116
|
+
}
|
117
|
+
|
118
|
+
@Override
|
119
|
+
public void close() {
|
120
|
+
output.close();
|
121
|
+
}
|
122
|
+
|
123
|
+
@Override
|
124
|
+
public void abort() {
|
125
|
+
output.abort();
|
126
|
+
}
|
127
|
+
|
128
|
+
@Override
|
129
|
+
public TaskReport commit() {
|
130
|
+
return output.commit();
|
131
|
+
}
|
132
|
+
|
133
|
+
String getTag() {
|
134
|
+
return plugin.getTag();
|
135
|
+
}
|
136
|
+
}
|
137
|
+
}
|
@@ -5,7 +5,6 @@ import org.embulk.config.ConfigDiff;
|
|
5
5
|
import org.embulk.config.ConfigSource;
|
6
6
|
import org.embulk.config.TaskReport;
|
7
7
|
import org.embulk.config.TaskSource;
|
8
|
-
import org.embulk.plugin.PluginType;
|
9
8
|
import org.embulk.spi.OutputPlugin;
|
10
9
|
import org.embulk.spi.Schema;
|
11
10
|
import org.embulk.spi.TransactionalPageOutput;
|
@@ -23,105 +22,92 @@ import java.util.concurrent.Future;
|
|
23
22
|
class OutputPluginDelegate {
|
24
23
|
private static final Logger LOGGER = LoggerFactory.getLogger(OutputPluginDelegate.class);
|
25
24
|
private static final String THREAD_NAME_FORMAT = "multi-output-plugin-%s-%%d";
|
26
|
-
private final
|
27
|
-
private final PluginType type;
|
25
|
+
private final String tag;
|
28
26
|
private final OutputPlugin plugin;
|
29
27
|
private final ConfigSource config;
|
30
28
|
private final TaskSource taskSource;
|
31
29
|
private final ExecutorService executorService;
|
32
30
|
|
33
31
|
OutputPluginDelegate(
|
34
|
-
|
35
|
-
PluginType type,
|
32
|
+
String tag,
|
36
33
|
OutputPlugin plugin,
|
37
34
|
ConfigSource config,
|
38
35
|
TaskSource taskSource
|
39
36
|
) {
|
40
|
-
this.
|
41
|
-
this.type = type;
|
37
|
+
this.tag = tag;
|
42
38
|
this.plugin = plugin;
|
43
39
|
this.config = config;
|
44
40
|
this.taskSource = taskSource;
|
45
41
|
this.executorService = Executors.newSingleThreadExecutor(
|
46
|
-
new ThreadFactoryBuilder().setNameFormat(String.format(THREAD_NAME_FORMAT,
|
42
|
+
new ThreadFactoryBuilder().setNameFormat(String.format(THREAD_NAME_FORMAT, tag)).build()
|
47
43
|
);
|
48
44
|
}
|
49
45
|
|
50
|
-
|
51
|
-
return executorService.submit(() -> {
|
46
|
+
Transaction transaction(Schema schema, int taskCount, AsyncRunControl runControl) {
|
47
|
+
return new Transaction(executorService.submit(() -> {
|
52
48
|
try {
|
53
|
-
LOGGER.debug("Run #transaction for {}",
|
54
|
-
return plugin.transaction(config, schema, taskCount, new Control(
|
49
|
+
LOGGER.debug("Run #transaction for {}", getTag());
|
50
|
+
return plugin.transaction(config, schema, taskCount, new Control(runControl));
|
55
51
|
} catch (CancellationException e) {
|
56
|
-
LOGGER.error("Canceled #transaction for {} by other plugin's error",
|
52
|
+
LOGGER.error("Canceled #transaction for {} by other plugin's error", getTag());
|
57
53
|
throw e;
|
58
54
|
} catch (Exception e) {
|
59
|
-
LOGGER.error("Transaction for {} failed.",
|
55
|
+
LOGGER.error("Transaction for {} failed.", getTag(), e);
|
60
56
|
runControl.cancel();
|
61
57
|
throw e;
|
62
58
|
} finally {
|
63
59
|
executorService.shutdown();
|
64
60
|
}
|
65
|
-
});
|
61
|
+
}));
|
66
62
|
}
|
67
63
|
|
68
|
-
|
69
|
-
return executorService.submit(() -> {
|
64
|
+
Transaction resume(Schema schema, int taskCount, AsyncRunControl runControl) {
|
65
|
+
return new Transaction(executorService.submit(() -> {
|
70
66
|
try {
|
71
|
-
LOGGER.debug("Run #resume for {}",
|
72
|
-
return plugin.resume(taskSource, schema, taskCount, new Control(
|
67
|
+
LOGGER.debug("Run #resume for {}", getTag());
|
68
|
+
return plugin.resume(taskSource, schema, taskCount, new Control(runControl));
|
73
69
|
} catch (CancellationException e) {
|
74
|
-
LOGGER.error("Canceled #resume for {} by other plugin's error",
|
70
|
+
LOGGER.error("Canceled #resume for {} by other plugin's error", getTag());
|
75
71
|
throw e;
|
76
72
|
} catch (Exception e) {
|
77
|
-
LOGGER.error("Resume for {} failed.",
|
73
|
+
LOGGER.error("Resume for {} failed.", getTag(), e);
|
78
74
|
runControl.cancel();
|
79
75
|
throw e;
|
80
76
|
} finally {
|
81
77
|
executorService.shutdown();
|
82
78
|
}
|
83
|
-
});
|
79
|
+
}));
|
84
80
|
}
|
85
81
|
|
86
82
|
void cleanup(Schema schema, int taskCount, List<TaskReport> successTaskReports) {
|
87
|
-
LOGGER.debug("Run #cleanup for {}",
|
83
|
+
LOGGER.debug("Run #cleanup for {}", getTag());
|
88
84
|
List<TaskReport> successReportsForPlugin = new ArrayList<>();
|
89
85
|
for (TaskReport successTaskReport : successTaskReports) {
|
90
|
-
final TaskReport report = successTaskReport.get(TaskReports.class, MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS).get(
|
86
|
+
final TaskReport report = successTaskReport.get(TaskReports.class, MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS).get(tag);
|
91
87
|
successReportsForPlugin.add(report);
|
92
88
|
}
|
93
89
|
plugin.cleanup(taskSource, schema, taskCount, successReportsForPlugin);
|
94
90
|
}
|
95
91
|
|
96
92
|
TransactionalPageOutput open(Schema schema, int taskIndex) {
|
97
|
-
LOGGER.debug("Run #open for {}",
|
93
|
+
LOGGER.debug("Run #open for {}", getTag());
|
98
94
|
return plugin.open(taskSource, schema, taskIndex);
|
99
95
|
}
|
100
96
|
|
101
|
-
|
102
|
-
return
|
97
|
+
String getTag() {
|
98
|
+
return tag;
|
103
99
|
}
|
104
100
|
|
105
|
-
|
106
|
-
return generatePluginCode(type, pluginIndex);
|
107
|
-
}
|
108
|
-
|
109
|
-
private static String generatePluginCode(PluginType type, int pluginIndex) {
|
110
|
-
return String.format("%s-%d", type.getName(), pluginIndex);
|
111
|
-
}
|
112
|
-
|
113
|
-
private static class Control implements OutputPlugin.Control {
|
114
|
-
private final int pluginIndex;
|
101
|
+
private class Control implements OutputPlugin.Control {
|
115
102
|
private final AsyncRunControl runControl;
|
116
103
|
|
117
|
-
Control(
|
118
|
-
this.pluginIndex = index;
|
104
|
+
Control(AsyncRunControl runControl) {
|
119
105
|
this.runControl = runControl;
|
120
106
|
}
|
121
107
|
|
122
108
|
@Override
|
123
109
|
public List<TaskReport> run(TaskSource taskSource) {
|
124
|
-
runControl.addTaskSource(
|
110
|
+
runControl.addTaskSource(tag, taskSource);
|
125
111
|
List<TaskReport> reports;
|
126
112
|
try {
|
127
113
|
reports = runControl.waitAndGetResult();
|
@@ -134,10 +120,26 @@ class OutputPluginDelegate {
|
|
134
120
|
|
135
121
|
final List<TaskReport> result = new ArrayList<>();
|
136
122
|
for (TaskReport taskReport : reports) {
|
137
|
-
final TaskReport report = taskReport.get(TaskReports.class, MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS).get(
|
123
|
+
final TaskReport report = taskReport.get(TaskReports.class, MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS).get(tag);
|
138
124
|
result.add(report);
|
139
125
|
}
|
140
126
|
return result;
|
141
127
|
}
|
142
128
|
}
|
129
|
+
|
130
|
+
class Transaction {
|
131
|
+
private final Future<ConfigDiff> future;
|
132
|
+
|
133
|
+
private Transaction(Future<ConfigDiff> future) {
|
134
|
+
this.future = future;
|
135
|
+
}
|
136
|
+
|
137
|
+
String getTag() {
|
138
|
+
return OutputPluginDelegate.this.getTag();
|
139
|
+
}
|
140
|
+
|
141
|
+
ConfigDiff getResult() throws ExecutionException, InterruptedException {
|
142
|
+
return future.get();
|
143
|
+
}
|
144
|
+
}
|
143
145
|
}
|
@@ -4,22 +4,22 @@ import com.fasterxml.jackson.annotation.JsonCreator;
|
|
4
4
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
5
5
|
import org.embulk.config.TaskReport;
|
6
6
|
|
7
|
-
import java.util.
|
7
|
+
import java.util.Map;
|
8
8
|
|
9
9
|
class TaskReports {
|
10
|
-
private final
|
10
|
+
private final Map<String, TaskReport> reports;
|
11
11
|
|
12
12
|
@JsonCreator
|
13
|
-
TaskReports(@JsonProperty("reports")
|
13
|
+
TaskReports(@JsonProperty("reports") Map<String, TaskReport> reports) {
|
14
14
|
this.reports = reports;
|
15
15
|
}
|
16
16
|
|
17
17
|
@JsonProperty("reports")
|
18
|
-
|
18
|
+
Map<String, TaskReport> getReports() {
|
19
19
|
return reports;
|
20
20
|
}
|
21
21
|
|
22
|
-
TaskReport get(
|
23
|
-
return reports.get(
|
22
|
+
TaskReport get(String tag) {
|
23
|
+
return reports.get(tag);
|
24
24
|
}
|
25
25
|
}
|
@@ -24,10 +24,10 @@ import static org.embulk.test.Utils.configFromResource;
|
|
24
24
|
import static org.embulk.test.Utils.record;
|
25
25
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
26
26
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
27
|
-
import static org.junit.jupiter.api.Assertions.assertTrue;
|
28
27
|
|
29
28
|
@EmbulkTest(MultiOutputPlugin.class)
|
30
29
|
class TestMultiOutputPlugin extends EmbulkPluginTest {
|
30
|
+
|
31
31
|
@Test
|
32
32
|
void testMultipleOutputWorking() throws IOException {
|
33
33
|
final ConfigSource inConfig = configFromResource("yaml/in_base.yml");
|
@@ -115,7 +115,7 @@ class TestMultiOutputPlugin extends EmbulkPluginTest {
|
|
115
115
|
}
|
116
116
|
|
117
117
|
@Test
|
118
|
-
void
|
118
|
+
void testAnOutputFailedAfterOpen() throws IOException {
|
119
119
|
final ConfigSource inConfig = configFromResource("yaml/in_base.yml");
|
120
120
|
final ConfigSource outConfig = configFromResource("yaml/out_failed_output_after_open.yml");
|
121
121
|
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
@@ -138,7 +138,7 @@ class TestMultiOutputPlugin extends EmbulkPluginTest {
|
|
138
138
|
}
|
139
139
|
|
140
140
|
@Test
|
141
|
-
void
|
141
|
+
void testAnOutputFailedBeforeOpen() throws IOException {
|
142
142
|
final ConfigSource inConfig = configFromResource("yaml/in_base.yml");
|
143
143
|
final ConfigSource outConfig = configFromResource("yaml/out_failed_output_before_open.yml");
|
144
144
|
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-multi
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shinichi ISHIMURA
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-01-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -50,7 +50,7 @@ files:
|
|
50
50
|
- LICENSE
|
51
51
|
- README.md
|
52
52
|
- build.gradle
|
53
|
-
- classpath/embulk-output-multi-0.
|
53
|
+
- classpath/embulk-output-multi-0.5.0.jar
|
54
54
|
- gradle.properties
|
55
55
|
- gradle/dependency-locks/compileClasspath.lockfile
|
56
56
|
- gradle/dependency-locks/testCompileClasspath.lockfile
|
@@ -62,10 +62,9 @@ files:
|
|
62
62
|
- settings.gradle
|
63
63
|
- src/main/java/org/embulk/output/multi/AsyncRunControl.java
|
64
64
|
- src/main/java/org/embulk/output/multi/MultiOutputPlugin.java
|
65
|
+
- src/main/java/org/embulk/output/multi/MultiTransactionalPageOutput.java
|
65
66
|
- src/main/java/org/embulk/output/multi/OutputPluginDelegate.java
|
66
|
-
- src/main/java/org/embulk/output/multi/PluginExecutionException.java
|
67
67
|
- src/main/java/org/embulk/output/multi/TaskReports.java
|
68
|
-
- src/main/java/org/embulk/output/multi/TransactionalPageOutputDelegate.java
|
69
68
|
- src/test/java/org/embulk/output/multi/TestMultiOutputPlugin.java
|
70
69
|
- src/test/resources/yaml/in_base.yml
|
71
70
|
- src/test/resources/yaml/out_base.yml
|
@@ -1,15 +0,0 @@
|
|
1
|
-
package org.embulk.output.multi;
|
2
|
-
|
3
|
-
class PluginExecutionException extends RuntimeException {
|
4
|
-
|
5
|
-
private final OutputPluginDelegate plugin;
|
6
|
-
|
7
|
-
PluginExecutionException(OutputPluginDelegate plugin, Throwable cause) {
|
8
|
-
super(cause);
|
9
|
-
this.plugin = plugin;
|
10
|
-
}
|
11
|
-
|
12
|
-
OutputPluginDelegate getPlugin() {
|
13
|
-
return plugin;
|
14
|
-
}
|
15
|
-
}
|
@@ -1,92 +0,0 @@
|
|
1
|
-
package org.embulk.output.multi;
|
2
|
-
|
3
|
-
import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
4
|
-
import org.embulk.config.TaskReport;
|
5
|
-
import org.embulk.spi.Page;
|
6
|
-
import org.embulk.spi.TransactionalPageOutput;
|
7
|
-
|
8
|
-
import java.util.concurrent.BlockingQueue;
|
9
|
-
import java.util.concurrent.Callable;
|
10
|
-
import java.util.concurrent.ExecutionException;
|
11
|
-
import java.util.concurrent.ExecutorService;
|
12
|
-
import java.util.concurrent.Executors;
|
13
|
-
import java.util.concurrent.Future;
|
14
|
-
import java.util.concurrent.LinkedBlockingQueue;
|
15
|
-
import java.util.function.Supplier;
|
16
|
-
|
17
|
-
class TransactionalPageOutputDelegate {
|
18
|
-
private static final String THREAD_NAME_FORMAT = "multi-page-output-%s-%d";
|
19
|
-
private final OutputPluginDelegate source;
|
20
|
-
private final TransactionalPageOutput delegate;
|
21
|
-
private final BlockingQueue<Supplier<Object>> taskQueue;
|
22
|
-
private final ExecutorService executorService;
|
23
|
-
private final Future<TaskReport> result;
|
24
|
-
|
25
|
-
TransactionalPageOutputDelegate(
|
26
|
-
int taskIndex,
|
27
|
-
OutputPluginDelegate source,
|
28
|
-
TransactionalPageOutput delegate
|
29
|
-
) {
|
30
|
-
this.source = source;
|
31
|
-
this.delegate = delegate;
|
32
|
-
this.taskQueue = new LinkedBlockingQueue<>();
|
33
|
-
this.executorService = Executors.newSingleThreadExecutor(
|
34
|
-
new ThreadFactoryBuilder().setNameFormat(String.format(THREAD_NAME_FORMAT, source.getPluginCode(), taskIndex)).build()
|
35
|
-
);
|
36
|
-
this.result = executorService.submit(new Worker());
|
37
|
-
}
|
38
|
-
|
39
|
-
void add(Page page) {
|
40
|
-
taskQueue.add(() -> {
|
41
|
-
delegate.add(page);
|
42
|
-
return null;
|
43
|
-
});
|
44
|
-
}
|
45
|
-
|
46
|
-
void finish() {
|
47
|
-
taskQueue.add(() -> {
|
48
|
-
delegate.finish();
|
49
|
-
return null;
|
50
|
-
});
|
51
|
-
}
|
52
|
-
|
53
|
-
void close() {
|
54
|
-
taskQueue.add(() -> {
|
55
|
-
delegate.close();
|
56
|
-
return null;
|
57
|
-
});
|
58
|
-
}
|
59
|
-
|
60
|
-
void abort() {
|
61
|
-
taskQueue.add(() -> {
|
62
|
-
delegate.abort();
|
63
|
-
return null;
|
64
|
-
});
|
65
|
-
}
|
66
|
-
|
67
|
-
TaskReport commit() {
|
68
|
-
taskQueue.add(delegate::commit);
|
69
|
-
try {
|
70
|
-
return result.get();
|
71
|
-
} catch (InterruptedException e) {
|
72
|
-
Thread.currentThread().interrupt();
|
73
|
-
throw new RuntimeException(e);
|
74
|
-
} catch (ExecutionException e) {
|
75
|
-
throw new PluginExecutionException(source, e.getCause());
|
76
|
-
} finally {
|
77
|
-
executorService.shutdown();
|
78
|
-
}
|
79
|
-
}
|
80
|
-
|
81
|
-
private class Worker implements Callable<TaskReport> {
|
82
|
-
@Override
|
83
|
-
public TaskReport call() throws InterruptedException {
|
84
|
-
while (true) {
|
85
|
-
final Object result = taskQueue.take().get();
|
86
|
-
if (result instanceof TaskReport) {
|
87
|
-
return (TaskReport) result;
|
88
|
-
}
|
89
|
-
}
|
90
|
-
}
|
91
|
-
}
|
92
|
-
}
|