embulk-output-multi 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/gradle.properties +1 -1
- data/src/main/java/org/embulk/output/multi/MultiOutputPlugin.java +1 -67
- data/src/main/java/org/embulk/output/multi/MultiTransactionalPageOutput.java +137 -0
- metadata +4 -4
- data/src/main/java/org/embulk/output/multi/TransactionalPageOutputDelegate.java +0 -105
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 0da2d60e10e0930e908e7fb4c261b07a6289c709
|
|
4
|
+
data.tar.gz: da8a2b1cfcfb09969965c4c54d7ce2a3d57ea2f3
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ad76fc7d3ba6dc6b6734c79b81a2cbe833e41de80bcdde9e6b07a4058fbef42743f77db6fb30c40ae21cf3addd1210fc2f327285e59dbeb63a65addc9439e995
|
|
7
|
+
data.tar.gz: aa39b7f67554bcc4ab62e928e7356f80c2851b36d403c895dae84ace06c31ade7627bfa521279fcbedbc7ff67e0245989626820b0305000babd4d8fad780e773
|
data/gradle.properties
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
version=0.
|
|
1
|
+
version=0.4.0
|
|
@@ -9,11 +9,9 @@ import org.embulk.config.Task;
|
|
|
9
9
|
import org.embulk.config.TaskReport;
|
|
10
10
|
import org.embulk.config.TaskSource;
|
|
11
11
|
import org.embulk.plugin.PluginType;
|
|
12
|
-
import org.embulk.spi.Buffer;
|
|
13
12
|
import org.embulk.spi.Exec;
|
|
14
13
|
import org.embulk.spi.ExecSession;
|
|
15
14
|
import org.embulk.spi.OutputPlugin;
|
|
16
|
-
import org.embulk.spi.Page;
|
|
17
15
|
import org.embulk.spi.Schema;
|
|
18
16
|
import org.embulk.spi.TransactionalPageOutput;
|
|
19
17
|
import org.slf4j.Logger;
|
|
@@ -26,7 +24,6 @@ import java.util.Map;
|
|
|
26
24
|
import java.util.Optional;
|
|
27
25
|
import java.util.concurrent.ExecutionException;
|
|
28
26
|
import java.util.function.Function;
|
|
29
|
-
import java.util.stream.Collectors;
|
|
30
27
|
|
|
31
28
|
public class MultiOutputPlugin implements OutputPlugin {
|
|
32
29
|
public interface PluginTask extends Task {
|
|
@@ -82,70 +79,7 @@ public class MultiOutputPlugin implements OutputPlugin {
|
|
|
82
79
|
public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) {
|
|
83
80
|
final PluginTask task = taskSource.loadTask(PluginTask.class);
|
|
84
81
|
final ExecSession session = Exec.session();
|
|
85
|
-
|
|
86
|
-
TransactionalPageOutputDelegate.open(schema, taskIndex, delegate)
|
|
87
|
-
);
|
|
88
|
-
|
|
89
|
-
return new TransactionalPageOutput() {
|
|
90
|
-
@Override
|
|
91
|
-
public void add(Page original) {
|
|
92
|
-
final Buffer originalBuffer = original.buffer();
|
|
93
|
-
for (TransactionalPageOutputDelegate output : delegates) {
|
|
94
|
-
final Buffer copiedBuffer = Buffer.wrap(originalBuffer.array());
|
|
95
|
-
copiedBuffer.offset(originalBuffer.offset());
|
|
96
|
-
copiedBuffer.limit(originalBuffer.limit());
|
|
97
|
-
|
|
98
|
-
final Page copiedPage = Page.wrap(copiedBuffer);
|
|
99
|
-
copiedPage.setStringReferences(new ArrayList<>(original.getStringReferences()));
|
|
100
|
-
copiedPage.setValueReferences(new ArrayList<>(original.getValueReferences()));
|
|
101
|
-
|
|
102
|
-
output.add(copiedPage);
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
@Override
|
|
107
|
-
public void finish() {
|
|
108
|
-
for (TransactionalPageOutputDelegate output : delegates) {
|
|
109
|
-
output.finish();
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
@Override
|
|
114
|
-
public void close() {
|
|
115
|
-
for (TransactionalPageOutputDelegate output : delegates) {
|
|
116
|
-
output.close();
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
@Override
|
|
121
|
-
public void abort() {
|
|
122
|
-
for (TransactionalPageOutputDelegate output : delegates) {
|
|
123
|
-
output.abort();
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
@Override
|
|
128
|
-
public TaskReport commit() {
|
|
129
|
-
final TaskReport report = Exec.newTaskReport();
|
|
130
|
-
final Map<String, TaskReport> reports = new HashMap<>();
|
|
131
|
-
final List<OutputPluginDelegate> errorPlugins = new ArrayList<>();
|
|
132
|
-
for (TransactionalPageOutputDelegate output : delegates) {
|
|
133
|
-
try {
|
|
134
|
-
reports.put(output.getTag(), output.commit());
|
|
135
|
-
} catch (PluginExecutionException e) {
|
|
136
|
-
errorPlugins.add(e.getPlugin());
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
if (!errorPlugins.isEmpty()) {
|
|
140
|
-
throw new RuntimeException(
|
|
141
|
-
String.format("Following plugins failed to output [%s]",
|
|
142
|
-
errorPlugins.stream().map(OutputPluginDelegate::getTag).collect(Collectors.joining(", "))
|
|
143
|
-
));
|
|
144
|
-
}
|
|
145
|
-
report.set(CONFIG_NAME_OUTPUT_TASK_REPORTS, new TaskReports(reports));
|
|
146
|
-
return report;
|
|
147
|
-
}
|
|
148
|
-
};
|
|
82
|
+
return MultiTransactionalPageOutput.open(schema, taskIndex, mapWithPluginDelegate(task, session, Function.identity()));
|
|
149
83
|
}
|
|
150
84
|
|
|
151
85
|
private static ConfigDiff buildConfigDiff(List<OutputPluginDelegate.Transaction> transactions) {
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
package org.embulk.output.multi;
|
|
2
|
+
|
|
3
|
+
import org.embulk.config.TaskReport;
|
|
4
|
+
import org.embulk.spi.Buffer;
|
|
5
|
+
import org.embulk.spi.Exec;
|
|
6
|
+
import org.embulk.spi.Page;
|
|
7
|
+
import org.embulk.spi.Schema;
|
|
8
|
+
import org.embulk.spi.TransactionalPageOutput;
|
|
9
|
+
import org.slf4j.Logger;
|
|
10
|
+
import org.slf4j.LoggerFactory;
|
|
11
|
+
|
|
12
|
+
import java.util.ArrayList;
|
|
13
|
+
import java.util.HashMap;
|
|
14
|
+
import java.util.List;
|
|
15
|
+
import java.util.Map;
|
|
16
|
+
import java.util.function.Consumer;
|
|
17
|
+
import java.util.stream.Collectors;
|
|
18
|
+
|
|
19
|
+
public class MultiTransactionalPageOutput implements TransactionalPageOutput {
|
|
20
|
+
private static final Logger LOGGER = LoggerFactory.getLogger(MultiTransactionalPageOutput.class);
|
|
21
|
+
private final int taskIndex;
|
|
22
|
+
private final List<Delegate> delegates;
|
|
23
|
+
|
|
24
|
+
static MultiTransactionalPageOutput open(Schema schema, int taskIndex, List<OutputPluginDelegate> plugins) {
|
|
25
|
+
return new MultiTransactionalPageOutput(
|
|
26
|
+
taskIndex,
|
|
27
|
+
plugins.stream()
|
|
28
|
+
.map(plugin -> new Delegate(plugin, plugin.open(schema, taskIndex)))
|
|
29
|
+
.collect(Collectors.toList())
|
|
30
|
+
);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
private MultiTransactionalPageOutput(int taskIndex, List<Delegate> delegates) {
|
|
34
|
+
this.taskIndex = taskIndex;
|
|
35
|
+
this.delegates = delegates;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
@Override
|
|
39
|
+
public void add(Page page) {
|
|
40
|
+
applyToAllPlugins(delegate -> delegate.add(copyPage(page)));
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
@Override
|
|
44
|
+
public void finish() {
|
|
45
|
+
applyToAllPlugins(Delegate::finish);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
@Override
|
|
49
|
+
public void close() {
|
|
50
|
+
applyToAllPlugins(Delegate::close);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
@Override
|
|
54
|
+
public void abort() {
|
|
55
|
+
applyToAllPlugins(Delegate::abort);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
@Override
|
|
59
|
+
public TaskReport commit() {
|
|
60
|
+
final Map<String, TaskReport> reports = new HashMap<>();
|
|
61
|
+
applyToAllPlugins(delegate -> reports.put(delegate.getTag(), delegate.commit()));
|
|
62
|
+
final TaskReport report = Exec.newTaskReport();
|
|
63
|
+
report.set(MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS, new TaskReports(reports));
|
|
64
|
+
return report;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
private void applyToAllPlugins(Consumer<Delegate> command) {
|
|
68
|
+
final List<OutputPluginDelegate> errorPlugins = new ArrayList<>();
|
|
69
|
+
for (Delegate delegate : delegates) {
|
|
70
|
+
try {
|
|
71
|
+
command.accept(delegate);
|
|
72
|
+
} catch (Exception e) {
|
|
73
|
+
LOGGER.warn(String.format("Output for %s on index %d failed.", delegate.plugin.getTag(), taskIndex), e);
|
|
74
|
+
errorPlugins.add(delegate.plugin);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
if (!errorPlugins.isEmpty()) {
|
|
79
|
+
throw new RuntimeException(
|
|
80
|
+
String.format("Following plugins failed to output [%s] on index %d",
|
|
81
|
+
errorPlugins.stream().map(OutputPluginDelegate::getTag).collect(Collectors.joining(", ")),
|
|
82
|
+
taskIndex
|
|
83
|
+
));
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
private static Page copyPage(Page original) {
|
|
88
|
+
final Buffer originalBuffer = original.buffer();
|
|
89
|
+
final Buffer copiedBuffer = Buffer.wrap(originalBuffer.array());
|
|
90
|
+
copiedBuffer.offset(originalBuffer.offset());
|
|
91
|
+
copiedBuffer.limit(originalBuffer.limit());
|
|
92
|
+
|
|
93
|
+
final Page copiedPage = Page.wrap(copiedBuffer);
|
|
94
|
+
copiedPage.setStringReferences(new ArrayList<>(original.getStringReferences()));
|
|
95
|
+
copiedPage.setValueReferences(new ArrayList<>(original.getValueReferences()));
|
|
96
|
+
return copiedPage;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
static class Delegate implements TransactionalPageOutput {
|
|
100
|
+
private final OutputPluginDelegate plugin;
|
|
101
|
+
private final TransactionalPageOutput output;
|
|
102
|
+
|
|
103
|
+
private Delegate(OutputPluginDelegate plugin, TransactionalPageOutput output) {
|
|
104
|
+
this.plugin = plugin;
|
|
105
|
+
this.output = output;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
@Override
|
|
109
|
+
public void add(Page page) {
|
|
110
|
+
output.add(page);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
@Override
|
|
114
|
+
public void finish() {
|
|
115
|
+
output.finish();
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
@Override
|
|
119
|
+
public void close() {
|
|
120
|
+
output.close();
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
@Override
|
|
124
|
+
public void abort() {
|
|
125
|
+
output.abort();
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
@Override
|
|
129
|
+
public TaskReport commit() {
|
|
130
|
+
return output.commit();
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
String getTag() {
|
|
134
|
+
return plugin.getTag();
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-output-multi
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Shinichi ISHIMURA
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2019-03-
|
|
11
|
+
date: 2019-03-11 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -50,7 +50,7 @@ files:
|
|
|
50
50
|
- LICENSE
|
|
51
51
|
- README.md
|
|
52
52
|
- build.gradle
|
|
53
|
-
- classpath/embulk-output-multi-0.
|
|
53
|
+
- classpath/embulk-output-multi-0.4.0.jar
|
|
54
54
|
- gradle.properties
|
|
55
55
|
- gradle/dependency-locks/compileClasspath.lockfile
|
|
56
56
|
- gradle/dependency-locks/testCompileClasspath.lockfile
|
|
@@ -62,10 +62,10 @@ files:
|
|
|
62
62
|
- settings.gradle
|
|
63
63
|
- src/main/java/org/embulk/output/multi/AsyncRunControl.java
|
|
64
64
|
- src/main/java/org/embulk/output/multi/MultiOutputPlugin.java
|
|
65
|
+
- src/main/java/org/embulk/output/multi/MultiTransactionalPageOutput.java
|
|
65
66
|
- src/main/java/org/embulk/output/multi/OutputPluginDelegate.java
|
|
66
67
|
- src/main/java/org/embulk/output/multi/PluginExecutionException.java
|
|
67
68
|
- src/main/java/org/embulk/output/multi/TaskReports.java
|
|
68
|
-
- src/main/java/org/embulk/output/multi/TransactionalPageOutputDelegate.java
|
|
69
69
|
- src/test/java/org/embulk/output/multi/TestMultiOutputPlugin.java
|
|
70
70
|
- src/test/resources/yaml/in_base.yml
|
|
71
71
|
- src/test/resources/yaml/out_base.yml
|
|
@@ -1,105 +0,0 @@
|
|
|
1
|
-
package org.embulk.output.multi;
|
|
2
|
-
|
|
3
|
-
import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
|
4
|
-
import org.embulk.config.TaskReport;
|
|
5
|
-
import org.embulk.spi.Page;
|
|
6
|
-
import org.embulk.spi.Schema;
|
|
7
|
-
import org.embulk.spi.TransactionalPageOutput;
|
|
8
|
-
|
|
9
|
-
import java.util.concurrent.BlockingQueue;
|
|
10
|
-
import java.util.concurrent.Callable;
|
|
11
|
-
import java.util.concurrent.ExecutionException;
|
|
12
|
-
import java.util.concurrent.ExecutorService;
|
|
13
|
-
import java.util.concurrent.Executors;
|
|
14
|
-
import java.util.concurrent.Future;
|
|
15
|
-
import java.util.concurrent.LinkedBlockingQueue;
|
|
16
|
-
import java.util.function.Supplier;
|
|
17
|
-
|
|
18
|
-
class TransactionalPageOutputDelegate {
|
|
19
|
-
private static final String THREAD_NAME_FORMAT = "multi-page-output-%s-%d";
|
|
20
|
-
private final OutputPluginDelegate source;
|
|
21
|
-
private final TransactionalPageOutput delegate;
|
|
22
|
-
private final BlockingQueue<Supplier<Object>> taskQueue;
|
|
23
|
-
private final ExecutorService executorService;
|
|
24
|
-
private final Future<TaskReport> worker;
|
|
25
|
-
private boolean isFailed;
|
|
26
|
-
|
|
27
|
-
static TransactionalPageOutputDelegate open(Schema schema, int taskIndex, OutputPluginDelegate delegate) {
|
|
28
|
-
return new TransactionalPageOutputDelegate(taskIndex, delegate, delegate.open(schema, taskIndex));
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
private TransactionalPageOutputDelegate(
|
|
32
|
-
int taskIndex,
|
|
33
|
-
OutputPluginDelegate source,
|
|
34
|
-
TransactionalPageOutput delegate
|
|
35
|
-
) {
|
|
36
|
-
this.source = source;
|
|
37
|
-
this.delegate = delegate;
|
|
38
|
-
this.taskQueue = new LinkedBlockingQueue<>();
|
|
39
|
-
this.executorService = Executors.newSingleThreadExecutor(
|
|
40
|
-
new ThreadFactoryBuilder().setNameFormat(String.format(THREAD_NAME_FORMAT, source.getTag(), taskIndex)).build()
|
|
41
|
-
);
|
|
42
|
-
this.worker = executorService.submit(new Worker());
|
|
43
|
-
this.isFailed = false;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
void add(Page page) {
|
|
47
|
-
taskQueue.add(() -> {
|
|
48
|
-
delegate.add(page);
|
|
49
|
-
return null;
|
|
50
|
-
});
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
void finish() {
|
|
54
|
-
taskQueue.add(() -> {
|
|
55
|
-
delegate.finish();
|
|
56
|
-
return null;
|
|
57
|
-
});
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
void close() {
|
|
61
|
-
taskQueue.add(() -> {
|
|
62
|
-
delegate.close();
|
|
63
|
-
return null;
|
|
64
|
-
});
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
void abort() {
|
|
68
|
-
// Run abort only if the output failed.
|
|
69
|
-
if (isFailed) {
|
|
70
|
-
delegate.abort();
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
TaskReport commit() {
|
|
75
|
-
taskQueue.add(delegate::commit);
|
|
76
|
-
try {
|
|
77
|
-
return worker.get();
|
|
78
|
-
} catch (InterruptedException e) {
|
|
79
|
-
isFailed = true;
|
|
80
|
-
Thread.currentThread().interrupt();
|
|
81
|
-
throw new RuntimeException(e);
|
|
82
|
-
} catch (ExecutionException e) {
|
|
83
|
-
isFailed = true;
|
|
84
|
-
throw new PluginExecutionException(source, e.getCause());
|
|
85
|
-
} finally {
|
|
86
|
-
executorService.shutdown();
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
String getTag() {
|
|
91
|
-
return source.getTag();
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
private class Worker implements Callable<TaskReport> {
|
|
95
|
-
@Override
|
|
96
|
-
public TaskReport call() throws InterruptedException {
|
|
97
|
-
while (true) {
|
|
98
|
-
final Object result = taskQueue.take().get();
|
|
99
|
-
if (result instanceof TaskReport) {
|
|
100
|
-
return (TaskReport) result;
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
}
|