embulk 0.8.14-java → 0.8.15-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/build.gradle +3 -3
- data/embulk-core/src/main/java/org/embulk/EmbulkEmbed.java +1 -1
- data/embulk-core/src/main/java/org/embulk/exec/BulkLoader.java +35 -7
- data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java +2 -0
- data/embulk-core/src/main/java/org/embulk/exec/ExecutionResult.java +8 -1
- data/embulk-core/src/main/java/org/embulk/exec/SkipTransactionException.java +23 -0
- data/embulk-docs/src/built-in.rst +299 -62
- data/embulk-docs/src/release.rst +1 -0
- data/embulk-docs/src/release/release-0.8.15.rst +17 -0
- data/embulk-standards/src/main/java/org/embulk/standards/RenameFilterPlugin.java +374 -1
- data/embulk-standards/src/test/java/org/embulk/standards/TestRenameFilterPlugin.java +872 -0
- data/embulk-test/build.gradle +6 -0
- data/embulk-test/src/main/java/org/embulk/test/EmbulkTests.java +75 -0
- data/embulk-test/src/main/java/org/embulk/test/TestingBulkLoader.java +124 -0
- data/embulk-test/src/main/java/org/embulk/test/TestingEmbulk.java +223 -0
- data/lib/embulk/version.rb +1 -1
- data/settings.gradle +1 -0
- metadata +11 -5
@@ -0,0 +1,75 @@
|
|
1
|
+
package org.embulk.test;
|
2
|
+
|
3
|
+
import com.google.common.base.Throwables;
|
4
|
+
import com.google.common.io.CharStreams;
|
5
|
+
import com.google.common.io.Resources;
|
6
|
+
import java.io.File;
|
7
|
+
import java.io.IOException;
|
8
|
+
import java.io.InputStream;
|
9
|
+
import java.io.InputStreamReader;
|
10
|
+
import java.nio.file.Files;
|
11
|
+
import java.nio.file.Path;
|
12
|
+
import java.util.Collections;
|
13
|
+
import java.util.List;
|
14
|
+
import org.embulk.EmbulkEmbed;
|
15
|
+
import org.embulk.config.ConfigSource;
|
16
|
+
import static com.google.common.base.Strings.isNullOrEmpty;
|
17
|
+
import static java.nio.charset.StandardCharsets.UTF_8;
|
18
|
+
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
|
19
|
+
import static org.hamcrest.Matchers.is;
|
20
|
+
import static org.junit.Assume.assumeThat;
|
21
|
+
|
22
|
+
public class EmbulkTests
|
23
|
+
{
|
24
|
+
private EmbulkTests()
|
25
|
+
{ }
|
26
|
+
|
27
|
+
public static ConfigSource config(String envName)
|
28
|
+
{
|
29
|
+
String path = System.getenv(envName);
|
30
|
+
assumeThat(isNullOrEmpty(path), is(false));
|
31
|
+
try {
|
32
|
+
return EmbulkEmbed.newSystemConfigLoader().fromYamlFile(new File(path));
|
33
|
+
} catch (IOException ex) {
|
34
|
+
throw Throwables.propagate(ex);
|
35
|
+
}
|
36
|
+
}
|
37
|
+
|
38
|
+
public static String readResource(String name)
|
39
|
+
{
|
40
|
+
try (InputStream in = Resources.getResource(name).openStream()) {
|
41
|
+
return CharStreams.toString(new InputStreamReader(in, UTF_8));
|
42
|
+
}
|
43
|
+
catch (IOException ex) {
|
44
|
+
throw new RuntimeException(ex);
|
45
|
+
}
|
46
|
+
}
|
47
|
+
|
48
|
+
public static void copyResource(String resource, Path dest)
|
49
|
+
throws IOException
|
50
|
+
{
|
51
|
+
Files.createDirectories(dest.getParent());
|
52
|
+
try (InputStream input = Resources.getResource(resource).openStream()) {
|
53
|
+
Files.copy(input, dest, REPLACE_EXISTING);
|
54
|
+
}
|
55
|
+
}
|
56
|
+
|
57
|
+
public static String readFile(Path path) throws IOException
|
58
|
+
{
|
59
|
+
try (InputStream in = Files.newInputStream(path)) {
|
60
|
+
return CharStreams.toString(new InputStreamReader(in, UTF_8));
|
61
|
+
}
|
62
|
+
}
|
63
|
+
|
64
|
+
public static String readSortedFile(Path path) throws IOException
|
65
|
+
{
|
66
|
+
List<String> lines = Files.readAllLines(path, UTF_8);
|
67
|
+
Collections.sort(lines);
|
68
|
+
StringBuilder sb = new StringBuilder();
|
69
|
+
for (String line : lines) {
|
70
|
+
sb.append(line);
|
71
|
+
sb.append("\n");
|
72
|
+
}
|
73
|
+
return sb.toString();
|
74
|
+
}
|
75
|
+
}
|
@@ -0,0 +1,124 @@
|
|
1
|
+
package org.embulk.test;
|
2
|
+
|
3
|
+
import com.google.common.base.Function;
|
4
|
+
import com.google.common.base.Optional;
|
5
|
+
import com.google.common.collect.ImmutableList;
|
6
|
+
import com.google.inject.Binder;
|
7
|
+
import com.google.inject.Inject;
|
8
|
+
import com.google.inject.Injector;
|
9
|
+
import com.google.inject.Module;
|
10
|
+
import com.google.inject.util.Modules;
|
11
|
+
import java.util.List;
|
12
|
+
import org.embulk.config.ConfigSource;
|
13
|
+
import org.embulk.config.TaskReport;
|
14
|
+
import org.embulk.exec.BulkLoader;
|
15
|
+
import org.embulk.exec.ExecutionResult;
|
16
|
+
import org.embulk.exec.ForSystemConfig;
|
17
|
+
import org.embulk.exec.ResumeState;
|
18
|
+
import org.embulk.spi.Exec;
|
19
|
+
import org.embulk.spi.ExecSession;
|
20
|
+
import org.embulk.spi.Schema;
|
21
|
+
import org.slf4j.Logger;
|
22
|
+
|
23
|
+
class TestingBulkLoader
|
24
|
+
extends BulkLoader
|
25
|
+
{
|
26
|
+
static Function<List<Module>, List<Module>> override()
|
27
|
+
{
|
28
|
+
return new Function<List<Module>, List<Module>>() {
|
29
|
+
@Override
|
30
|
+
public List<Module> apply(List<Module> modules)
|
31
|
+
{
|
32
|
+
Module override = new Module() {
|
33
|
+
public void configure(Binder binder)
|
34
|
+
{
|
35
|
+
binder.bind(BulkLoader.class).to(TestingBulkLoader.class);
|
36
|
+
}
|
37
|
+
};
|
38
|
+
return ImmutableList.of(Modules.override(modules).with(ImmutableList.of(override)));
|
39
|
+
}
|
40
|
+
};
|
41
|
+
}
|
42
|
+
|
43
|
+
@Inject
|
44
|
+
public TestingBulkLoader(Injector injector,
|
45
|
+
@ForSystemConfig ConfigSource systemConfig)
|
46
|
+
{
|
47
|
+
super(injector, systemConfig);
|
48
|
+
}
|
49
|
+
|
50
|
+
@Override
|
51
|
+
protected LoaderState newLoaderState(Logger logger, ProcessPluginSet plugins)
|
52
|
+
{
|
53
|
+
return new TestingLoaderState(logger, plugins);
|
54
|
+
}
|
55
|
+
|
56
|
+
protected static class TestingLoaderState
|
57
|
+
extends LoaderState
|
58
|
+
{
|
59
|
+
public TestingLoaderState(Logger logger, ProcessPluginSet plugins)
|
60
|
+
{
|
61
|
+
super(logger, plugins);
|
62
|
+
}
|
63
|
+
|
64
|
+
@Override
|
65
|
+
public ExecutionResult buildExecuteResultWithWarningException(Throwable ex)
|
66
|
+
{
|
67
|
+
ExecutionResult result = super.buildExecuteResultWithWarningException(ex);
|
68
|
+
return new TestingExecutionResult(result, buildResumeState(Exec.session()), Exec.session());
|
69
|
+
}
|
70
|
+
}
|
71
|
+
|
72
|
+
static class TestingExecutionResult
|
73
|
+
extends ExecutionResult
|
74
|
+
implements TestingEmbulk.RunResult
|
75
|
+
{
|
76
|
+
private final Schema inputSchema;
|
77
|
+
private final Schema outputSchema;
|
78
|
+
private final List<TaskReport> inputTaskReports;
|
79
|
+
private final List<TaskReport> outputTaskReports;
|
80
|
+
|
81
|
+
public TestingExecutionResult(ExecutionResult orig,
|
82
|
+
ResumeState resumeState, ExecSession session)
|
83
|
+
{
|
84
|
+
super(orig.getConfigDiff(), orig.isSkipped(), orig.getIgnoredExceptions());
|
85
|
+
this.inputSchema = resumeState.getInputSchema();
|
86
|
+
this.outputSchema = resumeState.getOutputSchema();
|
87
|
+
this.inputTaskReports = buildReports(resumeState.getInputTaskReports(), session);
|
88
|
+
this.outputTaskReports = buildReports(resumeState.getOutputTaskReports(), session);
|
89
|
+
}
|
90
|
+
|
91
|
+
private static List<TaskReport> buildReports(List<Optional<TaskReport>> optionalReports, ExecSession session)
|
92
|
+
{
|
93
|
+
ImmutableList.Builder<TaskReport> reports = ImmutableList.builder();
|
94
|
+
for (Optional<TaskReport> report : optionalReports) {
|
95
|
+
reports.add(report.or(session.newTaskReport()));
|
96
|
+
}
|
97
|
+
return reports.build();
|
98
|
+
}
|
99
|
+
|
100
|
+
@Override
|
101
|
+
public Schema getInputSchema()
|
102
|
+
{
|
103
|
+
return inputSchema;
|
104
|
+
}
|
105
|
+
|
106
|
+
@Override
|
107
|
+
public Schema getOutputSchema()
|
108
|
+
{
|
109
|
+
return outputSchema;
|
110
|
+
}
|
111
|
+
|
112
|
+
@Override
|
113
|
+
public List<TaskReport> getInputTaskReports()
|
114
|
+
{
|
115
|
+
return inputTaskReports;
|
116
|
+
}
|
117
|
+
|
118
|
+
@Override
|
119
|
+
public List<TaskReport> getOutputTaskReports()
|
120
|
+
{
|
121
|
+
return outputTaskReports;
|
122
|
+
}
|
123
|
+
}
|
124
|
+
}
|
@@ -0,0 +1,223 @@
|
|
1
|
+
package org.embulk.test;
|
2
|
+
|
3
|
+
import com.google.common.collect.ImmutableList;
|
4
|
+
import com.google.common.base.Optional;
|
5
|
+
import com.google.common.io.ByteStreams;
|
6
|
+
import com.google.inject.Binder;
|
7
|
+
import com.google.inject.Injector;
|
8
|
+
import com.google.inject.Module;
|
9
|
+
import java.io.IOException;
|
10
|
+
import java.io.InputStream;
|
11
|
+
import java.io.OutputStream;
|
12
|
+
import java.nio.file.DirectoryStream;
|
13
|
+
import java.nio.file.Files;
|
14
|
+
import java.nio.file.Path;
|
15
|
+
import java.util.ArrayList;
|
16
|
+
import java.util.Collections;
|
17
|
+
import java.util.List;
|
18
|
+
import org.embulk.EmbulkEmbed;
|
19
|
+
import org.embulk.config.Config;
|
20
|
+
import org.embulk.config.ConfigDiff;
|
21
|
+
import org.embulk.config.ConfigLoader;
|
22
|
+
import org.embulk.config.ConfigSource;
|
23
|
+
import org.embulk.config.TaskReport;
|
24
|
+
import org.embulk.exec.ExecutionResult;
|
25
|
+
import org.embulk.spi.Schema;
|
26
|
+
import org.embulk.spi.TempFileException;
|
27
|
+
import org.embulk.spi.TempFileSpace;
|
28
|
+
import org.junit.rules.TestRule;
|
29
|
+
import org.junit.rules.TestWatcher;
|
30
|
+
import org.junit.runner.Description;
|
31
|
+
import org.junit.runners.model.Statement;
|
32
|
+
import static com.google.common.base.Preconditions.checkArgument;
|
33
|
+
import static org.embulk.plugin.InjectedPluginSource.registerPluginTo;
|
34
|
+
|
35
|
+
public class TestingEmbulk
|
36
|
+
implements TestRule
|
37
|
+
{
|
38
|
+
public static class Builder
|
39
|
+
{
|
40
|
+
private List<Module> modules = new ArrayList<>();
|
41
|
+
|
42
|
+
Builder()
|
43
|
+
{ }
|
44
|
+
|
45
|
+
public <T> Builder registerPlugin(final Class<T> iface, final String name, final Class<?> impl)
|
46
|
+
{
|
47
|
+
modules.add(new Module() {
|
48
|
+
public void configure(Binder binder)
|
49
|
+
{
|
50
|
+
registerPluginTo(binder, iface, name, impl);
|
51
|
+
}
|
52
|
+
});
|
53
|
+
return this;
|
54
|
+
}
|
55
|
+
|
56
|
+
public TestingEmbulk build()
|
57
|
+
{
|
58
|
+
return new TestingEmbulk(this);
|
59
|
+
}
|
60
|
+
}
|
61
|
+
|
62
|
+
public static Builder builder()
|
63
|
+
{
|
64
|
+
return new Builder();
|
65
|
+
}
|
66
|
+
|
67
|
+
private final List<Module> modules;
|
68
|
+
|
69
|
+
private EmbulkEmbed embed;
|
70
|
+
private TempFileSpace tempFiles;
|
71
|
+
|
72
|
+
TestingEmbulk(Builder builder)
|
73
|
+
{
|
74
|
+
this.modules = ImmutableList.copyOf(builder.modules);
|
75
|
+
reset();
|
76
|
+
}
|
77
|
+
|
78
|
+
public void reset()
|
79
|
+
{
|
80
|
+
destroy();
|
81
|
+
|
82
|
+
this.embed = new EmbulkEmbed.Bootstrap()
|
83
|
+
.addModules(modules)
|
84
|
+
.overrideModules(TestingBulkLoader.override())
|
85
|
+
.initializeCloseable();
|
86
|
+
|
87
|
+
try {
|
88
|
+
this.tempFiles = new TempFileSpace(Files.createTempDirectory("embulk-test-temp-").toFile());
|
89
|
+
}
|
90
|
+
catch (IOException ex) {
|
91
|
+
throw new TempFileException(ex);
|
92
|
+
}
|
93
|
+
}
|
94
|
+
|
95
|
+
public void destroy()
|
96
|
+
{
|
97
|
+
if (embed != null) {
|
98
|
+
embed.destroy();
|
99
|
+
embed = null;
|
100
|
+
}
|
101
|
+
if (tempFiles != null) {
|
102
|
+
tempFiles.cleanup();
|
103
|
+
tempFiles = null;
|
104
|
+
}
|
105
|
+
}
|
106
|
+
|
107
|
+
@Override
|
108
|
+
public Statement apply(Statement base, Description description)
|
109
|
+
{
|
110
|
+
return new EmbulkTestingEmbedWatcher().apply(base, description);
|
111
|
+
}
|
112
|
+
|
113
|
+
private class EmbulkTestingEmbedWatcher
|
114
|
+
extends TestWatcher
|
115
|
+
{
|
116
|
+
@Override
|
117
|
+
protected void starting(Description description)
|
118
|
+
{
|
119
|
+
reset();
|
120
|
+
}
|
121
|
+
|
122
|
+
@Override
|
123
|
+
protected void finished(Description description)
|
124
|
+
{
|
125
|
+
destroy();
|
126
|
+
}
|
127
|
+
}
|
128
|
+
|
129
|
+
public Path createTempFile(String suffix)
|
130
|
+
{
|
131
|
+
return tempFiles.createTempFile(suffix).toPath();
|
132
|
+
}
|
133
|
+
|
134
|
+
public Injector injector()
|
135
|
+
{
|
136
|
+
return embed.getInjector();
|
137
|
+
}
|
138
|
+
|
139
|
+
public ConfigLoader configLoader()
|
140
|
+
{
|
141
|
+
return embed.newConfigLoader();
|
142
|
+
}
|
143
|
+
|
144
|
+
public ConfigSource newConfig()
|
145
|
+
{
|
146
|
+
return configLoader().newConfigSource();
|
147
|
+
}
|
148
|
+
|
149
|
+
public ConfigSource loadYamlResource(String name)
|
150
|
+
{
|
151
|
+
return configLoader()
|
152
|
+
.fromYamlString(EmbulkTests.readResource(name));
|
153
|
+
}
|
154
|
+
|
155
|
+
public static interface RunResult
|
156
|
+
{
|
157
|
+
ConfigDiff getConfigDiff();
|
158
|
+
|
159
|
+
List<Throwable> getIgnoredExceptions();
|
160
|
+
|
161
|
+
Schema getInputSchema();
|
162
|
+
|
163
|
+
Schema getOutputSchema();
|
164
|
+
|
165
|
+
List<TaskReport> getInputTaskReports();
|
166
|
+
|
167
|
+
List<TaskReport> getOutputTaskReports();
|
168
|
+
}
|
169
|
+
|
170
|
+
public RunResult runInput(ConfigSource inConfig, Path outputPath)
|
171
|
+
throws IOException
|
172
|
+
{
|
173
|
+
String fileName = outputPath.getFileName().toString();
|
174
|
+
checkArgument(fileName.endsWith(".csv"), "outputPath must end with .csv");
|
175
|
+
Path dir = outputPath.getParent().resolve(fileName.substring(0, fileName.length() - 4));
|
176
|
+
|
177
|
+
Files.createDirectories(dir);
|
178
|
+
|
179
|
+
ConfigSource execConfig = newConfig()
|
180
|
+
.set("min_output_tasks", 1);
|
181
|
+
|
182
|
+
ConfigSource outConfig = newConfig()
|
183
|
+
.set("type", "file")
|
184
|
+
.set("path_prefix", dir.resolve("fragments_").toString())
|
185
|
+
.set("file_ext", "csv")
|
186
|
+
.set("formatter", newConfig()
|
187
|
+
.set("type", "csv")
|
188
|
+
.set("header_line", false)
|
189
|
+
.set("newline", "LF"));
|
190
|
+
|
191
|
+
ConfigSource config = newConfig()
|
192
|
+
.set("exec", execConfig)
|
193
|
+
.set("in", inConfig)
|
194
|
+
.set("out", outConfig);
|
195
|
+
|
196
|
+
// embed.run returns TestingBulkLoader.TestingExecutionResult because
|
197
|
+
RunResult result = (RunResult) embed.run(config);
|
198
|
+
|
199
|
+
try (OutputStream out = Files.newOutputStream(outputPath)) {
|
200
|
+
List<Path> fragments = new ArrayList<Path>();
|
201
|
+
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, "fragments_*.csv")) {
|
202
|
+
for (Path fragment : stream) {
|
203
|
+
fragments.add(fragment);
|
204
|
+
}
|
205
|
+
}
|
206
|
+
Collections.sort(fragments);
|
207
|
+
for (Path fragment : fragments) {
|
208
|
+
try (InputStream in = Files.newInputStream(fragment)) {
|
209
|
+
ByteStreams.copy(in, out);
|
210
|
+
}
|
211
|
+
}
|
212
|
+
}
|
213
|
+
|
214
|
+
return result;
|
215
|
+
}
|
216
|
+
|
217
|
+
// TODO add runOutput(ConfigSource outConfig, Path inputPath) where inputPath is a path to a CSV file
|
218
|
+
// whose column types can be naturally guessed using csv guess plugin. Callers use EmbulkTests.copyResource
|
219
|
+
// to copy a resource file to a temp file before calling it.
|
220
|
+
|
221
|
+
// TODO add runFilter(ConfigSource filterConfig, Path inputPath, Path outputPath) where inputPath is a path to
|
222
|
+
// a CSV file whose column types can be naturally guessed using csv guess plugin.
|
223
|
+
}
|
data/lib/embulk/version.rb
CHANGED
data/settings.gradle
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.15
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-11-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -148,9 +148,9 @@ files:
|
|
148
148
|
- classpath/commons-beanutils-core-1.8.3.jar
|
149
149
|
- classpath/commons-compress-1.10.jar
|
150
150
|
- classpath/commons-lang3-3.1.jar
|
151
|
-
- classpath/embulk-cli-0.8.
|
152
|
-
- classpath/embulk-core-0.8.
|
153
|
-
- classpath/embulk-standards-0.8.
|
151
|
+
- classpath/embulk-cli-0.8.15.jar
|
152
|
+
- classpath/embulk-core-0.8.15.jar
|
153
|
+
- classpath/embulk-standards-0.8.15.jar
|
154
154
|
- classpath/guava-18.0.jar
|
155
155
|
- classpath/guice-4.0.jar
|
156
156
|
- classpath/guice-bootstrap-0.1.1.jar
|
@@ -228,6 +228,7 @@ files:
|
|
228
228
|
- embulk-core/src/main/java/org/embulk/exec/ResumeState.java
|
229
229
|
- embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java
|
230
230
|
- embulk-core/src/main/java/org/embulk/exec/SetCurrentThreadName.java
|
231
|
+
- embulk-core/src/main/java/org/embulk/exec/SkipTransactionException.java
|
231
232
|
- embulk-core/src/main/java/org/embulk/exec/SystemConfigModule.java
|
232
233
|
- embulk-core/src/main/java/org/embulk/exec/TempFileAllocator.java
|
233
234
|
- embulk-core/src/main/java/org/embulk/exec/TransactionStage.java
|
@@ -468,6 +469,7 @@ files:
|
|
468
469
|
- embulk-docs/src/release/release-0.8.12.rst
|
469
470
|
- embulk-docs/src/release/release-0.8.13.rst
|
470
471
|
- embulk-docs/src/release/release-0.8.14.rst
|
472
|
+
- embulk-docs/src/release/release-0.8.15.rst
|
471
473
|
- embulk-docs/src/release/release-0.8.2.rst
|
472
474
|
- embulk-docs/src/release/release-0.8.3.rst
|
473
475
|
- embulk-docs/src/release/release-0.8.4.rst
|
@@ -498,6 +500,10 @@ files:
|
|
498
500
|
- embulk-standards/src/test/java/org/embulk/standards/TestCsvTokenizer.java
|
499
501
|
- embulk-standards/src/test/java/org/embulk/standards/TestJsonParserPlugin.java
|
500
502
|
- embulk-standards/src/test/java/org/embulk/standards/TestRenameFilterPlugin.java
|
503
|
+
- embulk-test/build.gradle
|
504
|
+
- embulk-test/src/main/java/org/embulk/test/EmbulkTests.java
|
505
|
+
- embulk-test/src/main/java/org/embulk/test/TestingBulkLoader.java
|
506
|
+
- embulk-test/src/main/java/org/embulk/test/TestingEmbulk.java
|
501
507
|
- embulk.gemspec
|
502
508
|
- gradle/wrapper/gradle-wrapper.jar
|
503
509
|
- gradle/wrapper/gradle-wrapper.properties
|