embulk 0.8.14-java → 0.8.15-java

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,6 @@
1
+ dependencies {
2
+ compile 'junit:junit:4.12'
3
+ compile 'org.hamcrest:hamcrest-library:1.3'
4
+
5
+ compile project(':embulk-core')
6
+ }
@@ -0,0 +1,75 @@
1
+ package org.embulk.test;
2
+
3
+ import com.google.common.base.Throwables;
4
+ import com.google.common.io.CharStreams;
5
+ import com.google.common.io.Resources;
6
+ import java.io.File;
7
+ import java.io.IOException;
8
+ import java.io.InputStream;
9
+ import java.io.InputStreamReader;
10
+ import java.nio.file.Files;
11
+ import java.nio.file.Path;
12
+ import java.util.Collections;
13
+ import java.util.List;
14
+ import org.embulk.EmbulkEmbed;
15
+ import org.embulk.config.ConfigSource;
16
+ import static com.google.common.base.Strings.isNullOrEmpty;
17
+ import static java.nio.charset.StandardCharsets.UTF_8;
18
+ import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
19
+ import static org.hamcrest.Matchers.is;
20
+ import static org.junit.Assume.assumeThat;
21
+
22
+ public class EmbulkTests
23
+ {
24
+ private EmbulkTests()
25
+ { }
26
+
27
+ public static ConfigSource config(String envName)
28
+ {
29
+ String path = System.getenv(envName);
30
+ assumeThat(isNullOrEmpty(path), is(false));
31
+ try {
32
+ return EmbulkEmbed.newSystemConfigLoader().fromYamlFile(new File(path));
33
+ } catch (IOException ex) {
34
+ throw Throwables.propagate(ex);
35
+ }
36
+ }
37
+
38
+ public static String readResource(String name)
39
+ {
40
+ try (InputStream in = Resources.getResource(name).openStream()) {
41
+ return CharStreams.toString(new InputStreamReader(in, UTF_8));
42
+ }
43
+ catch (IOException ex) {
44
+ throw new RuntimeException(ex);
45
+ }
46
+ }
47
+
48
+ public static void copyResource(String resource, Path dest)
49
+ throws IOException
50
+ {
51
+ Files.createDirectories(dest.getParent());
52
+ try (InputStream input = Resources.getResource(resource).openStream()) {
53
+ Files.copy(input, dest, REPLACE_EXISTING);
54
+ }
55
+ }
56
+
57
+ public static String readFile(Path path) throws IOException
58
+ {
59
+ try (InputStream in = Files.newInputStream(path)) {
60
+ return CharStreams.toString(new InputStreamReader(in, UTF_8));
61
+ }
62
+ }
63
+
64
+ public static String readSortedFile(Path path) throws IOException
65
+ {
66
+ List<String> lines = Files.readAllLines(path, UTF_8);
67
+ Collections.sort(lines);
68
+ StringBuilder sb = new StringBuilder();
69
+ for (String line : lines) {
70
+ sb.append(line);
71
+ sb.append("\n");
72
+ }
73
+ return sb.toString();
74
+ }
75
+ }
@@ -0,0 +1,124 @@
1
+ package org.embulk.test;
2
+
3
+ import com.google.common.base.Function;
4
+ import com.google.common.base.Optional;
5
+ import com.google.common.collect.ImmutableList;
6
+ import com.google.inject.Binder;
7
+ import com.google.inject.Inject;
8
+ import com.google.inject.Injector;
9
+ import com.google.inject.Module;
10
+ import com.google.inject.util.Modules;
11
+ import java.util.List;
12
+ import org.embulk.config.ConfigSource;
13
+ import org.embulk.config.TaskReport;
14
+ import org.embulk.exec.BulkLoader;
15
+ import org.embulk.exec.ExecutionResult;
16
+ import org.embulk.exec.ForSystemConfig;
17
+ import org.embulk.exec.ResumeState;
18
+ import org.embulk.spi.Exec;
19
+ import org.embulk.spi.ExecSession;
20
+ import org.embulk.spi.Schema;
21
+ import org.slf4j.Logger;
22
+
23
+ class TestingBulkLoader
24
+ extends BulkLoader
25
+ {
26
+ static Function<List<Module>, List<Module>> override()
27
+ {
28
+ return new Function<List<Module>, List<Module>>() {
29
+ @Override
30
+ public List<Module> apply(List<Module> modules)
31
+ {
32
+ Module override = new Module() {
33
+ public void configure(Binder binder)
34
+ {
35
+ binder.bind(BulkLoader.class).to(TestingBulkLoader.class);
36
+ }
37
+ };
38
+ return ImmutableList.of(Modules.override(modules).with(ImmutableList.of(override)));
39
+ }
40
+ };
41
+ }
42
+
43
+ @Inject
44
+ public TestingBulkLoader(Injector injector,
45
+ @ForSystemConfig ConfigSource systemConfig)
46
+ {
47
+ super(injector, systemConfig);
48
+ }
49
+
50
+ @Override
51
+ protected LoaderState newLoaderState(Logger logger, ProcessPluginSet plugins)
52
+ {
53
+ return new TestingLoaderState(logger, plugins);
54
+ }
55
+
56
+ protected static class TestingLoaderState
57
+ extends LoaderState
58
+ {
59
+ public TestingLoaderState(Logger logger, ProcessPluginSet plugins)
60
+ {
61
+ super(logger, plugins);
62
+ }
63
+
64
+ @Override
65
+ public ExecutionResult buildExecuteResultWithWarningException(Throwable ex)
66
+ {
67
+ ExecutionResult result = super.buildExecuteResultWithWarningException(ex);
68
+ return new TestingExecutionResult(result, buildResumeState(Exec.session()), Exec.session());
69
+ }
70
+ }
71
+
72
+ static class TestingExecutionResult
73
+ extends ExecutionResult
74
+ implements TestingEmbulk.RunResult
75
+ {
76
+ private final Schema inputSchema;
77
+ private final Schema outputSchema;
78
+ private final List<TaskReport> inputTaskReports;
79
+ private final List<TaskReport> outputTaskReports;
80
+
81
+ public TestingExecutionResult(ExecutionResult orig,
82
+ ResumeState resumeState, ExecSession session)
83
+ {
84
+ super(orig.getConfigDiff(), orig.isSkipped(), orig.getIgnoredExceptions());
85
+ this.inputSchema = resumeState.getInputSchema();
86
+ this.outputSchema = resumeState.getOutputSchema();
87
+ this.inputTaskReports = buildReports(resumeState.getInputTaskReports(), session);
88
+ this.outputTaskReports = buildReports(resumeState.getOutputTaskReports(), session);
89
+ }
90
+
91
+ private static List<TaskReport> buildReports(List<Optional<TaskReport>> optionalReports, ExecSession session)
92
+ {
93
+ ImmutableList.Builder<TaskReport> reports = ImmutableList.builder();
94
+ for (Optional<TaskReport> report : optionalReports) {
95
+ reports.add(report.or(session.newTaskReport()));
96
+ }
97
+ return reports.build();
98
+ }
99
+
100
+ @Override
101
+ public Schema getInputSchema()
102
+ {
103
+ return inputSchema;
104
+ }
105
+
106
+ @Override
107
+ public Schema getOutputSchema()
108
+ {
109
+ return outputSchema;
110
+ }
111
+
112
+ @Override
113
+ public List<TaskReport> getInputTaskReports()
114
+ {
115
+ return inputTaskReports;
116
+ }
117
+
118
+ @Override
119
+ public List<TaskReport> getOutputTaskReports()
120
+ {
121
+ return outputTaskReports;
122
+ }
123
+ }
124
+ }
@@ -0,0 +1,223 @@
1
+ package org.embulk.test;
2
+
3
+ import com.google.common.collect.ImmutableList;
4
+ import com.google.common.base.Optional;
5
+ import com.google.common.io.ByteStreams;
6
+ import com.google.inject.Binder;
7
+ import com.google.inject.Injector;
8
+ import com.google.inject.Module;
9
+ import java.io.IOException;
10
+ import java.io.InputStream;
11
+ import java.io.OutputStream;
12
+ import java.nio.file.DirectoryStream;
13
+ import java.nio.file.Files;
14
+ import java.nio.file.Path;
15
+ import java.util.ArrayList;
16
+ import java.util.Collections;
17
+ import java.util.List;
18
+ import org.embulk.EmbulkEmbed;
19
+ import org.embulk.config.Config;
20
+ import org.embulk.config.ConfigDiff;
21
+ import org.embulk.config.ConfigLoader;
22
+ import org.embulk.config.ConfigSource;
23
+ import org.embulk.config.TaskReport;
24
+ import org.embulk.exec.ExecutionResult;
25
+ import org.embulk.spi.Schema;
26
+ import org.embulk.spi.TempFileException;
27
+ import org.embulk.spi.TempFileSpace;
28
+ import org.junit.rules.TestRule;
29
+ import org.junit.rules.TestWatcher;
30
+ import org.junit.runner.Description;
31
+ import org.junit.runners.model.Statement;
32
+ import static com.google.common.base.Preconditions.checkArgument;
33
+ import static org.embulk.plugin.InjectedPluginSource.registerPluginTo;
34
+
35
+ public class TestingEmbulk
36
+ implements TestRule
37
+ {
38
+ public static class Builder
39
+ {
40
+ private List<Module> modules = new ArrayList<>();
41
+
42
+ Builder()
43
+ { }
44
+
45
+ public <T> Builder registerPlugin(final Class<T> iface, final String name, final Class<?> impl)
46
+ {
47
+ modules.add(new Module() {
48
+ public void configure(Binder binder)
49
+ {
50
+ registerPluginTo(binder, iface, name, impl);
51
+ }
52
+ });
53
+ return this;
54
+ }
55
+
56
+ public TestingEmbulk build()
57
+ {
58
+ return new TestingEmbulk(this);
59
+ }
60
+ }
61
+
62
+ public static Builder builder()
63
+ {
64
+ return new Builder();
65
+ }
66
+
67
+ private final List<Module> modules;
68
+
69
+ private EmbulkEmbed embed;
70
+ private TempFileSpace tempFiles;
71
+
72
+ TestingEmbulk(Builder builder)
73
+ {
74
+ this.modules = ImmutableList.copyOf(builder.modules);
75
+ reset();
76
+ }
77
+
78
+ public void reset()
79
+ {
80
+ destroy();
81
+
82
+ this.embed = new EmbulkEmbed.Bootstrap()
83
+ .addModules(modules)
84
+ .overrideModules(TestingBulkLoader.override())
85
+ .initializeCloseable();
86
+
87
+ try {
88
+ this.tempFiles = new TempFileSpace(Files.createTempDirectory("embulk-test-temp-").toFile());
89
+ }
90
+ catch (IOException ex) {
91
+ throw new TempFileException(ex);
92
+ }
93
+ }
94
+
95
+ public void destroy()
96
+ {
97
+ if (embed != null) {
98
+ embed.destroy();
99
+ embed = null;
100
+ }
101
+ if (tempFiles != null) {
102
+ tempFiles.cleanup();
103
+ tempFiles = null;
104
+ }
105
+ }
106
+
107
+ @Override
108
+ public Statement apply(Statement base, Description description)
109
+ {
110
+ return new EmbulkTestingEmbedWatcher().apply(base, description);
111
+ }
112
+
113
+ private class EmbulkTestingEmbedWatcher
114
+ extends TestWatcher
115
+ {
116
+ @Override
117
+ protected void starting(Description description)
118
+ {
119
+ reset();
120
+ }
121
+
122
+ @Override
123
+ protected void finished(Description description)
124
+ {
125
+ destroy();
126
+ }
127
+ }
128
+
129
+ public Path createTempFile(String suffix)
130
+ {
131
+ return tempFiles.createTempFile(suffix).toPath();
132
+ }
133
+
134
+ public Injector injector()
135
+ {
136
+ return embed.getInjector();
137
+ }
138
+
139
+ public ConfigLoader configLoader()
140
+ {
141
+ return embed.newConfigLoader();
142
+ }
143
+
144
+ public ConfigSource newConfig()
145
+ {
146
+ return configLoader().newConfigSource();
147
+ }
148
+
149
+ public ConfigSource loadYamlResource(String name)
150
+ {
151
+ return configLoader()
152
+ .fromYamlString(EmbulkTests.readResource(name));
153
+ }
154
+
155
+ public static interface RunResult
156
+ {
157
+ ConfigDiff getConfigDiff();
158
+
159
+ List<Throwable> getIgnoredExceptions();
160
+
161
+ Schema getInputSchema();
162
+
163
+ Schema getOutputSchema();
164
+
165
+ List<TaskReport> getInputTaskReports();
166
+
167
+ List<TaskReport> getOutputTaskReports();
168
+ }
169
+
170
+ public RunResult runInput(ConfigSource inConfig, Path outputPath)
171
+ throws IOException
172
+ {
173
+ String fileName = outputPath.getFileName().toString();
174
+ checkArgument(fileName.endsWith(".csv"), "outputPath must end with .csv");
175
+ Path dir = outputPath.getParent().resolve(fileName.substring(0, fileName.length() - 4));
176
+
177
+ Files.createDirectories(dir);
178
+
179
+ ConfigSource execConfig = newConfig()
180
+ .set("min_output_tasks", 1);
181
+
182
+ ConfigSource outConfig = newConfig()
183
+ .set("type", "file")
184
+ .set("path_prefix", dir.resolve("fragments_").toString())
185
+ .set("file_ext", "csv")
186
+ .set("formatter", newConfig()
187
+ .set("type", "csv")
188
+ .set("header_line", false)
189
+ .set("newline", "LF"));
190
+
191
+ ConfigSource config = newConfig()
192
+ .set("exec", execConfig)
193
+ .set("in", inConfig)
194
+ .set("out", outConfig);
195
+
196
+ // embed.run returns TestingBulkLoader.TestingExecutionResult because
197
+ RunResult result = (RunResult) embed.run(config);
198
+
199
+ try (OutputStream out = Files.newOutputStream(outputPath)) {
200
+ List<Path> fragments = new ArrayList<Path>();
201
+ try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, "fragments_*.csv")) {
202
+ for (Path fragment : stream) {
203
+ fragments.add(fragment);
204
+ }
205
+ }
206
+ Collections.sort(fragments);
207
+ for (Path fragment : fragments) {
208
+ try (InputStream in = Files.newInputStream(fragment)) {
209
+ ByteStreams.copy(in, out);
210
+ }
211
+ }
212
+ }
213
+
214
+ return result;
215
+ }
216
+
217
+ // TODO add runOutput(ConfigSource outConfig, Path inputPath) where inputPath is a path to a CSV file
218
+ // whose column types can be naturally guessed using csv guess plugin. Callers use EmbulkTests.copyResource
219
+ // to copy a resource file to a temp file before calling it.
220
+
221
+ // TODO add runFilter(ConfigSource filterConfig, Path inputPath, Path outputPath) where inputPath is a path to
222
+ // a CSV file whose column types can be naturally guessed using csv guess plugin.
223
+ }
@@ -1,3 +1,3 @@
1
1
  module Embulk
2
- VERSION = '0.8.14'
2
+ VERSION = '0.8.15'
3
3
  end
@@ -3,4 +3,5 @@ rootProject.name = 'embulk'
3
3
  include 'embulk-core'
4
4
  include 'embulk-standards'
5
5
  include 'embulk-cli'
6
+ include 'embulk-test'
6
7
  include 'embulk-docs'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.14
4
+ version: 0.8.15
5
5
  platform: java
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-10-22 00:00:00.000000000 Z
11
+ date: 2016-11-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -148,9 +148,9 @@ files:
148
148
  - classpath/commons-beanutils-core-1.8.3.jar
149
149
  - classpath/commons-compress-1.10.jar
150
150
  - classpath/commons-lang3-3.1.jar
151
- - classpath/embulk-cli-0.8.14.jar
152
- - classpath/embulk-core-0.8.14.jar
153
- - classpath/embulk-standards-0.8.14.jar
151
+ - classpath/embulk-cli-0.8.15.jar
152
+ - classpath/embulk-core-0.8.15.jar
153
+ - classpath/embulk-standards-0.8.15.jar
154
154
  - classpath/guava-18.0.jar
155
155
  - classpath/guice-4.0.jar
156
156
  - classpath/guice-bootstrap-0.1.1.jar
@@ -228,6 +228,7 @@ files:
228
228
  - embulk-core/src/main/java/org/embulk/exec/ResumeState.java
229
229
  - embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java
230
230
  - embulk-core/src/main/java/org/embulk/exec/SetCurrentThreadName.java
231
+ - embulk-core/src/main/java/org/embulk/exec/SkipTransactionException.java
231
232
  - embulk-core/src/main/java/org/embulk/exec/SystemConfigModule.java
232
233
  - embulk-core/src/main/java/org/embulk/exec/TempFileAllocator.java
233
234
  - embulk-core/src/main/java/org/embulk/exec/TransactionStage.java
@@ -468,6 +469,7 @@ files:
468
469
  - embulk-docs/src/release/release-0.8.12.rst
469
470
  - embulk-docs/src/release/release-0.8.13.rst
470
471
  - embulk-docs/src/release/release-0.8.14.rst
472
+ - embulk-docs/src/release/release-0.8.15.rst
471
473
  - embulk-docs/src/release/release-0.8.2.rst
472
474
  - embulk-docs/src/release/release-0.8.3.rst
473
475
  - embulk-docs/src/release/release-0.8.4.rst
@@ -498,6 +500,10 @@ files:
498
500
  - embulk-standards/src/test/java/org/embulk/standards/TestCsvTokenizer.java
499
501
  - embulk-standards/src/test/java/org/embulk/standards/TestJsonParserPlugin.java
500
502
  - embulk-standards/src/test/java/org/embulk/standards/TestRenameFilterPlugin.java
503
+ - embulk-test/build.gradle
504
+ - embulk-test/src/main/java/org/embulk/test/EmbulkTests.java
505
+ - embulk-test/src/main/java/org/embulk/test/TestingBulkLoader.java
506
+ - embulk-test/src/main/java/org/embulk/test/TestingEmbulk.java
501
507
  - embulk.gemspec
502
508
  - gradle/wrapper/gradle-wrapper.jar
503
509
  - gradle/wrapper/gradle-wrapper.properties