embulk 0.8.18-java → 0.8.19-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +10 -0
- data/build.gradle +10 -3
- data/embulk-cli/build.gradle +2 -0
- data/embulk-cli/src/main/bat/selfrun.bat +98 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkExample.java +82 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkMigrate.java +458 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkNew.java +419 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkSelfUpdate.java +248 -0
- data/embulk-cli/src/main/sh/selfrun.sh +0 -103
- data/embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java +158 -143
- data/embulk-core/build.gradle +2 -2
- data/embulk-core/src/main/java/org/embulk/EmbulkVersion.java +109 -0
- data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java +11 -0
- data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java +29 -3
- data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java +47 -13
- data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java +6 -3
- data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java +385 -64
- data/embulk-core/src/main/java/org/embulk/spi/TempFileSpace.java +2 -1
- data/embulk-core/src/test/java/org/embulk/spi/TestPageBuilderReader.java +62 -0
- data/embulk-docs/src/built-in.rst +59 -21
- data/embulk-docs/src/customization.rst +8 -8
- data/embulk-docs/src/developers/index.rst +45 -0
- data/embulk-docs/src/index.rst +11 -7
- data/embulk-docs/src/recipe.rst +1 -1
- data/embulk-docs/src/recipe/{scheduled-csv-load-to-elasticsearch-kibana4.rst → scheduled-csv-load-to-elasticsearch-kibana5.rst} +26 -24
- data/embulk-docs/src/release.rst +1 -0
- data/embulk-docs/src/release/release-0.4.0.rst +1 -1
- data/embulk-docs/src/release/release-0.5.0.rst +1 -1
- data/embulk-docs/src/release/release-0.6.0.rst +1 -1
- data/embulk-docs/src/release/release-0.6.20.rst +1 -1
- data/embulk-docs/src/release/release-0.8.19.rst +43 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +2 -2
- data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java +30 -1
- data/embulk-standards/src/test/java/org/embulk/standards/guess/TestCsvGuessPlugin.java +10 -0
- data/embulk-standards/src/test/java/org/embulk/standards/preview/TestFilePreview.java +73 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records.csv +5 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_guessed.yml +2 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_seed.yml +1 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes.csv +5 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_exec.yml +1 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_load.yml +19 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_previewed.csv +1 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple.csv +5 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_load.yml +19 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_previewed.csv +4 -0
- data/embulk-test/src/main/java/org/embulk/test/PreviewResultInputPlugin.java +65 -0
- data/embulk-test/src/main/java/org/embulk/test/TestingBulkLoader.java +5 -0
- data/embulk-test/src/main/java/org/embulk/test/TestingEmbulk.java +59 -2
- data/embulk.gemspec +2 -1
- data/lib/embulk/command/embulk_run.rb +11 -49
- data/lib/embulk/data/new/README.md.vm +106 -0
- data/lib/embulk/data/new/{gitignore.erb → gitignore.vm} +3 -3
- data/lib/embulk/data/new/java/{build.gradle.erb → build.gradle.vm} +8 -8
- data/lib/embulk/data/new/java/{decoder.java.erb → decoder.java.vm} +6 -4
- data/lib/embulk/data/new/java/{encoder.java.erb → encoder.java.vm} +7 -5
- data/lib/embulk/data/new/java/{file_input.java.erb → file_input.java.vm} +9 -7
- data/lib/embulk/data/new/java/{file_output.java.erb → file_output.java.vm} +7 -5
- data/lib/embulk/data/new/java/{filter.java.erb → filter.java.vm} +4 -3
- data/lib/embulk/data/new/java/{formatter.java.erb → formatter.java.vm} +5 -4
- data/lib/embulk/data/new/java/{input.java.erb → input.java.vm} +6 -4
- data/lib/embulk/data/new/java/{output.java.erb → output.java.vm} +7 -5
- data/lib/embulk/data/new/java/{parser.java.erb → parser.java.vm} +5 -4
- data/lib/embulk/data/new/java/plugin_loader.rb.vm +3 -0
- data/lib/embulk/data/new/java/test.java.vm +5 -0
- data/lib/embulk/data/new/ruby/decoder_guess.rb.vm +25 -0
- data/lib/embulk/data/new/ruby/{filter.rb.erb → filter.rb.vm} +2 -2
- data/lib/embulk/data/new/ruby/{formatter.rb.erb → formatter.rb.vm} +2 -2
- data/lib/embulk/data/new/ruby/gemspec.vm +20 -0
- data/lib/embulk/data/new/ruby/{input.rb.erb → input.rb.vm} +10 -10
- data/lib/embulk/data/new/ruby/{output.rb.erb → output.rb.vm} +7 -7
- data/lib/embulk/data/new/ruby/{parser.rb.erb → parser.rb.vm} +2 -2
- data/lib/embulk/data/new/ruby/parser_guess.rb.vm +65 -0
- data/lib/embulk/guess/csv.rb +5 -0
- data/lib/embulk/version.rb +22 -1
- metadata +55 -35
- data/lib/embulk/command/embulk_example.rb +0 -33
- data/lib/embulk/command/embulk_generate_bin.rb +0 -62
- data/lib/embulk/command/embulk_migrate_plugin.rb +0 -244
- data/lib/embulk/command/embulk_new_plugin.rb +0 -126
- data/lib/embulk/command/embulk_selfupdate.rb +0 -121
- data/lib/embulk/data/new/README.md.erb +0 -111
- data/lib/embulk/data/new/java/plugin_loader.rb.erb +0 -3
- data/lib/embulk/data/new/java/test.java.erb +0 -5
- data/lib/embulk/data/new/ruby/decoder_guess.rb.erb +0 -25
- data/lib/embulk/data/new/ruby/gemspec.erb +0 -20
- data/lib/embulk/data/new/ruby/parser_guess.rb.erb +0 -65
@@ -0,0 +1,419 @@
|
|
1
|
+
package org.embulk.cli;
|
2
|
+
|
3
|
+
import java.io.BufferedWriter;
|
4
|
+
import java.io.IOException;
|
5
|
+
import java.io.InputStreamReader;
|
6
|
+
import java.nio.charset.StandardCharsets;
|
7
|
+
import java.nio.file.Files;
|
8
|
+
import java.nio.file.FileVisitResult;
|
9
|
+
import java.nio.file.Path;
|
10
|
+
import java.nio.file.Paths;
|
11
|
+
import java.nio.file.SimpleFileVisitor;
|
12
|
+
import java.nio.file.attribute.BasicFileAttributes;
|
13
|
+
import java.nio.file.attribute.PosixFilePermission;
|
14
|
+
import java.util.ArrayList;
|
15
|
+
import java.util.HashMap;
|
16
|
+
import java.util.HashSet;
|
17
|
+
import java.util.Map;
|
18
|
+
import java.util.Set;
|
19
|
+
|
20
|
+
import com.google.common.base.CaseFormat;
|
21
|
+
import com.google.common.base.Joiner;
|
22
|
+
import com.google.common.io.CharStreams;
|
23
|
+
|
24
|
+
import org.apache.velocity.VelocityContext;
|
25
|
+
import org.apache.velocity.app.VelocityEngine;
|
26
|
+
|
27
|
+
public class EmbulkNew
|
28
|
+
{
|
29
|
+
public EmbulkNew(final String categoryWithLanguage, final String nameGiven, final String embulkVersion)
|
30
|
+
throws IOException
|
31
|
+
{
|
32
|
+
this.basePath = Paths.get(".").toAbsolutePath();
|
33
|
+
|
34
|
+
final LanguageAndCategory languageAndCategory = LanguageAndCategory.of(categoryWithLanguage);
|
35
|
+
this.language = languageAndCategory.getLanguage();
|
36
|
+
this.category = languageAndCategory.getCategory();
|
37
|
+
this.nameGiven = nameGiven;
|
38
|
+
this.embulkVersion = embulkVersion;
|
39
|
+
|
40
|
+
if (category.equals("file_input")) {
|
41
|
+
this.embulkCategory = "input";
|
42
|
+
}
|
43
|
+
else if (category.equals("file_output")) {
|
44
|
+
this.embulkCategory = "output";
|
45
|
+
}
|
46
|
+
else {
|
47
|
+
this.embulkCategory = category;
|
48
|
+
}
|
49
|
+
|
50
|
+
this.name = nameGiven.replaceAll("[^a-zA-Z0-9_]+", "_");
|
51
|
+
|
52
|
+
this.fullProjectName = "embulk-" + embulkCategory + "-" + name;
|
53
|
+
this.pluginDirectory = "lib/embulk";
|
54
|
+
this.pluginPath = pluginDirectory + "/" + embulkCategory + "/" + name + ".rb";
|
55
|
+
|
56
|
+
this.pluginBasePath = this.basePath.resolve(fullProjectName);
|
57
|
+
|
58
|
+
this.velocityEngine = new VelocityEngine();
|
59
|
+
this.velocityEngine.init();
|
60
|
+
this.velocityEngine.setProperty(VelocityEngine.RUNTIME_LOG_LOGSYSTEM_CLASS,
|
61
|
+
"org.apache.velocity.runtime.log.NullLogSystem");
|
62
|
+
}
|
63
|
+
|
64
|
+
public boolean newPlugin()
|
65
|
+
throws IOException
|
66
|
+
{
|
67
|
+
if (Files.exists(this.pluginBasePath)) {
|
68
|
+
throw new IOException("./" + this.fullProjectName + " already exists. Please delete it first.");
|
69
|
+
}
|
70
|
+
|
71
|
+
Files.createDirectories(this.pluginBasePath);
|
72
|
+
|
73
|
+
System.out.println("Creating " + this.fullProjectName + "/");
|
74
|
+
|
75
|
+
boolean success = false;
|
76
|
+
try {
|
77
|
+
//
|
78
|
+
// Generate gemspec
|
79
|
+
//
|
80
|
+
final String author = getGitConfig("user.name", "YOUR_NAME");
|
81
|
+
final String email = getGitConfig("user.email", "YOUR_NAME");
|
82
|
+
final String expectedGitHubAccount = email.split("@")[0];
|
83
|
+
|
84
|
+
// variables used in Velocity templates
|
85
|
+
final String rubyClassName = CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, name);
|
86
|
+
final String javaClassName =
|
87
|
+
CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, name) +
|
88
|
+
CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, category) +
|
89
|
+
"Plugin";
|
90
|
+
final String javaPackageName = "org.embulk." + embulkCategory + "." + name;
|
91
|
+
final String displayName = getDisplayName(name);
|
92
|
+
final String displayCategory = category.replace("_", " ");
|
93
|
+
|
94
|
+
final HashMap<String, String> extraGuesses = new HashMap<String, String>();
|
95
|
+
|
96
|
+
final String description;
|
97
|
+
switch (category) {
|
98
|
+
case "input":
|
99
|
+
description = String.format("Loads records from %s.", displayName);
|
100
|
+
break;
|
101
|
+
case "file_input":
|
102
|
+
description = String.format("Reads files stored on %s.", displayName);
|
103
|
+
break;
|
104
|
+
case "parser":
|
105
|
+
description = String.format("Parses %s files read by other file input plugins.", displayName);
|
106
|
+
extraGuesses.put("embulk/data/new/ruby/parser_guess.rb.vm",
|
107
|
+
String.format("%s/guess/%s.rb", pluginDirectory, name));
|
108
|
+
break;
|
109
|
+
case "decoder":
|
110
|
+
description = String.format("Decodes %s-encoded files read by other file input plugins.", displayName);
|
111
|
+
extraGuesses.put("embulk/data/new/ruby/decoder_guess.rb.vm",
|
112
|
+
String.format("%s/guess/%s.rb", pluginDirectory, name));
|
113
|
+
break;
|
114
|
+
case "output":
|
115
|
+
description = String.format("Dumps records to %s.", displayName);
|
116
|
+
break;
|
117
|
+
case "file_output":
|
118
|
+
description = String.format("Stores files on %s.", displayName);
|
119
|
+
break;
|
120
|
+
case "formatter":
|
121
|
+
description = String.format("Formats %s files for other file output plugins.", displayName);
|
122
|
+
break;
|
123
|
+
case "encoder":
|
124
|
+
description = String.format("Encodes files using %s for other file output plugins.", displayName);
|
125
|
+
break;
|
126
|
+
case "filter":
|
127
|
+
description = String.format("%s", displayName);
|
128
|
+
break;
|
129
|
+
default:
|
130
|
+
throw new RuntimeException("FATAL: Invalid plugin category.");
|
131
|
+
}
|
132
|
+
|
133
|
+
//
|
134
|
+
// Generate project repository
|
135
|
+
//
|
136
|
+
final VelocityContext velocityContext = createVelocityContext(
|
137
|
+
author,
|
138
|
+
category,
|
139
|
+
description,
|
140
|
+
displayName,
|
141
|
+
displayCategory,
|
142
|
+
email,
|
143
|
+
embulkCategory,
|
144
|
+
this.embulkVersion,
|
145
|
+
expectedGitHubAccount,
|
146
|
+
fullProjectName,
|
147
|
+
javaClassName,
|
148
|
+
javaPackageName,
|
149
|
+
language,
|
150
|
+
name,
|
151
|
+
rubyClassName);
|
152
|
+
copyTemplated("embulk/data/new/README.md.vm", "README.md", velocityContext);
|
153
|
+
copy("embulk/data/new/LICENSE.txt", "LICENSE.txt");
|
154
|
+
copyTemplated("embulk/data/new/gitignore.vm", ".gitignore", velocityContext);
|
155
|
+
|
156
|
+
switch (language) {
|
157
|
+
case "ruby":
|
158
|
+
copy("embulk/data/new/ruby/Rakefile", "Rakefile");
|
159
|
+
copy("embulk/data/new/ruby/Gemfile", "Gemfile");
|
160
|
+
copy("embulk/data/new/ruby/.ruby-version", ".ruby-version");
|
161
|
+
copyTemplated("embulk/data/new/ruby/gemspec.vm",
|
162
|
+
fullProjectName + ".gemspec",
|
163
|
+
velocityContext);
|
164
|
+
copyTemplated(String.format("embulk/data/new/ruby/%s.rb.vm", category),
|
165
|
+
this.pluginPath,
|
166
|
+
velocityContext);
|
167
|
+
break;
|
168
|
+
case "java":
|
169
|
+
copy("embulk/data/new/java/gradle/wrapper/gradle-wrapper.jar",
|
170
|
+
"gradle/wrapper/gradle-wrapper.jar");
|
171
|
+
copy("embulk/data/new/java/gradle/wrapper/gradle-wrapper.properties",
|
172
|
+
"gradle/wrapper/gradle-wrapper.properties");
|
173
|
+
copy("embulk/data/new/java/gradlew.bat", "gradlew.bat");
|
174
|
+
copy("embulk/data/new/java/gradlew", "gradlew");
|
175
|
+
setExecutable("gradlew");
|
176
|
+
copy("embulk/data/new/java/config/checkstyle/checkstyle.xml", "config/checkstyle/checkstyle.xml");
|
177
|
+
copy("embulk/data/new/java/config/checkstyle/default.xml", "config/checkstyle/default.xml");
|
178
|
+
copyTemplated("embulk/data/new/java/build.gradle.vm", "build.gradle", velocityContext);
|
179
|
+
copyTemplated("embulk/data/new/java/plugin_loader.rb.vm", this.pluginPath, velocityContext);
|
180
|
+
copyTemplated(String.format("embulk/data/new/java/%s.java.vm", category),
|
181
|
+
String.format("src/main/java/%s/%s.java",
|
182
|
+
javaPackageName.replaceAll("\\.", "/"),
|
183
|
+
javaClassName),
|
184
|
+
velocityContext);
|
185
|
+
copyTemplated("embulk/data/new/java/test.java.vm",
|
186
|
+
String.format("src/test/java/%s/Test%s.java",
|
187
|
+
javaPackageName.replaceAll("\\.", "/"),
|
188
|
+
javaClassName),
|
189
|
+
velocityContext);
|
190
|
+
break;
|
191
|
+
}
|
192
|
+
|
193
|
+
for (Map.Entry<String, String> entry : extraGuesses.entrySet()) {
|
194
|
+
copyTemplated(entry.getKey(), entry.getValue(), velocityContext);
|
195
|
+
}
|
196
|
+
|
197
|
+
System.out.println("");
|
198
|
+
System.out.println("Plugin template is successfully generated.");
|
199
|
+
|
200
|
+
switch (language) {
|
201
|
+
case "ruby":
|
202
|
+
System.out.println("Next steps:");
|
203
|
+
System.out.println("");
|
204
|
+
System.out.printf(" $ cd %s\n", fullProjectName);
|
205
|
+
System.out.println(" $ bundle install # install one using rbenv & rbenv-build");
|
206
|
+
System.out.println(" $ bundle exec rake # build gem to be released");
|
207
|
+
System.out.println(" $ bundle exec embulk run config.yml # you can run plugin using this command");
|
208
|
+
break;
|
209
|
+
case "java":
|
210
|
+
System.out.println("Next steps:");
|
211
|
+
System.out.println("");
|
212
|
+
System.out.printf(" $ cd %s\n", fullProjectName);
|
213
|
+
System.out.println(" $ ./gradlew package");
|
214
|
+
}
|
215
|
+
|
216
|
+
success = true;
|
217
|
+
System.out.println("");
|
218
|
+
}
|
219
|
+
catch (Exception ex) {
|
220
|
+
ex.printStackTrace();
|
221
|
+
}
|
222
|
+
finally {
|
223
|
+
if (!success) {
|
224
|
+
System.out.println("Failed. Removing the directory created.");
|
225
|
+
deleteDirectoryTree(Paths.get(fullProjectName));
|
226
|
+
}
|
227
|
+
}
|
228
|
+
return success;
|
229
|
+
}
|
230
|
+
|
231
|
+
private static class LanguageAndCategory
|
232
|
+
{
|
233
|
+
private LanguageAndCategory(final String language, final String category)
|
234
|
+
{
|
235
|
+
this.language = language;
|
236
|
+
this.category = category;
|
237
|
+
}
|
238
|
+
|
239
|
+
public static LanguageAndCategory of(final String categoryWithLanguage)
|
240
|
+
{
|
241
|
+
switch (categoryWithLanguage) {
|
242
|
+
case "java-input": return new LanguageAndCategory("java", "input");
|
243
|
+
case "java-output": return new LanguageAndCategory("java", "output");
|
244
|
+
case "java-filter": return new LanguageAndCategory("java", "filter");
|
245
|
+
case "java-file-input": return new LanguageAndCategory("java", "file_input");
|
246
|
+
case "java-file-output": return new LanguageAndCategory("java", "file_output");
|
247
|
+
case "java-parser": return new LanguageAndCategory("java", "parser");
|
248
|
+
case "java-formatter": return new LanguageAndCategory("java", "formatter");
|
249
|
+
case "java-decoder": return new LanguageAndCategory("java", "decoder");
|
250
|
+
case "java-encoder": return new LanguageAndCategory("java", "encoder");
|
251
|
+
case "ruby-input": return new LanguageAndCategory("ruby", "input");
|
252
|
+
case "ruby-output": return new LanguageAndCategory("ruby", "output");
|
253
|
+
case "ruby-filter": return new LanguageAndCategory("ruby", "filter");
|
254
|
+
case "ruby-file-input":
|
255
|
+
throw new RuntimeException("ruby-file-input is not implemented yet. See #21 on github.");
|
256
|
+
case "ruby-file-output":
|
257
|
+
throw new RuntimeException("ruby-file-output is not implemented yet. See #22 on github.");
|
258
|
+
case "ruby-parser": return new LanguageAndCategory("ruby", "parser");
|
259
|
+
case "ruby-formatter": return new LanguageAndCategory("ruby", "formatter");
|
260
|
+
case "ruby-decoder":
|
261
|
+
throw new RuntimeException("ruby-decoder is not implemented yet. See #31 on github.");
|
262
|
+
case "ruby-encoder":
|
263
|
+
throw new RuntimeException("ruby-decoder is not implemented yet. See #32 on github.");
|
264
|
+
default:
|
265
|
+
throw new RuntimeException(String.format("Unknown category '%s'", categoryWithLanguage));
|
266
|
+
}
|
267
|
+
}
|
268
|
+
|
269
|
+
public String getLanguage()
|
270
|
+
{
|
271
|
+
return this.language;
|
272
|
+
}
|
273
|
+
|
274
|
+
public String getCategory()
|
275
|
+
{
|
276
|
+
return this.category;
|
277
|
+
}
|
278
|
+
|
279
|
+
private final String language;
|
280
|
+
private final String category;
|
281
|
+
}
|
282
|
+
|
283
|
+
private String getGitConfig(final String configName, final String defaultValue)
|
284
|
+
{
|
285
|
+
try {
|
286
|
+
final Process process = new ProcessBuilder("git", "config", configName).redirectErrorStream(true).start();
|
287
|
+
return CharStreams.toString(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8)).trim();
|
288
|
+
}
|
289
|
+
catch (Throwable ex) {
|
290
|
+
return "YOUR_NAME";
|
291
|
+
}
|
292
|
+
}
|
293
|
+
|
294
|
+
private Path deleteDirectoryTree(final Path path)
|
295
|
+
throws IOException
|
296
|
+
{
|
297
|
+
return Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
|
298
|
+
@Override
|
299
|
+
public FileVisitResult visitFile(Path file, BasicFileAttributes attributes)
|
300
|
+
throws IOException
|
301
|
+
{
|
302
|
+
Files.delete(file);
|
303
|
+
return FileVisitResult.CONTINUE;
|
304
|
+
}
|
305
|
+
|
306
|
+
@Override
|
307
|
+
public FileVisitResult postVisitDirectory(Path directory, IOException exception)
|
308
|
+
throws IOException
|
309
|
+
{
|
310
|
+
Files.delete(directory);
|
311
|
+
return FileVisitResult.CONTINUE;
|
312
|
+
}
|
313
|
+
});
|
314
|
+
}
|
315
|
+
|
316
|
+
private String getDisplayName(final String name)
|
317
|
+
{
|
318
|
+
final String[] nameSplit = name.split("_");
|
319
|
+
final ArrayList<String> nameComposition = new ArrayList<String>();
|
320
|
+
for (String namePart : nameSplit) {
|
321
|
+
nameComposition.add(CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, namePart));
|
322
|
+
}
|
323
|
+
return Joiner.on(" ").join(nameComposition);
|
324
|
+
}
|
325
|
+
|
326
|
+
private VelocityContext createVelocityContext(final String author,
|
327
|
+
final String category,
|
328
|
+
final String description,
|
329
|
+
final String displayName,
|
330
|
+
final String displayCategory,
|
331
|
+
final String email,
|
332
|
+
final String embulkCategory,
|
333
|
+
final String embulkVersion,
|
334
|
+
final String expectedGitHubAccount,
|
335
|
+
final String fullProjectName,
|
336
|
+
final String javaClassName,
|
337
|
+
final String javaPackageName,
|
338
|
+
final String language,
|
339
|
+
final String name,
|
340
|
+
final String rubyClassName)
|
341
|
+
{
|
342
|
+
final VelocityContext velocityContext = new VelocityContext();
|
343
|
+
// TODO(dmikurube): Revisit this |argumentToRunEmbulkJava|.
|
344
|
+
// This is in the Velocity context because the value could not be in Velocity templates.
|
345
|
+
velocityContext.put("argumentToRunEmbulkJava", "\'-L ${file(\".\").absolutePath}\'");
|
346
|
+
velocityContext.put("author", author);
|
347
|
+
velocityContext.put("category", category);
|
348
|
+
velocityContext.put("description", description);
|
349
|
+
velocityContext.put("displayName", displayName);
|
350
|
+
velocityContext.put("displayCategory", displayCategory);
|
351
|
+
velocityContext.put("email", email);
|
352
|
+
velocityContext.put("embulkCategory", embulkCategory);
|
353
|
+
velocityContext.put("embulkVersion", embulkVersion);
|
354
|
+
velocityContext.put("expectedGitHubAccount", expectedGitHubAccount);
|
355
|
+
velocityContext.put("fullProjectName", fullProjectName);
|
356
|
+
velocityContext.put("javaClassName", javaClassName);
|
357
|
+
velocityContext.put("javaGuessClassName", javaClassName.replace("Plugin", "GuessPlugin"));
|
358
|
+
velocityContext.put("javaPackageName", javaPackageName);
|
359
|
+
velocityContext.put("language", language);
|
360
|
+
velocityContext.put("name", name);
|
361
|
+
velocityContext.put("rubyClassName", rubyClassName);
|
362
|
+
velocityContext.put("rubyGuessClassName", rubyClassName.replace("Plugin", "GuessPlugin"));
|
363
|
+
return velocityContext;
|
364
|
+
}
|
365
|
+
|
366
|
+
private void copy(String sourceResourcePath, String destinationFileName)
|
367
|
+
throws IOException
|
368
|
+
{
|
369
|
+
final Path destinationPath = this.pluginBasePath.resolve(destinationFileName);
|
370
|
+
Files.createDirectories(destinationPath.getParent());
|
371
|
+
Files.copy(EmbulkNew.class.getClassLoader().getResourceAsStream(sourceResourcePath), destinationPath);
|
372
|
+
}
|
373
|
+
|
374
|
+
private void copyTemplated(String sourceResourcePath, String destinationFileName, VelocityContext velocityContext)
|
375
|
+
throws IOException
|
376
|
+
{
|
377
|
+
try (InputStreamReader reader = new InputStreamReader(
|
378
|
+
EmbulkNew.class.getClassLoader().getResourceAsStream(sourceResourcePath))) {
|
379
|
+
final Path destinationPath = this.pluginBasePath.resolve(destinationFileName);
|
380
|
+
Files.createDirectories(destinationPath.getParent());
|
381
|
+
try (BufferedWriter writer = Files.newBufferedWriter(destinationPath, StandardCharsets.UTF_8)) {
|
382
|
+
this.velocityEngine.evaluate(velocityContext,
|
383
|
+
writer,
|
384
|
+
"embulk-new",
|
385
|
+
reader);
|
386
|
+
}
|
387
|
+
}
|
388
|
+
}
|
389
|
+
|
390
|
+
private void setExecutable(String targetFileName)
|
391
|
+
throws IOException
|
392
|
+
{
|
393
|
+
final Path targetPath = this.pluginBasePath.resolve(targetFileName);
|
394
|
+
final Set<PosixFilePermission> permissions =
|
395
|
+
new HashSet<PosixFilePermission>(Files.getPosixFilePermissions(targetPath));
|
396
|
+
permissions.add(PosixFilePermission.OWNER_EXECUTE);
|
397
|
+
permissions.add(PosixFilePermission.GROUP_EXECUTE);
|
398
|
+
permissions.add(PosixFilePermission.OTHERS_EXECUTE);
|
399
|
+
Files.setPosixFilePermissions(targetPath, permissions);
|
400
|
+
}
|
401
|
+
|
402
|
+
private final Path basePath;
|
403
|
+
|
404
|
+
private final String nameGiven;
|
405
|
+
private final String language;
|
406
|
+
private final String category;
|
407
|
+
private final String embulkVersion;
|
408
|
+
|
409
|
+
private final String embulkCategory;
|
410
|
+
private final String name;
|
411
|
+
|
412
|
+
private final String fullProjectName;
|
413
|
+
private final String pluginDirectory;
|
414
|
+
private final String pluginPath;
|
415
|
+
|
416
|
+
private final Path pluginBasePath;
|
417
|
+
|
418
|
+
private final VelocityEngine velocityEngine;
|
419
|
+
}
|
@@ -0,0 +1,248 @@
|
|
1
|
+
package org.embulk.cli;
|
2
|
+
|
3
|
+
import java.io.ByteArrayOutputStream;
|
4
|
+
import java.io.FileNotFoundException;
|
5
|
+
import java.io.InputStream;
|
6
|
+
import java.io.IOException;
|
7
|
+
import java.net.HttpURLConnection;
|
8
|
+
import java.net.URL;
|
9
|
+
import java.net.URISyntaxException;
|
10
|
+
import java.nio.file.Files;
|
11
|
+
import java.nio.file.Path;
|
12
|
+
import java.nio.file.Paths;
|
13
|
+
import java.nio.file.StandardCopyOption;
|
14
|
+
import java.util.jar.Attributes;
|
15
|
+
import java.util.jar.JarFile;
|
16
|
+
import java.util.jar.Manifest;
|
17
|
+
import java.util.regex.Matcher;
|
18
|
+
import java.util.regex.Pattern;
|
19
|
+
|
20
|
+
import org.apache.maven.artifact.versioning.ComparableVersion;
|
21
|
+
|
22
|
+
// It uses |java.net.HttpURLConnection| so that embulk-cli does not need additional dependedcies.
|
23
|
+
// TODO(dmikurube): Support HTTP proxy. The original Ruby version did not support as well, though.
|
24
|
+
public class EmbulkSelfUpdate
|
25
|
+
{
|
26
|
+
// TODO(dmikurube): Stop catching Exceptions here when embulk_run.rb is replaced to Java.
|
27
|
+
public void updateSelf(final String runningVersionString,
|
28
|
+
final String specifiedVersionString,
|
29
|
+
final String embulkRunRubyPathString,
|
30
|
+
final boolean isForced)
|
31
|
+
throws IOException, URISyntaxException
|
32
|
+
{
|
33
|
+
try {
|
34
|
+
updateSelfWithExceptions(runningVersionString, specifiedVersionString, embulkRunRubyPathString, isForced);
|
35
|
+
}
|
36
|
+
catch (Throwable ex) {
|
37
|
+
ex.printStackTrace();
|
38
|
+
throw ex;
|
39
|
+
}
|
40
|
+
}
|
41
|
+
|
42
|
+
private void updateSelfWithExceptions(final String runningVersionString,
|
43
|
+
final String specifiedVersionString,
|
44
|
+
final String embulkRunRubyPathString,
|
45
|
+
final boolean isForced)
|
46
|
+
throws IOException, URISyntaxException
|
47
|
+
{
|
48
|
+
final Path jarPathJava = Paths.get(
|
49
|
+
EmbulkSelfUpdate.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath());
|
50
|
+
|
51
|
+
if ((!Files.exists(jarPathJava)) || (!Files.isRegularFile(jarPathJava))) {
|
52
|
+
throw exceptionNoSingleJar();
|
53
|
+
}
|
54
|
+
|
55
|
+
if (embulkRunRubyPathString != null) {
|
56
|
+
final String[] splitRubyFile = embulkRunRubyPathString.split("!", 2);
|
57
|
+
if (splitRubyFile.length < 2) {
|
58
|
+
throw exceptionNoSingleJar();
|
59
|
+
}
|
60
|
+
final Path jarPathRuby = Paths.get(splitRubyFile[0]);
|
61
|
+
if (!jarPathJava.equals(jarPathRuby)) {
|
62
|
+
throw exceptionNoSingleJar();
|
63
|
+
}
|
64
|
+
}
|
65
|
+
|
66
|
+
final String targetVersionString;
|
67
|
+
if (specifiedVersionString != null) {
|
68
|
+
System.out.printf("Checking version %s...\n", specifiedVersionString);
|
69
|
+
targetVersionString = checkTargetVersion(specifiedVersionString);
|
70
|
+
if (targetVersionString == null) {
|
71
|
+
throw new RuntimeException(String.format("Specified version does not exist: %s", specifiedVersionString));
|
72
|
+
}
|
73
|
+
System.out.printf("Found version %s.\n", specifiedVersionString);
|
74
|
+
}
|
75
|
+
else {
|
76
|
+
System.out.println("Checking the latest version...");
|
77
|
+
final ComparableVersion runningVersion = new ComparableVersion(runningVersionString);
|
78
|
+
targetVersionString = checkLatestVersion();
|
79
|
+
final ComparableVersion targetVersion = new ComparableVersion(targetVersionString);
|
80
|
+
if (targetVersion.compareTo(runningVersion) <= 0) {
|
81
|
+
System.out.printf("Already up-to-date. %s is the latest version.\n", runningVersion);
|
82
|
+
return;
|
83
|
+
}
|
84
|
+
System.out.printf("Found a newer version %s.\n", targetVersion);
|
85
|
+
}
|
86
|
+
|
87
|
+
if (!Files.isWritable(jarPathJava)) {
|
88
|
+
throw new RuntimeException(String.format("The existing %s is not writable. May need to sudo?",
|
89
|
+
jarPathJava.toString()));
|
90
|
+
}
|
91
|
+
|
92
|
+
final URL downloadUrl = new URL(String.format("https://dl.bintray.com/embulk/maven/embulk-%s.jar",
|
93
|
+
targetVersionString));
|
94
|
+
System.out.printf("Downloading %s ...\n", downloadUrl.toString());
|
95
|
+
|
96
|
+
Path jarPathTemp = Files.createTempFile("embulk-selfupdate", ".jar");
|
97
|
+
try {
|
98
|
+
final HttpURLConnection connection = (HttpURLConnection) downloadUrl.openConnection();
|
99
|
+
try {
|
100
|
+
// Follow the redicrect from the Bintray URL.
|
101
|
+
connection.setInstanceFollowRedirects(true);
|
102
|
+
connection.setRequestMethod("GET");
|
103
|
+
connection.connect();
|
104
|
+
final int statusCode = connection.getResponseCode();
|
105
|
+
if (HttpURLConnection.HTTP_OK != statusCode) {
|
106
|
+
throw new FileNotFoundException(
|
107
|
+
String.format("Unexpected HTTP status code: %d", statusCode));
|
108
|
+
}
|
109
|
+
InputStream input = connection.getInputStream();
|
110
|
+
// TODO(dmikurube): Confirm if it is okay to replace a temp file created by Files.createTempFile.
|
111
|
+
Files.copy(input, jarPathTemp, StandardCopyOption.REPLACE_EXISTING);
|
112
|
+
Files.setPosixFilePermissions(jarPathTemp, Files.getPosixFilePermissions(jarPathJava));
|
113
|
+
}
|
114
|
+
finally {
|
115
|
+
connection.disconnect();
|
116
|
+
}
|
117
|
+
|
118
|
+
if (!isForced) { // Check corruption
|
119
|
+
final String versionJarTemp;
|
120
|
+
try {
|
121
|
+
versionJarTemp = getJarVersion(jarPathTemp);
|
122
|
+
}
|
123
|
+
catch (FileNotFoundException ex) {
|
124
|
+
throw new RuntimeException("Failed to check corruption. Downloaded version may include incompatible changes. Try the '-f' option to force updating without checking.", ex);
|
125
|
+
}
|
126
|
+
if (!versionJarTemp.equals(targetVersionString)) {
|
127
|
+
throw new RuntimeException(
|
128
|
+
String.format("Downloaded version does not match: %s (downloaded) / %s (target)",
|
129
|
+
versionJarTemp,
|
130
|
+
targetVersionString));
|
131
|
+
}
|
132
|
+
}
|
133
|
+
Files.move(jarPathTemp, jarPathJava, StandardCopyOption.REPLACE_EXISTING);
|
134
|
+
}
|
135
|
+
finally {
|
136
|
+
Files.deleteIfExists(jarPathTemp);
|
137
|
+
}
|
138
|
+
System.out.println(String.format("Updated to %s.", targetVersionString));
|
139
|
+
}
|
140
|
+
|
141
|
+
private RuntimeException exceptionNoSingleJar()
|
142
|
+
{
|
143
|
+
return new RuntimeException("Embulk is not installed as a single jar. \"selfupdate\" does not work. If you installed Embulk through gem, run \"gem install embulk\" instead.");
|
144
|
+
}
|
145
|
+
|
146
|
+
/**
|
147
|
+
* Checks the latest version from bintray.com.
|
148
|
+
*
|
149
|
+
* It passes all {@code IOException} and {@code RuntimeException} through out.
|
150
|
+
*/
|
151
|
+
private String checkLatestVersion()
|
152
|
+
throws IOException
|
153
|
+
{
|
154
|
+
final URL bintrayUrl = new URL("https://bintray.com/embulk/maven/embulk/_latestVersion");
|
155
|
+
final HttpURLConnection connection = (HttpURLConnection) bintrayUrl.openConnection();
|
156
|
+
try {
|
157
|
+
// Stop HttpURLConnection from following redirects when the status code is 301 or 302.
|
158
|
+
connection.setInstanceFollowRedirects(false);
|
159
|
+
connection.setRequestMethod("GET");
|
160
|
+
connection.connect();
|
161
|
+
final int statusCode = connection.getResponseCode();
|
162
|
+
if (HttpURLConnection.HTTP_MOVED_TEMP != statusCode) {
|
163
|
+
throw new FileNotFoundException(
|
164
|
+
String.format("Unexpected HTTP status code: %d", statusCode));
|
165
|
+
}
|
166
|
+
final String location = connection.getHeaderField("Location");
|
167
|
+
final Matcher versionMatcher = VERSION_URL_PATTERN.matcher(location);
|
168
|
+
if (!versionMatcher.matches()) {
|
169
|
+
throw new FileNotFoundException(
|
170
|
+
String.format("Invalid version number in \"Location\" header: %s", location));
|
171
|
+
}
|
172
|
+
return versionMatcher.group(1);
|
173
|
+
}
|
174
|
+
finally {
|
175
|
+
connection.disconnect();
|
176
|
+
}
|
177
|
+
}
|
178
|
+
|
179
|
+
/**
|
180
|
+
* Checks the target version in bintray.com.
|
181
|
+
*
|
182
|
+
* It passes all {@code IOException} and {@code RuntimeException} through out.
|
183
|
+
*/
|
184
|
+
private String checkTargetVersion(String version)
|
185
|
+
throws IOException
|
186
|
+
{
|
187
|
+
final URL bintrayUrl = new URL(String.format("https://bintray.com/embulk/maven/embulk/%s", version));
|
188
|
+
final HttpURLConnection connection = (HttpURLConnection) bintrayUrl.openConnection();
|
189
|
+
try {
|
190
|
+
connection.setInstanceFollowRedirects(false);
|
191
|
+
connection.setRequestMethod("GET");
|
192
|
+
connection.connect();
|
193
|
+
final int statusCode = connection.getResponseCode();
|
194
|
+
if (HttpURLConnection.HTTP_NOT_FOUND == statusCode) {
|
195
|
+
return null;
|
196
|
+
}
|
197
|
+
else if (HttpURLConnection.HTTP_OK != statusCode) {
|
198
|
+
throw new FileNotFoundException(
|
199
|
+
String.format("Unexpected HTTP status code: %d", statusCode));
|
200
|
+
}
|
201
|
+
else {
|
202
|
+
return version;
|
203
|
+
}
|
204
|
+
}
|
205
|
+
finally {
|
206
|
+
connection.disconnect();
|
207
|
+
}
|
208
|
+
}
|
209
|
+
|
210
|
+
private String getJarVersion(Path jarPath)
|
211
|
+
throws IOException
|
212
|
+
{
|
213
|
+
try (final JarFile jarFile = new JarFile(jarPath.toFile())) {
|
214
|
+
final Manifest manifest;
|
215
|
+
try {
|
216
|
+
manifest = jarFile.getManifest();
|
217
|
+
}
|
218
|
+
catch (IOException ex) {
|
219
|
+
throw new IOException("Version not found. Failed to load the manifest.", ex);
|
220
|
+
}
|
221
|
+
String manifestContents;
|
222
|
+
try (final ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
|
223
|
+
manifest.write(outputStream);
|
224
|
+
manifestContents = outputStream.toString();
|
225
|
+
}
|
226
|
+
catch (IOException ex) {
|
227
|
+
manifestContents = "(Failed to read the contents of the manifest.)";
|
228
|
+
}
|
229
|
+
final Attributes mainAttributes = manifest.getMainAttributes();
|
230
|
+
final String implementationVersion = mainAttributes.getValue(Attributes.Name.IMPLEMENTATION_VERSION);
|
231
|
+
if (implementationVersion == null) {
|
232
|
+
throw new IOException("Version not found. Failed to read \""
|
233
|
+
+ Attributes.Name.IMPLEMENTATION_VERSION
|
234
|
+
+ "\": " + manifestContents);
|
235
|
+
}
|
236
|
+
return implementationVersion;
|
237
|
+
}
|
238
|
+
catch (IOException ex) {
|
239
|
+
throw new IOException("Version not found. Failed to load the jar file.", ex);
|
240
|
+
}
|
241
|
+
|
242
|
+
// NOTE: Checking embulk/version.rb is no longer needed.
|
243
|
+
// The jar manifest with "Implementation-Version" has been included in Embulk jars from v0.4.0.
|
244
|
+
}
|
245
|
+
|
246
|
+
private static final Pattern VERSION_URL_PATTERN = Pattern.compile("^https?://.*/embulk/(\\d+\\.\\d+[^\\/]+).*$");
|
247
|
+
private static final Pattern VERSION_RUBY_PATTERN = Pattern.compile("^\\s*VERSION\\s*\\=\\s*(\\p{Graph}+)\\s*$");
|
248
|
+
}
|