embulk 0.8.18-java → 0.8.19-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (87) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +10 -0
  3. data/build.gradle +10 -3
  4. data/embulk-cli/build.gradle +2 -0
  5. data/embulk-cli/src/main/bat/selfrun.bat +98 -0
  6. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkExample.java +82 -0
  7. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkMigrate.java +458 -0
  8. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkNew.java +419 -0
  9. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkSelfUpdate.java +248 -0
  10. data/embulk-cli/src/main/sh/selfrun.sh +0 -103
  11. data/embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java +158 -143
  12. data/embulk-core/build.gradle +2 -2
  13. data/embulk-core/src/main/java/org/embulk/EmbulkVersion.java +109 -0
  14. data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java +11 -0
  15. data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java +29 -3
  16. data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java +47 -13
  17. data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java +6 -3
  18. data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java +385 -64
  19. data/embulk-core/src/main/java/org/embulk/spi/TempFileSpace.java +2 -1
  20. data/embulk-core/src/test/java/org/embulk/spi/TestPageBuilderReader.java +62 -0
  21. data/embulk-docs/src/built-in.rst +59 -21
  22. data/embulk-docs/src/customization.rst +8 -8
  23. data/embulk-docs/src/developers/index.rst +45 -0
  24. data/embulk-docs/src/index.rst +11 -7
  25. data/embulk-docs/src/recipe.rst +1 -1
  26. data/embulk-docs/src/recipe/{scheduled-csv-load-to-elasticsearch-kibana4.rst → scheduled-csv-load-to-elasticsearch-kibana5.rst} +26 -24
  27. data/embulk-docs/src/release.rst +1 -0
  28. data/embulk-docs/src/release/release-0.4.0.rst +1 -1
  29. data/embulk-docs/src/release/release-0.5.0.rst +1 -1
  30. data/embulk-docs/src/release/release-0.6.0.rst +1 -1
  31. data/embulk-docs/src/release/release-0.6.20.rst +1 -1
  32. data/embulk-docs/src/release/release-0.8.19.rst +43 -0
  33. data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +2 -2
  34. data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java +30 -1
  35. data/embulk-standards/src/test/java/org/embulk/standards/guess/TestCsvGuessPlugin.java +10 -0
  36. data/embulk-standards/src/test/java/org/embulk/standards/preview/TestFilePreview.java +73 -0
  37. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records.csv +5 -0
  38. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_guessed.yml +2 -0
  39. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_seed.yml +1 -0
  40. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes.csv +5 -0
  41. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_exec.yml +1 -0
  42. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_load.yml +19 -0
  43. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_previewed.csv +1 -0
  44. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple.csv +5 -0
  45. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_load.yml +19 -0
  46. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_previewed.csv +4 -0
  47. data/embulk-test/src/main/java/org/embulk/test/PreviewResultInputPlugin.java +65 -0
  48. data/embulk-test/src/main/java/org/embulk/test/TestingBulkLoader.java +5 -0
  49. data/embulk-test/src/main/java/org/embulk/test/TestingEmbulk.java +59 -2
  50. data/embulk.gemspec +2 -1
  51. data/lib/embulk/command/embulk_run.rb +11 -49
  52. data/lib/embulk/data/new/README.md.vm +106 -0
  53. data/lib/embulk/data/new/{gitignore.erb → gitignore.vm} +3 -3
  54. data/lib/embulk/data/new/java/{build.gradle.erb → build.gradle.vm} +8 -8
  55. data/lib/embulk/data/new/java/{decoder.java.erb → decoder.java.vm} +6 -4
  56. data/lib/embulk/data/new/java/{encoder.java.erb → encoder.java.vm} +7 -5
  57. data/lib/embulk/data/new/java/{file_input.java.erb → file_input.java.vm} +9 -7
  58. data/lib/embulk/data/new/java/{file_output.java.erb → file_output.java.vm} +7 -5
  59. data/lib/embulk/data/new/java/{filter.java.erb → filter.java.vm} +4 -3
  60. data/lib/embulk/data/new/java/{formatter.java.erb → formatter.java.vm} +5 -4
  61. data/lib/embulk/data/new/java/{input.java.erb → input.java.vm} +6 -4
  62. data/lib/embulk/data/new/java/{output.java.erb → output.java.vm} +7 -5
  63. data/lib/embulk/data/new/java/{parser.java.erb → parser.java.vm} +5 -4
  64. data/lib/embulk/data/new/java/plugin_loader.rb.vm +3 -0
  65. data/lib/embulk/data/new/java/test.java.vm +5 -0
  66. data/lib/embulk/data/new/ruby/decoder_guess.rb.vm +25 -0
  67. data/lib/embulk/data/new/ruby/{filter.rb.erb → filter.rb.vm} +2 -2
  68. data/lib/embulk/data/new/ruby/{formatter.rb.erb → formatter.rb.vm} +2 -2
  69. data/lib/embulk/data/new/ruby/gemspec.vm +20 -0
  70. data/lib/embulk/data/new/ruby/{input.rb.erb → input.rb.vm} +10 -10
  71. data/lib/embulk/data/new/ruby/{output.rb.erb → output.rb.vm} +7 -7
  72. data/lib/embulk/data/new/ruby/{parser.rb.erb → parser.rb.vm} +2 -2
  73. data/lib/embulk/data/new/ruby/parser_guess.rb.vm +65 -0
  74. data/lib/embulk/guess/csv.rb +5 -0
  75. data/lib/embulk/version.rb +22 -1
  76. metadata +55 -35
  77. data/lib/embulk/command/embulk_example.rb +0 -33
  78. data/lib/embulk/command/embulk_generate_bin.rb +0 -62
  79. data/lib/embulk/command/embulk_migrate_plugin.rb +0 -244
  80. data/lib/embulk/command/embulk_new_plugin.rb +0 -126
  81. data/lib/embulk/command/embulk_selfupdate.rb +0 -121
  82. data/lib/embulk/data/new/README.md.erb +0 -111
  83. data/lib/embulk/data/new/java/plugin_loader.rb.erb +0 -3
  84. data/lib/embulk/data/new/java/test.java.erb +0 -5
  85. data/lib/embulk/data/new/ruby/decoder_guess.rb.erb +0 -25
  86. data/lib/embulk/data/new/ruby/gemspec.erb +0 -20
  87. data/lib/embulk/data/new/ruby/parser_guess.rb.erb +0 -65
@@ -0,0 +1,419 @@
1
+ package org.embulk.cli;
2
+
3
+ import java.io.BufferedWriter;
4
+ import java.io.IOException;
5
+ import java.io.InputStreamReader;
6
+ import java.nio.charset.StandardCharsets;
7
+ import java.nio.file.Files;
8
+ import java.nio.file.FileVisitResult;
9
+ import java.nio.file.Path;
10
+ import java.nio.file.Paths;
11
+ import java.nio.file.SimpleFileVisitor;
12
+ import java.nio.file.attribute.BasicFileAttributes;
13
+ import java.nio.file.attribute.PosixFilePermission;
14
+ import java.util.ArrayList;
15
+ import java.util.HashMap;
16
+ import java.util.HashSet;
17
+ import java.util.Map;
18
+ import java.util.Set;
19
+
20
+ import com.google.common.base.CaseFormat;
21
+ import com.google.common.base.Joiner;
22
+ import com.google.common.io.CharStreams;
23
+
24
+ import org.apache.velocity.VelocityContext;
25
+ import org.apache.velocity.app.VelocityEngine;
26
+
27
+ public class EmbulkNew
28
+ {
29
+ public EmbulkNew(final String categoryWithLanguage, final String nameGiven, final String embulkVersion)
30
+ throws IOException
31
+ {
32
+ this.basePath = Paths.get(".").toAbsolutePath();
33
+
34
+ final LanguageAndCategory languageAndCategory = LanguageAndCategory.of(categoryWithLanguage);
35
+ this.language = languageAndCategory.getLanguage();
36
+ this.category = languageAndCategory.getCategory();
37
+ this.nameGiven = nameGiven;
38
+ this.embulkVersion = embulkVersion;
39
+
40
+ if (category.equals("file_input")) {
41
+ this.embulkCategory = "input";
42
+ }
43
+ else if (category.equals("file_output")) {
44
+ this.embulkCategory = "output";
45
+ }
46
+ else {
47
+ this.embulkCategory = category;
48
+ }
49
+
50
+ this.name = nameGiven.replaceAll("[^a-zA-Z0-9_]+", "_");
51
+
52
+ this.fullProjectName = "embulk-" + embulkCategory + "-" + name;
53
+ this.pluginDirectory = "lib/embulk";
54
+ this.pluginPath = pluginDirectory + "/" + embulkCategory + "/" + name + ".rb";
55
+
56
+ this.pluginBasePath = this.basePath.resolve(fullProjectName);
57
+
58
+ this.velocityEngine = new VelocityEngine();
59
+ this.velocityEngine.init();
60
+ this.velocityEngine.setProperty(VelocityEngine.RUNTIME_LOG_LOGSYSTEM_CLASS,
61
+ "org.apache.velocity.runtime.log.NullLogSystem");
62
+ }
63
+
64
+ public boolean newPlugin()
65
+ throws IOException
66
+ {
67
+ if (Files.exists(this.pluginBasePath)) {
68
+ throw new IOException("./" + this.fullProjectName + " already exists. Please delete it first.");
69
+ }
70
+
71
+ Files.createDirectories(this.pluginBasePath);
72
+
73
+ System.out.println("Creating " + this.fullProjectName + "/");
74
+
75
+ boolean success = false;
76
+ try {
77
+ //
78
+ // Generate gemspec
79
+ //
80
+ final String author = getGitConfig("user.name", "YOUR_NAME");
81
+ final String email = getGitConfig("user.email", "YOUR_NAME");
82
+ final String expectedGitHubAccount = email.split("@")[0];
83
+
84
+ // variables used in Velocity templates
85
+ final String rubyClassName = CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, name);
86
+ final String javaClassName =
87
+ CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, name) +
88
+ CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, category) +
89
+ "Plugin";
90
+ final String javaPackageName = "org.embulk." + embulkCategory + "." + name;
91
+ final String displayName = getDisplayName(name);
92
+ final String displayCategory = category.replace("_", " ");
93
+
94
+ final HashMap<String, String> extraGuesses = new HashMap<String, String>();
95
+
96
+ final String description;
97
+ switch (category) {
98
+ case "input":
99
+ description = String.format("Loads records from %s.", displayName);
100
+ break;
101
+ case "file_input":
102
+ description = String.format("Reads files stored on %s.", displayName);
103
+ break;
104
+ case "parser":
105
+ description = String.format("Parses %s files read by other file input plugins.", displayName);
106
+ extraGuesses.put("embulk/data/new/ruby/parser_guess.rb.vm",
107
+ String.format("%s/guess/%s.rb", pluginDirectory, name));
108
+ break;
109
+ case "decoder":
110
+ description = String.format("Decodes %s-encoded files read by other file input plugins.", displayName);
111
+ extraGuesses.put("embulk/data/new/ruby/decoder_guess.rb.vm",
112
+ String.format("%s/guess/%s.rb", pluginDirectory, name));
113
+ break;
114
+ case "output":
115
+ description = String.format("Dumps records to %s.", displayName);
116
+ break;
117
+ case "file_output":
118
+ description = String.format("Stores files on %s.", displayName);
119
+ break;
120
+ case "formatter":
121
+ description = String.format("Formats %s files for other file output plugins.", displayName);
122
+ break;
123
+ case "encoder":
124
+ description = String.format("Encodes files using %s for other file output plugins.", displayName);
125
+ break;
126
+ case "filter":
127
+ description = String.format("%s", displayName);
128
+ break;
129
+ default:
130
+ throw new RuntimeException("FATAL: Invalid plugin category.");
131
+ }
132
+
133
+ //
134
+ // Generate project repository
135
+ //
136
+ final VelocityContext velocityContext = createVelocityContext(
137
+ author,
138
+ category,
139
+ description,
140
+ displayName,
141
+ displayCategory,
142
+ email,
143
+ embulkCategory,
144
+ this.embulkVersion,
145
+ expectedGitHubAccount,
146
+ fullProjectName,
147
+ javaClassName,
148
+ javaPackageName,
149
+ language,
150
+ name,
151
+ rubyClassName);
152
+ copyTemplated("embulk/data/new/README.md.vm", "README.md", velocityContext);
153
+ copy("embulk/data/new/LICENSE.txt", "LICENSE.txt");
154
+ copyTemplated("embulk/data/new/gitignore.vm", ".gitignore", velocityContext);
155
+
156
+ switch (language) {
157
+ case "ruby":
158
+ copy("embulk/data/new/ruby/Rakefile", "Rakefile");
159
+ copy("embulk/data/new/ruby/Gemfile", "Gemfile");
160
+ copy("embulk/data/new/ruby/.ruby-version", ".ruby-version");
161
+ copyTemplated("embulk/data/new/ruby/gemspec.vm",
162
+ fullProjectName + ".gemspec",
163
+ velocityContext);
164
+ copyTemplated(String.format("embulk/data/new/ruby/%s.rb.vm", category),
165
+ this.pluginPath,
166
+ velocityContext);
167
+ break;
168
+ case "java":
169
+ copy("embulk/data/new/java/gradle/wrapper/gradle-wrapper.jar",
170
+ "gradle/wrapper/gradle-wrapper.jar");
171
+ copy("embulk/data/new/java/gradle/wrapper/gradle-wrapper.properties",
172
+ "gradle/wrapper/gradle-wrapper.properties");
173
+ copy("embulk/data/new/java/gradlew.bat", "gradlew.bat");
174
+ copy("embulk/data/new/java/gradlew", "gradlew");
175
+ setExecutable("gradlew");
176
+ copy("embulk/data/new/java/config/checkstyle/checkstyle.xml", "config/checkstyle/checkstyle.xml");
177
+ copy("embulk/data/new/java/config/checkstyle/default.xml", "config/checkstyle/default.xml");
178
+ copyTemplated("embulk/data/new/java/build.gradle.vm", "build.gradle", velocityContext);
179
+ copyTemplated("embulk/data/new/java/plugin_loader.rb.vm", this.pluginPath, velocityContext);
180
+ copyTemplated(String.format("embulk/data/new/java/%s.java.vm", category),
181
+ String.format("src/main/java/%s/%s.java",
182
+ javaPackageName.replaceAll("\\.", "/"),
183
+ javaClassName),
184
+ velocityContext);
185
+ copyTemplated("embulk/data/new/java/test.java.vm",
186
+ String.format("src/test/java/%s/Test%s.java",
187
+ javaPackageName.replaceAll("\\.", "/"),
188
+ javaClassName),
189
+ velocityContext);
190
+ break;
191
+ }
192
+
193
+ for (Map.Entry<String, String> entry : extraGuesses.entrySet()) {
194
+ copyTemplated(entry.getKey(), entry.getValue(), velocityContext);
195
+ }
196
+
197
+ System.out.println("");
198
+ System.out.println("Plugin template is successfully generated.");
199
+
200
+ switch (language) {
201
+ case "ruby":
202
+ System.out.println("Next steps:");
203
+ System.out.println("");
204
+ System.out.printf(" $ cd %s\n", fullProjectName);
205
+ System.out.println(" $ bundle install # install one using rbenv & rbenv-build");
206
+ System.out.println(" $ bundle exec rake # build gem to be released");
207
+ System.out.println(" $ bundle exec embulk run config.yml # you can run plugin using this command");
208
+ break;
209
+ case "java":
210
+ System.out.println("Next steps:");
211
+ System.out.println("");
212
+ System.out.printf(" $ cd %s\n", fullProjectName);
213
+ System.out.println(" $ ./gradlew package");
214
+ }
215
+
216
+ success = true;
217
+ System.out.println("");
218
+ }
219
+ catch (Exception ex) {
220
+ ex.printStackTrace();
221
+ }
222
+ finally {
223
+ if (!success) {
224
+ System.out.println("Failed. Removing the directory created.");
225
+ deleteDirectoryTree(Paths.get(fullProjectName));
226
+ }
227
+ }
228
+ return success;
229
+ }
230
+
231
+ private static class LanguageAndCategory
232
+ {
233
+ private LanguageAndCategory(final String language, final String category)
234
+ {
235
+ this.language = language;
236
+ this.category = category;
237
+ }
238
+
239
+ public static LanguageAndCategory of(final String categoryWithLanguage)
240
+ {
241
+ switch (categoryWithLanguage) {
242
+ case "java-input": return new LanguageAndCategory("java", "input");
243
+ case "java-output": return new LanguageAndCategory("java", "output");
244
+ case "java-filter": return new LanguageAndCategory("java", "filter");
245
+ case "java-file-input": return new LanguageAndCategory("java", "file_input");
246
+ case "java-file-output": return new LanguageAndCategory("java", "file_output");
247
+ case "java-parser": return new LanguageAndCategory("java", "parser");
248
+ case "java-formatter": return new LanguageAndCategory("java", "formatter");
249
+ case "java-decoder": return new LanguageAndCategory("java", "decoder");
250
+ case "java-encoder": return new LanguageAndCategory("java", "encoder");
251
+ case "ruby-input": return new LanguageAndCategory("ruby", "input");
252
+ case "ruby-output": return new LanguageAndCategory("ruby", "output");
253
+ case "ruby-filter": return new LanguageAndCategory("ruby", "filter");
254
+ case "ruby-file-input":
255
+ throw new RuntimeException("ruby-file-input is not implemented yet. See #21 on github.");
256
+ case "ruby-file-output":
257
+ throw new RuntimeException("ruby-file-output is not implemented yet. See #22 on github.");
258
+ case "ruby-parser": return new LanguageAndCategory("ruby", "parser");
259
+ case "ruby-formatter": return new LanguageAndCategory("ruby", "formatter");
260
+ case "ruby-decoder":
261
+ throw new RuntimeException("ruby-decoder is not implemented yet. See #31 on github.");
262
+ case "ruby-encoder":
263
+ throw new RuntimeException("ruby-decoder is not implemented yet. See #32 on github.");
264
+ default:
265
+ throw new RuntimeException(String.format("Unknown category '%s'", categoryWithLanguage));
266
+ }
267
+ }
268
+
269
+ public String getLanguage()
270
+ {
271
+ return this.language;
272
+ }
273
+
274
+ public String getCategory()
275
+ {
276
+ return this.category;
277
+ }
278
+
279
+ private final String language;
280
+ private final String category;
281
+ }
282
+
283
+ private String getGitConfig(final String configName, final String defaultValue)
284
+ {
285
+ try {
286
+ final Process process = new ProcessBuilder("git", "config", configName).redirectErrorStream(true).start();
287
+ return CharStreams.toString(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8)).trim();
288
+ }
289
+ catch (Throwable ex) {
290
+ return "YOUR_NAME";
291
+ }
292
+ }
293
+
294
+ private Path deleteDirectoryTree(final Path path)
295
+ throws IOException
296
+ {
297
+ return Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
298
+ @Override
299
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attributes)
300
+ throws IOException
301
+ {
302
+ Files.delete(file);
303
+ return FileVisitResult.CONTINUE;
304
+ }
305
+
306
+ @Override
307
+ public FileVisitResult postVisitDirectory(Path directory, IOException exception)
308
+ throws IOException
309
+ {
310
+ Files.delete(directory);
311
+ return FileVisitResult.CONTINUE;
312
+ }
313
+ });
314
+ }
315
+
316
+ private String getDisplayName(final String name)
317
+ {
318
+ final String[] nameSplit = name.split("_");
319
+ final ArrayList<String> nameComposition = new ArrayList<String>();
320
+ for (String namePart : nameSplit) {
321
+ nameComposition.add(CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, namePart));
322
+ }
323
+ return Joiner.on(" ").join(nameComposition);
324
+ }
325
+
326
+ private VelocityContext createVelocityContext(final String author,
327
+ final String category,
328
+ final String description,
329
+ final String displayName,
330
+ final String displayCategory,
331
+ final String email,
332
+ final String embulkCategory,
333
+ final String embulkVersion,
334
+ final String expectedGitHubAccount,
335
+ final String fullProjectName,
336
+ final String javaClassName,
337
+ final String javaPackageName,
338
+ final String language,
339
+ final String name,
340
+ final String rubyClassName)
341
+ {
342
+ final VelocityContext velocityContext = new VelocityContext();
343
+ // TODO(dmikurube): Revisit this |argumentToRunEmbulkJava|.
344
+ // This is in the Velocity context because the value could not be in Velocity templates.
345
+ velocityContext.put("argumentToRunEmbulkJava", "\'-L ${file(\".\").absolutePath}\'");
346
+ velocityContext.put("author", author);
347
+ velocityContext.put("category", category);
348
+ velocityContext.put("description", description);
349
+ velocityContext.put("displayName", displayName);
350
+ velocityContext.put("displayCategory", displayCategory);
351
+ velocityContext.put("email", email);
352
+ velocityContext.put("embulkCategory", embulkCategory);
353
+ velocityContext.put("embulkVersion", embulkVersion);
354
+ velocityContext.put("expectedGitHubAccount", expectedGitHubAccount);
355
+ velocityContext.put("fullProjectName", fullProjectName);
356
+ velocityContext.put("javaClassName", javaClassName);
357
+ velocityContext.put("javaGuessClassName", javaClassName.replace("Plugin", "GuessPlugin"));
358
+ velocityContext.put("javaPackageName", javaPackageName);
359
+ velocityContext.put("language", language);
360
+ velocityContext.put("name", name);
361
+ velocityContext.put("rubyClassName", rubyClassName);
362
+ velocityContext.put("rubyGuessClassName", rubyClassName.replace("Plugin", "GuessPlugin"));
363
+ return velocityContext;
364
+ }
365
+
366
+ private void copy(String sourceResourcePath, String destinationFileName)
367
+ throws IOException
368
+ {
369
+ final Path destinationPath = this.pluginBasePath.resolve(destinationFileName);
370
+ Files.createDirectories(destinationPath.getParent());
371
+ Files.copy(EmbulkNew.class.getClassLoader().getResourceAsStream(sourceResourcePath), destinationPath);
372
+ }
373
+
374
+ private void copyTemplated(String sourceResourcePath, String destinationFileName, VelocityContext velocityContext)
375
+ throws IOException
376
+ {
377
+ try (InputStreamReader reader = new InputStreamReader(
378
+ EmbulkNew.class.getClassLoader().getResourceAsStream(sourceResourcePath))) {
379
+ final Path destinationPath = this.pluginBasePath.resolve(destinationFileName);
380
+ Files.createDirectories(destinationPath.getParent());
381
+ try (BufferedWriter writer = Files.newBufferedWriter(destinationPath, StandardCharsets.UTF_8)) {
382
+ this.velocityEngine.evaluate(velocityContext,
383
+ writer,
384
+ "embulk-new",
385
+ reader);
386
+ }
387
+ }
388
+ }
389
+
390
+ private void setExecutable(String targetFileName)
391
+ throws IOException
392
+ {
393
+ final Path targetPath = this.pluginBasePath.resolve(targetFileName);
394
+ final Set<PosixFilePermission> permissions =
395
+ new HashSet<PosixFilePermission>(Files.getPosixFilePermissions(targetPath));
396
+ permissions.add(PosixFilePermission.OWNER_EXECUTE);
397
+ permissions.add(PosixFilePermission.GROUP_EXECUTE);
398
+ permissions.add(PosixFilePermission.OTHERS_EXECUTE);
399
+ Files.setPosixFilePermissions(targetPath, permissions);
400
+ }
401
+
402
+ private final Path basePath;
403
+
404
+ private final String nameGiven;
405
+ private final String language;
406
+ private final String category;
407
+ private final String embulkVersion;
408
+
409
+ private final String embulkCategory;
410
+ private final String name;
411
+
412
+ private final String fullProjectName;
413
+ private final String pluginDirectory;
414
+ private final String pluginPath;
415
+
416
+ private final Path pluginBasePath;
417
+
418
+ private final VelocityEngine velocityEngine;
419
+ }
@@ -0,0 +1,248 @@
1
+ package org.embulk.cli;
2
+
3
+ import java.io.ByteArrayOutputStream;
4
+ import java.io.FileNotFoundException;
5
+ import java.io.InputStream;
6
+ import java.io.IOException;
7
+ import java.net.HttpURLConnection;
8
+ import java.net.URL;
9
+ import java.net.URISyntaxException;
10
+ import java.nio.file.Files;
11
+ import java.nio.file.Path;
12
+ import java.nio.file.Paths;
13
+ import java.nio.file.StandardCopyOption;
14
+ import java.util.jar.Attributes;
15
+ import java.util.jar.JarFile;
16
+ import java.util.jar.Manifest;
17
+ import java.util.regex.Matcher;
18
+ import java.util.regex.Pattern;
19
+
20
+ import org.apache.maven.artifact.versioning.ComparableVersion;
21
+
22
+ // It uses |java.net.HttpURLConnection| so that embulk-cli does not need additional dependedcies.
23
+ // TODO(dmikurube): Support HTTP proxy. The original Ruby version did not support as well, though.
24
+ public class EmbulkSelfUpdate
25
+ {
26
+ // TODO(dmikurube): Stop catching Exceptions here when embulk_run.rb is replaced to Java.
27
+ public void updateSelf(final String runningVersionString,
28
+ final String specifiedVersionString,
29
+ final String embulkRunRubyPathString,
30
+ final boolean isForced)
31
+ throws IOException, URISyntaxException
32
+ {
33
+ try {
34
+ updateSelfWithExceptions(runningVersionString, specifiedVersionString, embulkRunRubyPathString, isForced);
35
+ }
36
+ catch (Throwable ex) {
37
+ ex.printStackTrace();
38
+ throw ex;
39
+ }
40
+ }
41
+
42
+ private void updateSelfWithExceptions(final String runningVersionString,
43
+ final String specifiedVersionString,
44
+ final String embulkRunRubyPathString,
45
+ final boolean isForced)
46
+ throws IOException, URISyntaxException
47
+ {
48
+ final Path jarPathJava = Paths.get(
49
+ EmbulkSelfUpdate.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath());
50
+
51
+ if ((!Files.exists(jarPathJava)) || (!Files.isRegularFile(jarPathJava))) {
52
+ throw exceptionNoSingleJar();
53
+ }
54
+
55
+ if (embulkRunRubyPathString != null) {
56
+ final String[] splitRubyFile = embulkRunRubyPathString.split("!", 2);
57
+ if (splitRubyFile.length < 2) {
58
+ throw exceptionNoSingleJar();
59
+ }
60
+ final Path jarPathRuby = Paths.get(splitRubyFile[0]);
61
+ if (!jarPathJava.equals(jarPathRuby)) {
62
+ throw exceptionNoSingleJar();
63
+ }
64
+ }
65
+
66
+ final String targetVersionString;
67
+ if (specifiedVersionString != null) {
68
+ System.out.printf("Checking version %s...\n", specifiedVersionString);
69
+ targetVersionString = checkTargetVersion(specifiedVersionString);
70
+ if (targetVersionString == null) {
71
+ throw new RuntimeException(String.format("Specified version does not exist: %s", specifiedVersionString));
72
+ }
73
+ System.out.printf("Found version %s.\n", specifiedVersionString);
74
+ }
75
+ else {
76
+ System.out.println("Checking the latest version...");
77
+ final ComparableVersion runningVersion = new ComparableVersion(runningVersionString);
78
+ targetVersionString = checkLatestVersion();
79
+ final ComparableVersion targetVersion = new ComparableVersion(targetVersionString);
80
+ if (targetVersion.compareTo(runningVersion) <= 0) {
81
+ System.out.printf("Already up-to-date. %s is the latest version.\n", runningVersion);
82
+ return;
83
+ }
84
+ System.out.printf("Found a newer version %s.\n", targetVersion);
85
+ }
86
+
87
+ if (!Files.isWritable(jarPathJava)) {
88
+ throw new RuntimeException(String.format("The existing %s is not writable. May need to sudo?",
89
+ jarPathJava.toString()));
90
+ }
91
+
92
+ final URL downloadUrl = new URL(String.format("https://dl.bintray.com/embulk/maven/embulk-%s.jar",
93
+ targetVersionString));
94
+ System.out.printf("Downloading %s ...\n", downloadUrl.toString());
95
+
96
+ Path jarPathTemp = Files.createTempFile("embulk-selfupdate", ".jar");
97
+ try {
98
+ final HttpURLConnection connection = (HttpURLConnection) downloadUrl.openConnection();
99
+ try {
100
+ // Follow the redicrect from the Bintray URL.
101
+ connection.setInstanceFollowRedirects(true);
102
+ connection.setRequestMethod("GET");
103
+ connection.connect();
104
+ final int statusCode = connection.getResponseCode();
105
+ if (HttpURLConnection.HTTP_OK != statusCode) {
106
+ throw new FileNotFoundException(
107
+ String.format("Unexpected HTTP status code: %d", statusCode));
108
+ }
109
+ InputStream input = connection.getInputStream();
110
+ // TODO(dmikurube): Confirm if it is okay to replace a temp file created by Files.createTempFile.
111
+ Files.copy(input, jarPathTemp, StandardCopyOption.REPLACE_EXISTING);
112
+ Files.setPosixFilePermissions(jarPathTemp, Files.getPosixFilePermissions(jarPathJava));
113
+ }
114
+ finally {
115
+ connection.disconnect();
116
+ }
117
+
118
+ if (!isForced) { // Check corruption
119
+ final String versionJarTemp;
120
+ try {
121
+ versionJarTemp = getJarVersion(jarPathTemp);
122
+ }
123
+ catch (FileNotFoundException ex) {
124
+ throw new RuntimeException("Failed to check corruption. Downloaded version may include incompatible changes. Try the '-f' option to force updating without checking.", ex);
125
+ }
126
+ if (!versionJarTemp.equals(targetVersionString)) {
127
+ throw new RuntimeException(
128
+ String.format("Downloaded version does not match: %s (downloaded) / %s (target)",
129
+ versionJarTemp,
130
+ targetVersionString));
131
+ }
132
+ }
133
+ Files.move(jarPathTemp, jarPathJava, StandardCopyOption.REPLACE_EXISTING);
134
+ }
135
+ finally {
136
+ Files.deleteIfExists(jarPathTemp);
137
+ }
138
+ System.out.println(String.format("Updated to %s.", targetVersionString));
139
+ }
140
+
141
+ private RuntimeException exceptionNoSingleJar()
142
+ {
143
+ return new RuntimeException("Embulk is not installed as a single jar. \"selfupdate\" does not work. If you installed Embulk through gem, run \"gem install embulk\" instead.");
144
+ }
145
+
146
+ /**
147
+ * Checks the latest version from bintray.com.
148
+ *
149
+ * It passes all {@code IOException} and {@code RuntimeException} through out.
150
+ */
151
+ private String checkLatestVersion()
152
+ throws IOException
153
+ {
154
+ final URL bintrayUrl = new URL("https://bintray.com/embulk/maven/embulk/_latestVersion");
155
+ final HttpURLConnection connection = (HttpURLConnection) bintrayUrl.openConnection();
156
+ try {
157
+ // Stop HttpURLConnection from following redirects when the status code is 301 or 302.
158
+ connection.setInstanceFollowRedirects(false);
159
+ connection.setRequestMethod("GET");
160
+ connection.connect();
161
+ final int statusCode = connection.getResponseCode();
162
+ if (HttpURLConnection.HTTP_MOVED_TEMP != statusCode) {
163
+ throw new FileNotFoundException(
164
+ String.format("Unexpected HTTP status code: %d", statusCode));
165
+ }
166
+ final String location = connection.getHeaderField("Location");
167
+ final Matcher versionMatcher = VERSION_URL_PATTERN.matcher(location);
168
+ if (!versionMatcher.matches()) {
169
+ throw new FileNotFoundException(
170
+ String.format("Invalid version number in \"Location\" header: %s", location));
171
+ }
172
+ return versionMatcher.group(1);
173
+ }
174
+ finally {
175
+ connection.disconnect();
176
+ }
177
+ }
178
+
179
+ /**
180
+ * Checks the target version in bintray.com.
181
+ *
182
+ * It passes all {@code IOException} and {@code RuntimeException} through out.
183
+ */
184
+ private String checkTargetVersion(String version)
185
+ throws IOException
186
+ {
187
+ final URL bintrayUrl = new URL(String.format("https://bintray.com/embulk/maven/embulk/%s", version));
188
+ final HttpURLConnection connection = (HttpURLConnection) bintrayUrl.openConnection();
189
+ try {
190
+ connection.setInstanceFollowRedirects(false);
191
+ connection.setRequestMethod("GET");
192
+ connection.connect();
193
+ final int statusCode = connection.getResponseCode();
194
+ if (HttpURLConnection.HTTP_NOT_FOUND == statusCode) {
195
+ return null;
196
+ }
197
+ else if (HttpURLConnection.HTTP_OK != statusCode) {
198
+ throw new FileNotFoundException(
199
+ String.format("Unexpected HTTP status code: %d", statusCode));
200
+ }
201
+ else {
202
+ return version;
203
+ }
204
+ }
205
+ finally {
206
+ connection.disconnect();
207
+ }
208
+ }
209
+
210
+ private String getJarVersion(Path jarPath)
211
+ throws IOException
212
+ {
213
+ try (final JarFile jarFile = new JarFile(jarPath.toFile())) {
214
+ final Manifest manifest;
215
+ try {
216
+ manifest = jarFile.getManifest();
217
+ }
218
+ catch (IOException ex) {
219
+ throw new IOException("Version not found. Failed to load the manifest.", ex);
220
+ }
221
+ String manifestContents;
222
+ try (final ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
223
+ manifest.write(outputStream);
224
+ manifestContents = outputStream.toString();
225
+ }
226
+ catch (IOException ex) {
227
+ manifestContents = "(Failed to read the contents of the manifest.)";
228
+ }
229
+ final Attributes mainAttributes = manifest.getMainAttributes();
230
+ final String implementationVersion = mainAttributes.getValue(Attributes.Name.IMPLEMENTATION_VERSION);
231
+ if (implementationVersion == null) {
232
+ throw new IOException("Version not found. Failed to read \""
233
+ + Attributes.Name.IMPLEMENTATION_VERSION
234
+ + "\": " + manifestContents);
235
+ }
236
+ return implementationVersion;
237
+ }
238
+ catch (IOException ex) {
239
+ throw new IOException("Version not found. Failed to load the jar file.", ex);
240
+ }
241
+
242
+ // NOTE: Checking embulk/version.rb is no longer needed.
243
+ // The jar manifest with "Implementation-Version" has been included in Embulk jars from v0.4.0.
244
+ }
245
+
246
+ private static final Pattern VERSION_URL_PATTERN = Pattern.compile("^https?://.*/embulk/(\\d+\\.\\d+[^\\/]+).*$");
247
+ private static final Pattern VERSION_RUBY_PATTERN = Pattern.compile("^\\s*VERSION\\s*\\=\\s*(\\p{Graph}+)\\s*$");
248
+ }