embulk 0.8.18-java → 0.8.19-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +10 -0
  3. data/build.gradle +10 -3
  4. data/embulk-cli/build.gradle +2 -0
  5. data/embulk-cli/src/main/bat/selfrun.bat +98 -0
  6. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkExample.java +82 -0
  7. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkMigrate.java +458 -0
  8. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkNew.java +419 -0
  9. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkSelfUpdate.java +248 -0
  10. data/embulk-cli/src/main/sh/selfrun.sh +0 -103
  11. data/embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java +158 -143
  12. data/embulk-core/build.gradle +2 -2
  13. data/embulk-core/src/main/java/org/embulk/EmbulkVersion.java +109 -0
  14. data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java +11 -0
  15. data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java +29 -3
  16. data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java +47 -13
  17. data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java +6 -3
  18. data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java +385 -64
  19. data/embulk-core/src/main/java/org/embulk/spi/TempFileSpace.java +2 -1
  20. data/embulk-core/src/test/java/org/embulk/spi/TestPageBuilderReader.java +62 -0
  21. data/embulk-docs/src/built-in.rst +59 -21
  22. data/embulk-docs/src/customization.rst +8 -8
  23. data/embulk-docs/src/developers/index.rst +45 -0
  24. data/embulk-docs/src/index.rst +11 -7
  25. data/embulk-docs/src/recipe.rst +1 -1
  26. data/embulk-docs/src/recipe/{scheduled-csv-load-to-elasticsearch-kibana4.rst → scheduled-csv-load-to-elasticsearch-kibana5.rst} +26 -24
  27. data/embulk-docs/src/release.rst +1 -0
  28. data/embulk-docs/src/release/release-0.4.0.rst +1 -1
  29. data/embulk-docs/src/release/release-0.5.0.rst +1 -1
  30. data/embulk-docs/src/release/release-0.6.0.rst +1 -1
  31. data/embulk-docs/src/release/release-0.6.20.rst +1 -1
  32. data/embulk-docs/src/release/release-0.8.19.rst +43 -0
  33. data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +2 -2
  34. data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java +30 -1
  35. data/embulk-standards/src/test/java/org/embulk/standards/guess/TestCsvGuessPlugin.java +10 -0
  36. data/embulk-standards/src/test/java/org/embulk/standards/preview/TestFilePreview.java +73 -0
  37. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records.csv +5 -0
  38. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_guessed.yml +2 -0
  39. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_seed.yml +1 -0
  40. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes.csv +5 -0
  41. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_exec.yml +1 -0
  42. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_load.yml +19 -0
  43. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_previewed.csv +1 -0
  44. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple.csv +5 -0
  45. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_load.yml +19 -0
  46. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_previewed.csv +4 -0
  47. data/embulk-test/src/main/java/org/embulk/test/PreviewResultInputPlugin.java +65 -0
  48. data/embulk-test/src/main/java/org/embulk/test/TestingBulkLoader.java +5 -0
  49. data/embulk-test/src/main/java/org/embulk/test/TestingEmbulk.java +59 -2
  50. data/embulk.gemspec +2 -1
  51. data/lib/embulk/command/embulk_run.rb +11 -49
  52. data/lib/embulk/data/new/README.md.vm +106 -0
  53. data/lib/embulk/data/new/{gitignore.erb → gitignore.vm} +3 -3
  54. data/lib/embulk/data/new/java/{build.gradle.erb → build.gradle.vm} +8 -8
  55. data/lib/embulk/data/new/java/{decoder.java.erb → decoder.java.vm} +6 -4
  56. data/lib/embulk/data/new/java/{encoder.java.erb → encoder.java.vm} +7 -5
  57. data/lib/embulk/data/new/java/{file_input.java.erb → file_input.java.vm} +9 -7
  58. data/lib/embulk/data/new/java/{file_output.java.erb → file_output.java.vm} +7 -5
  59. data/lib/embulk/data/new/java/{filter.java.erb → filter.java.vm} +4 -3
  60. data/lib/embulk/data/new/java/{formatter.java.erb → formatter.java.vm} +5 -4
  61. data/lib/embulk/data/new/java/{input.java.erb → input.java.vm} +6 -4
  62. data/lib/embulk/data/new/java/{output.java.erb → output.java.vm} +7 -5
  63. data/lib/embulk/data/new/java/{parser.java.erb → parser.java.vm} +5 -4
  64. data/lib/embulk/data/new/java/plugin_loader.rb.vm +3 -0
  65. data/lib/embulk/data/new/java/test.java.vm +5 -0
  66. data/lib/embulk/data/new/ruby/decoder_guess.rb.vm +25 -0
  67. data/lib/embulk/data/new/ruby/{filter.rb.erb → filter.rb.vm} +2 -2
  68. data/lib/embulk/data/new/ruby/{formatter.rb.erb → formatter.rb.vm} +2 -2
  69. data/lib/embulk/data/new/ruby/gemspec.vm +20 -0
  70. data/lib/embulk/data/new/ruby/{input.rb.erb → input.rb.vm} +10 -10
  71. data/lib/embulk/data/new/ruby/{output.rb.erb → output.rb.vm} +7 -7
  72. data/lib/embulk/data/new/ruby/{parser.rb.erb → parser.rb.vm} +2 -2
  73. data/lib/embulk/data/new/ruby/parser_guess.rb.vm +65 -0
  74. data/lib/embulk/guess/csv.rb +5 -0
  75. data/lib/embulk/version.rb +22 -1
  76. metadata +55 -35
  77. data/lib/embulk/command/embulk_example.rb +0 -33
  78. data/lib/embulk/command/embulk_generate_bin.rb +0 -62
  79. data/lib/embulk/command/embulk_migrate_plugin.rb +0 -244
  80. data/lib/embulk/command/embulk_new_plugin.rb +0 -126
  81. data/lib/embulk/command/embulk_selfupdate.rb +0 -121
  82. data/lib/embulk/data/new/README.md.erb +0 -111
  83. data/lib/embulk/data/new/java/plugin_loader.rb.erb +0 -3
  84. data/lib/embulk/data/new/java/test.java.erb +0 -5
  85. data/lib/embulk/data/new/ruby/decoder_guess.rb.erb +0 -25
  86. data/lib/embulk/data/new/ruby/gemspec.erb +0 -20
  87. data/lib/embulk/data/new/ruby/parser_guess.rb.erb +0 -65
@@ -0,0 +1,419 @@
1
+ package org.embulk.cli;
2
+
3
+ import java.io.BufferedWriter;
4
+ import java.io.IOException;
5
+ import java.io.InputStreamReader;
6
+ import java.nio.charset.StandardCharsets;
7
+ import java.nio.file.Files;
8
+ import java.nio.file.FileVisitResult;
9
+ import java.nio.file.Path;
10
+ import java.nio.file.Paths;
11
+ import java.nio.file.SimpleFileVisitor;
12
+ import java.nio.file.attribute.BasicFileAttributes;
13
+ import java.nio.file.attribute.PosixFilePermission;
14
+ import java.util.ArrayList;
15
+ import java.util.HashMap;
16
+ import java.util.HashSet;
17
+ import java.util.Map;
18
+ import java.util.Set;
19
+
20
+ import com.google.common.base.CaseFormat;
21
+ import com.google.common.base.Joiner;
22
+ import com.google.common.io.CharStreams;
23
+
24
+ import org.apache.velocity.VelocityContext;
25
+ import org.apache.velocity.app.VelocityEngine;
26
+
27
+ public class EmbulkNew
28
+ {
29
+ public EmbulkNew(final String categoryWithLanguage, final String nameGiven, final String embulkVersion)
30
+ throws IOException
31
+ {
32
+ this.basePath = Paths.get(".").toAbsolutePath();
33
+
34
+ final LanguageAndCategory languageAndCategory = LanguageAndCategory.of(categoryWithLanguage);
35
+ this.language = languageAndCategory.getLanguage();
36
+ this.category = languageAndCategory.getCategory();
37
+ this.nameGiven = nameGiven;
38
+ this.embulkVersion = embulkVersion;
39
+
40
+ if (category.equals("file_input")) {
41
+ this.embulkCategory = "input";
42
+ }
43
+ else if (category.equals("file_output")) {
44
+ this.embulkCategory = "output";
45
+ }
46
+ else {
47
+ this.embulkCategory = category;
48
+ }
49
+
50
+ this.name = nameGiven.replaceAll("[^a-zA-Z0-9_]+", "_");
51
+
52
+ this.fullProjectName = "embulk-" + embulkCategory + "-" + name;
53
+ this.pluginDirectory = "lib/embulk";
54
+ this.pluginPath = pluginDirectory + "/" + embulkCategory + "/" + name + ".rb";
55
+
56
+ this.pluginBasePath = this.basePath.resolve(fullProjectName);
57
+
58
+ this.velocityEngine = new VelocityEngine();
59
+ this.velocityEngine.init();
60
+ this.velocityEngine.setProperty(VelocityEngine.RUNTIME_LOG_LOGSYSTEM_CLASS,
61
+ "org.apache.velocity.runtime.log.NullLogSystem");
62
+ }
63
+
64
+ public boolean newPlugin()
65
+ throws IOException
66
+ {
67
+ if (Files.exists(this.pluginBasePath)) {
68
+ throw new IOException("./" + this.fullProjectName + " already exists. Please delete it first.");
69
+ }
70
+
71
+ Files.createDirectories(this.pluginBasePath);
72
+
73
+ System.out.println("Creating " + this.fullProjectName + "/");
74
+
75
+ boolean success = false;
76
+ try {
77
+ //
78
+ // Generate gemspec
79
+ //
80
+ final String author = getGitConfig("user.name", "YOUR_NAME");
81
+ final String email = getGitConfig("user.email", "YOUR_NAME");
82
+ final String expectedGitHubAccount = email.split("@")[0];
83
+
84
+ // variables used in Velocity templates
85
+ final String rubyClassName = CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, name);
86
+ final String javaClassName =
87
+ CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, name) +
88
+ CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, category) +
89
+ "Plugin";
90
+ final String javaPackageName = "org.embulk." + embulkCategory + "." + name;
91
+ final String displayName = getDisplayName(name);
92
+ final String displayCategory = category.replace("_", " ");
93
+
94
+ final HashMap<String, String> extraGuesses = new HashMap<String, String>();
95
+
96
+ final String description;
97
+ switch (category) {
98
+ case "input":
99
+ description = String.format("Loads records from %s.", displayName);
100
+ break;
101
+ case "file_input":
102
+ description = String.format("Reads files stored on %s.", displayName);
103
+ break;
104
+ case "parser":
105
+ description = String.format("Parses %s files read by other file input plugins.", displayName);
106
+ extraGuesses.put("embulk/data/new/ruby/parser_guess.rb.vm",
107
+ String.format("%s/guess/%s.rb", pluginDirectory, name));
108
+ break;
109
+ case "decoder":
110
+ description = String.format("Decodes %s-encoded files read by other file input plugins.", displayName);
111
+ extraGuesses.put("embulk/data/new/ruby/decoder_guess.rb.vm",
112
+ String.format("%s/guess/%s.rb", pluginDirectory, name));
113
+ break;
114
+ case "output":
115
+ description = String.format("Dumps records to %s.", displayName);
116
+ break;
117
+ case "file_output":
118
+ description = String.format("Stores files on %s.", displayName);
119
+ break;
120
+ case "formatter":
121
+ description = String.format("Formats %s files for other file output plugins.", displayName);
122
+ break;
123
+ case "encoder":
124
+ description = String.format("Encodes files using %s for other file output plugins.", displayName);
125
+ break;
126
+ case "filter":
127
+ description = String.format("%s", displayName);
128
+ break;
129
+ default:
130
+ throw new RuntimeException("FATAL: Invalid plugin category.");
131
+ }
132
+
133
+ //
134
+ // Generate project repository
135
+ //
136
+ final VelocityContext velocityContext = createVelocityContext(
137
+ author,
138
+ category,
139
+ description,
140
+ displayName,
141
+ displayCategory,
142
+ email,
143
+ embulkCategory,
144
+ this.embulkVersion,
145
+ expectedGitHubAccount,
146
+ fullProjectName,
147
+ javaClassName,
148
+ javaPackageName,
149
+ language,
150
+ name,
151
+ rubyClassName);
152
+ copyTemplated("embulk/data/new/README.md.vm", "README.md", velocityContext);
153
+ copy("embulk/data/new/LICENSE.txt", "LICENSE.txt");
154
+ copyTemplated("embulk/data/new/gitignore.vm", ".gitignore", velocityContext);
155
+
156
+ switch (language) {
157
+ case "ruby":
158
+ copy("embulk/data/new/ruby/Rakefile", "Rakefile");
159
+ copy("embulk/data/new/ruby/Gemfile", "Gemfile");
160
+ copy("embulk/data/new/ruby/.ruby-version", ".ruby-version");
161
+ copyTemplated("embulk/data/new/ruby/gemspec.vm",
162
+ fullProjectName + ".gemspec",
163
+ velocityContext);
164
+ copyTemplated(String.format("embulk/data/new/ruby/%s.rb.vm", category),
165
+ this.pluginPath,
166
+ velocityContext);
167
+ break;
168
+ case "java":
169
+ copy("embulk/data/new/java/gradle/wrapper/gradle-wrapper.jar",
170
+ "gradle/wrapper/gradle-wrapper.jar");
171
+ copy("embulk/data/new/java/gradle/wrapper/gradle-wrapper.properties",
172
+ "gradle/wrapper/gradle-wrapper.properties");
173
+ copy("embulk/data/new/java/gradlew.bat", "gradlew.bat");
174
+ copy("embulk/data/new/java/gradlew", "gradlew");
175
+ setExecutable("gradlew");
176
+ copy("embulk/data/new/java/config/checkstyle/checkstyle.xml", "config/checkstyle/checkstyle.xml");
177
+ copy("embulk/data/new/java/config/checkstyle/default.xml", "config/checkstyle/default.xml");
178
+ copyTemplated("embulk/data/new/java/build.gradle.vm", "build.gradle", velocityContext);
179
+ copyTemplated("embulk/data/new/java/plugin_loader.rb.vm", this.pluginPath, velocityContext);
180
+ copyTemplated(String.format("embulk/data/new/java/%s.java.vm", category),
181
+ String.format("src/main/java/%s/%s.java",
182
+ javaPackageName.replaceAll("\\.", "/"),
183
+ javaClassName),
184
+ velocityContext);
185
+ copyTemplated("embulk/data/new/java/test.java.vm",
186
+ String.format("src/test/java/%s/Test%s.java",
187
+ javaPackageName.replaceAll("\\.", "/"),
188
+ javaClassName),
189
+ velocityContext);
190
+ break;
191
+ }
192
+
193
+ for (Map.Entry<String, String> entry : extraGuesses.entrySet()) {
194
+ copyTemplated(entry.getKey(), entry.getValue(), velocityContext);
195
+ }
196
+
197
+ System.out.println("");
198
+ System.out.println("Plugin template is successfully generated.");
199
+
200
+ switch (language) {
201
+ case "ruby":
202
+ System.out.println("Next steps:");
203
+ System.out.println("");
204
+ System.out.printf(" $ cd %s\n", fullProjectName);
205
+ System.out.println(" $ bundle install # install one using rbenv & rbenv-build");
206
+ System.out.println(" $ bundle exec rake # build gem to be released");
207
+ System.out.println(" $ bundle exec embulk run config.yml # you can run plugin using this command");
208
+ break;
209
+ case "java":
210
+ System.out.println("Next steps:");
211
+ System.out.println("");
212
+ System.out.printf(" $ cd %s\n", fullProjectName);
213
+ System.out.println(" $ ./gradlew package");
214
+ }
215
+
216
+ success = true;
217
+ System.out.println("");
218
+ }
219
+ catch (Exception ex) {
220
+ ex.printStackTrace();
221
+ }
222
+ finally {
223
+ if (!success) {
224
+ System.out.println("Failed. Removing the directory created.");
225
+ deleteDirectoryTree(Paths.get(fullProjectName));
226
+ }
227
+ }
228
+ return success;
229
+ }
230
+
231
+ private static class LanguageAndCategory
232
+ {
233
+ private LanguageAndCategory(final String language, final String category)
234
+ {
235
+ this.language = language;
236
+ this.category = category;
237
+ }
238
+
239
+ public static LanguageAndCategory of(final String categoryWithLanguage)
240
+ {
241
+ switch (categoryWithLanguage) {
242
+ case "java-input": return new LanguageAndCategory("java", "input");
243
+ case "java-output": return new LanguageAndCategory("java", "output");
244
+ case "java-filter": return new LanguageAndCategory("java", "filter");
245
+ case "java-file-input": return new LanguageAndCategory("java", "file_input");
246
+ case "java-file-output": return new LanguageAndCategory("java", "file_output");
247
+ case "java-parser": return new LanguageAndCategory("java", "parser");
248
+ case "java-formatter": return new LanguageAndCategory("java", "formatter");
249
+ case "java-decoder": return new LanguageAndCategory("java", "decoder");
250
+ case "java-encoder": return new LanguageAndCategory("java", "encoder");
251
+ case "ruby-input": return new LanguageAndCategory("ruby", "input");
252
+ case "ruby-output": return new LanguageAndCategory("ruby", "output");
253
+ case "ruby-filter": return new LanguageAndCategory("ruby", "filter");
254
+ case "ruby-file-input":
255
+ throw new RuntimeException("ruby-file-input is not implemented yet. See #21 on github.");
256
+ case "ruby-file-output":
257
+ throw new RuntimeException("ruby-file-output is not implemented yet. See #22 on github.");
258
+ case "ruby-parser": return new LanguageAndCategory("ruby", "parser");
259
+ case "ruby-formatter": return new LanguageAndCategory("ruby", "formatter");
260
+ case "ruby-decoder":
261
+ throw new RuntimeException("ruby-decoder is not implemented yet. See #31 on github.");
262
+ case "ruby-encoder":
263
+ throw new RuntimeException("ruby-decoder is not implemented yet. See #32 on github.");
264
+ default:
265
+ throw new RuntimeException(String.format("Unknown category '%s'", categoryWithLanguage));
266
+ }
267
+ }
268
+
269
+ public String getLanguage()
270
+ {
271
+ return this.language;
272
+ }
273
+
274
+ public String getCategory()
275
+ {
276
+ return this.category;
277
+ }
278
+
279
+ private final String language;
280
+ private final String category;
281
+ }
282
+
283
+ private String getGitConfig(final String configName, final String defaultValue)
284
+ {
285
+ try {
286
+ final Process process = new ProcessBuilder("git", "config", configName).redirectErrorStream(true).start();
287
+ return CharStreams.toString(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8)).trim();
288
+ }
289
+ catch (Throwable ex) {
290
+ return "YOUR_NAME";
291
+ }
292
+ }
293
+
294
+ private Path deleteDirectoryTree(final Path path)
295
+ throws IOException
296
+ {
297
+ return Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
298
+ @Override
299
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attributes)
300
+ throws IOException
301
+ {
302
+ Files.delete(file);
303
+ return FileVisitResult.CONTINUE;
304
+ }
305
+
306
+ @Override
307
+ public FileVisitResult postVisitDirectory(Path directory, IOException exception)
308
+ throws IOException
309
+ {
310
+ Files.delete(directory);
311
+ return FileVisitResult.CONTINUE;
312
+ }
313
+ });
314
+ }
315
+
316
+ private String getDisplayName(final String name)
317
+ {
318
+ final String[] nameSplit = name.split("_");
319
+ final ArrayList<String> nameComposition = new ArrayList<String>();
320
+ for (String namePart : nameSplit) {
321
+ nameComposition.add(CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, namePart));
322
+ }
323
+ return Joiner.on(" ").join(nameComposition);
324
+ }
325
+
326
+ private VelocityContext createVelocityContext(final String author,
327
+ final String category,
328
+ final String description,
329
+ final String displayName,
330
+ final String displayCategory,
331
+ final String email,
332
+ final String embulkCategory,
333
+ final String embulkVersion,
334
+ final String expectedGitHubAccount,
335
+ final String fullProjectName,
336
+ final String javaClassName,
337
+ final String javaPackageName,
338
+ final String language,
339
+ final String name,
340
+ final String rubyClassName)
341
+ {
342
+ final VelocityContext velocityContext = new VelocityContext();
343
+ // TODO(dmikurube): Revisit this |argumentToRunEmbulkJava|.
344
+ // This is in the Velocity context because the value could not be in Velocity templates.
345
+ velocityContext.put("argumentToRunEmbulkJava", "\'-L ${file(\".\").absolutePath}\'");
346
+ velocityContext.put("author", author);
347
+ velocityContext.put("category", category);
348
+ velocityContext.put("description", description);
349
+ velocityContext.put("displayName", displayName);
350
+ velocityContext.put("displayCategory", displayCategory);
351
+ velocityContext.put("email", email);
352
+ velocityContext.put("embulkCategory", embulkCategory);
353
+ velocityContext.put("embulkVersion", embulkVersion);
354
+ velocityContext.put("expectedGitHubAccount", expectedGitHubAccount);
355
+ velocityContext.put("fullProjectName", fullProjectName);
356
+ velocityContext.put("javaClassName", javaClassName);
357
+ velocityContext.put("javaGuessClassName", javaClassName.replace("Plugin", "GuessPlugin"));
358
+ velocityContext.put("javaPackageName", javaPackageName);
359
+ velocityContext.put("language", language);
360
+ velocityContext.put("name", name);
361
+ velocityContext.put("rubyClassName", rubyClassName);
362
+ velocityContext.put("rubyGuessClassName", rubyClassName.replace("Plugin", "GuessPlugin"));
363
+ return velocityContext;
364
+ }
365
+
366
+ private void copy(String sourceResourcePath, String destinationFileName)
367
+ throws IOException
368
+ {
369
+ final Path destinationPath = this.pluginBasePath.resolve(destinationFileName);
370
+ Files.createDirectories(destinationPath.getParent());
371
+ Files.copy(EmbulkNew.class.getClassLoader().getResourceAsStream(sourceResourcePath), destinationPath);
372
+ }
373
+
374
+ private void copyTemplated(String sourceResourcePath, String destinationFileName, VelocityContext velocityContext)
375
+ throws IOException
376
+ {
377
+ try (InputStreamReader reader = new InputStreamReader(
378
+ EmbulkNew.class.getClassLoader().getResourceAsStream(sourceResourcePath))) {
379
+ final Path destinationPath = this.pluginBasePath.resolve(destinationFileName);
380
+ Files.createDirectories(destinationPath.getParent());
381
+ try (BufferedWriter writer = Files.newBufferedWriter(destinationPath, StandardCharsets.UTF_8)) {
382
+ this.velocityEngine.evaluate(velocityContext,
383
+ writer,
384
+ "embulk-new",
385
+ reader);
386
+ }
387
+ }
388
+ }
389
+
390
+ private void setExecutable(String targetFileName)
391
+ throws IOException
392
+ {
393
+ final Path targetPath = this.pluginBasePath.resolve(targetFileName);
394
+ final Set<PosixFilePermission> permissions =
395
+ new HashSet<PosixFilePermission>(Files.getPosixFilePermissions(targetPath));
396
+ permissions.add(PosixFilePermission.OWNER_EXECUTE);
397
+ permissions.add(PosixFilePermission.GROUP_EXECUTE);
398
+ permissions.add(PosixFilePermission.OTHERS_EXECUTE);
399
+ Files.setPosixFilePermissions(targetPath, permissions);
400
+ }
401
+
402
+ private final Path basePath;
403
+
404
+ private final String nameGiven;
405
+ private final String language;
406
+ private final String category;
407
+ private final String embulkVersion;
408
+
409
+ private final String embulkCategory;
410
+ private final String name;
411
+
412
+ private final String fullProjectName;
413
+ private final String pluginDirectory;
414
+ private final String pluginPath;
415
+
416
+ private final Path pluginBasePath;
417
+
418
+ private final VelocityEngine velocityEngine;
419
+ }
@@ -0,0 +1,248 @@
1
+ package org.embulk.cli;
2
+
3
+ import java.io.ByteArrayOutputStream;
4
+ import java.io.FileNotFoundException;
5
+ import java.io.InputStream;
6
+ import java.io.IOException;
7
+ import java.net.HttpURLConnection;
8
+ import java.net.URL;
9
+ import java.net.URISyntaxException;
10
+ import java.nio.file.Files;
11
+ import java.nio.file.Path;
12
+ import java.nio.file.Paths;
13
+ import java.nio.file.StandardCopyOption;
14
+ import java.util.jar.Attributes;
15
+ import java.util.jar.JarFile;
16
+ import java.util.jar.Manifest;
17
+ import java.util.regex.Matcher;
18
+ import java.util.regex.Pattern;
19
+
20
+ import org.apache.maven.artifact.versioning.ComparableVersion;
21
+
22
+ // It uses |java.net.HttpURLConnection| so that embulk-cli does not need additional dependedcies.
23
+ // TODO(dmikurube): Support HTTP proxy. The original Ruby version did not support as well, though.
24
+ public class EmbulkSelfUpdate
25
+ {
26
+ // TODO(dmikurube): Stop catching Exceptions here when embulk_run.rb is replaced to Java.
27
+ public void updateSelf(final String runningVersionString,
28
+ final String specifiedVersionString,
29
+ final String embulkRunRubyPathString,
30
+ final boolean isForced)
31
+ throws IOException, URISyntaxException
32
+ {
33
+ try {
34
+ updateSelfWithExceptions(runningVersionString, specifiedVersionString, embulkRunRubyPathString, isForced);
35
+ }
36
+ catch (Throwable ex) {
37
+ ex.printStackTrace();
38
+ throw ex;
39
+ }
40
+ }
41
+
42
+ private void updateSelfWithExceptions(final String runningVersionString,
43
+ final String specifiedVersionString,
44
+ final String embulkRunRubyPathString,
45
+ final boolean isForced)
46
+ throws IOException, URISyntaxException
47
+ {
48
+ final Path jarPathJava = Paths.get(
49
+ EmbulkSelfUpdate.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath());
50
+
51
+ if ((!Files.exists(jarPathJava)) || (!Files.isRegularFile(jarPathJava))) {
52
+ throw exceptionNoSingleJar();
53
+ }
54
+
55
+ if (embulkRunRubyPathString != null) {
56
+ final String[] splitRubyFile = embulkRunRubyPathString.split("!", 2);
57
+ if (splitRubyFile.length < 2) {
58
+ throw exceptionNoSingleJar();
59
+ }
60
+ final Path jarPathRuby = Paths.get(splitRubyFile[0]);
61
+ if (!jarPathJava.equals(jarPathRuby)) {
62
+ throw exceptionNoSingleJar();
63
+ }
64
+ }
65
+
66
+ final String targetVersionString;
67
+ if (specifiedVersionString != null) {
68
+ System.out.printf("Checking version %s...\n", specifiedVersionString);
69
+ targetVersionString = checkTargetVersion(specifiedVersionString);
70
+ if (targetVersionString == null) {
71
+ throw new RuntimeException(String.format("Specified version does not exist: %s", specifiedVersionString));
72
+ }
73
+ System.out.printf("Found version %s.\n", specifiedVersionString);
74
+ }
75
+ else {
76
+ System.out.println("Checking the latest version...");
77
+ final ComparableVersion runningVersion = new ComparableVersion(runningVersionString);
78
+ targetVersionString = checkLatestVersion();
79
+ final ComparableVersion targetVersion = new ComparableVersion(targetVersionString);
80
+ if (targetVersion.compareTo(runningVersion) <= 0) {
81
+ System.out.printf("Already up-to-date. %s is the latest version.\n", runningVersion);
82
+ return;
83
+ }
84
+ System.out.printf("Found a newer version %s.\n", targetVersion);
85
+ }
86
+
87
+ if (!Files.isWritable(jarPathJava)) {
88
+ throw new RuntimeException(String.format("The existing %s is not writable. May need to sudo?",
89
+ jarPathJava.toString()));
90
+ }
91
+
92
+ final URL downloadUrl = new URL(String.format("https://dl.bintray.com/embulk/maven/embulk-%s.jar",
93
+ targetVersionString));
94
+ System.out.printf("Downloading %s ...\n", downloadUrl.toString());
95
+
96
+ Path jarPathTemp = Files.createTempFile("embulk-selfupdate", ".jar");
97
+ try {
98
+ final HttpURLConnection connection = (HttpURLConnection) downloadUrl.openConnection();
99
+ try {
100
+ // Follow the redicrect from the Bintray URL.
101
+ connection.setInstanceFollowRedirects(true);
102
+ connection.setRequestMethod("GET");
103
+ connection.connect();
104
+ final int statusCode = connection.getResponseCode();
105
+ if (HttpURLConnection.HTTP_OK != statusCode) {
106
+ throw new FileNotFoundException(
107
+ String.format("Unexpected HTTP status code: %d", statusCode));
108
+ }
109
+ InputStream input = connection.getInputStream();
110
+ // TODO(dmikurube): Confirm if it is okay to replace a temp file created by Files.createTempFile.
111
+ Files.copy(input, jarPathTemp, StandardCopyOption.REPLACE_EXISTING);
112
+ Files.setPosixFilePermissions(jarPathTemp, Files.getPosixFilePermissions(jarPathJava));
113
+ }
114
+ finally {
115
+ connection.disconnect();
116
+ }
117
+
118
+ if (!isForced) { // Check corruption
119
+ final String versionJarTemp;
120
+ try {
121
+ versionJarTemp = getJarVersion(jarPathTemp);
122
+ }
123
+ catch (FileNotFoundException ex) {
124
+ throw new RuntimeException("Failed to check corruption. Downloaded version may include incompatible changes. Try the '-f' option to force updating without checking.", ex);
125
+ }
126
+ if (!versionJarTemp.equals(targetVersionString)) {
127
+ throw new RuntimeException(
128
+ String.format("Downloaded version does not match: %s (downloaded) / %s (target)",
129
+ versionJarTemp,
130
+ targetVersionString));
131
+ }
132
+ }
133
+ Files.move(jarPathTemp, jarPathJava, StandardCopyOption.REPLACE_EXISTING);
134
+ }
135
+ finally {
136
+ Files.deleteIfExists(jarPathTemp);
137
+ }
138
+ System.out.println(String.format("Updated to %s.", targetVersionString));
139
+ }
140
+
141
+ private RuntimeException exceptionNoSingleJar()
142
+ {
143
+ return new RuntimeException("Embulk is not installed as a single jar. \"selfupdate\" does not work. If you installed Embulk through gem, run \"gem install embulk\" instead.");
144
+ }
145
+
146
+ /**
147
+ * Checks the latest version from bintray.com.
148
+ *
149
+ * It passes all {@code IOException} and {@code RuntimeException} through out.
150
+ */
151
+ private String checkLatestVersion()
152
+ throws IOException
153
+ {
154
+ final URL bintrayUrl = new URL("https://bintray.com/embulk/maven/embulk/_latestVersion");
155
+ final HttpURLConnection connection = (HttpURLConnection) bintrayUrl.openConnection();
156
+ try {
157
+ // Stop HttpURLConnection from following redirects when the status code is 301 or 302.
158
+ connection.setInstanceFollowRedirects(false);
159
+ connection.setRequestMethod("GET");
160
+ connection.connect();
161
+ final int statusCode = connection.getResponseCode();
162
+ if (HttpURLConnection.HTTP_MOVED_TEMP != statusCode) {
163
+ throw new FileNotFoundException(
164
+ String.format("Unexpected HTTP status code: %d", statusCode));
165
+ }
166
+ final String location = connection.getHeaderField("Location");
167
+ final Matcher versionMatcher = VERSION_URL_PATTERN.matcher(location);
168
+ if (!versionMatcher.matches()) {
169
+ throw new FileNotFoundException(
170
+ String.format("Invalid version number in \"Location\" header: %s", location));
171
+ }
172
+ return versionMatcher.group(1);
173
+ }
174
+ finally {
175
+ connection.disconnect();
176
+ }
177
+ }
178
+
179
+ /**
180
+ * Checks the target version in bintray.com.
181
+ *
182
+ * It passes all {@code IOException} and {@code RuntimeException} through out.
183
+ */
184
+ private String checkTargetVersion(String version)
185
+ throws IOException
186
+ {
187
+ final URL bintrayUrl = new URL(String.format("https://bintray.com/embulk/maven/embulk/%s", version));
188
+ final HttpURLConnection connection = (HttpURLConnection) bintrayUrl.openConnection();
189
+ try {
190
+ connection.setInstanceFollowRedirects(false);
191
+ connection.setRequestMethod("GET");
192
+ connection.connect();
193
+ final int statusCode = connection.getResponseCode();
194
+ if (HttpURLConnection.HTTP_NOT_FOUND == statusCode) {
195
+ return null;
196
+ }
197
+ else if (HttpURLConnection.HTTP_OK != statusCode) {
198
+ throw new FileNotFoundException(
199
+ String.format("Unexpected HTTP status code: %d", statusCode));
200
+ }
201
+ else {
202
+ return version;
203
+ }
204
+ }
205
+ finally {
206
+ connection.disconnect();
207
+ }
208
+ }
209
+
210
+ private String getJarVersion(Path jarPath)
211
+ throws IOException
212
+ {
213
+ try (final JarFile jarFile = new JarFile(jarPath.toFile())) {
214
+ final Manifest manifest;
215
+ try {
216
+ manifest = jarFile.getManifest();
217
+ }
218
+ catch (IOException ex) {
219
+ throw new IOException("Version not found. Failed to load the manifest.", ex);
220
+ }
221
+ String manifestContents;
222
+ try (final ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
223
+ manifest.write(outputStream);
224
+ manifestContents = outputStream.toString();
225
+ }
226
+ catch (IOException ex) {
227
+ manifestContents = "(Failed to read the contents of the manifest.)";
228
+ }
229
+ final Attributes mainAttributes = manifest.getMainAttributes();
230
+ final String implementationVersion = mainAttributes.getValue(Attributes.Name.IMPLEMENTATION_VERSION);
231
+ if (implementationVersion == null) {
232
+ throw new IOException("Version not found. Failed to read \""
233
+ + Attributes.Name.IMPLEMENTATION_VERSION
234
+ + "\": " + manifestContents);
235
+ }
236
+ return implementationVersion;
237
+ }
238
+ catch (IOException ex) {
239
+ throw new IOException("Version not found. Failed to load the jar file.", ex);
240
+ }
241
+
242
+ // NOTE: Checking embulk/version.rb is no longer needed.
243
+ // The jar manifest with "Implementation-Version" has been included in Embulk jars from v0.4.0.
244
+ }
245
+
246
+ private static final Pattern VERSION_URL_PATTERN = Pattern.compile("^https?://.*/embulk/(\\d+\\.\\d+[^\\/]+).*$");
247
+ private static final Pattern VERSION_RUBY_PATTERN = Pattern.compile("^\\s*VERSION\\s*\\=\\s*(\\p{Graph}+)\\s*$");
248
+ }