embulk 0.8.18-java → 0.8.19-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (87) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +10 -0
  3. data/build.gradle +10 -3
  4. data/embulk-cli/build.gradle +2 -0
  5. data/embulk-cli/src/main/bat/selfrun.bat +98 -0
  6. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkExample.java +82 -0
  7. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkMigrate.java +458 -0
  8. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkNew.java +419 -0
  9. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkSelfUpdate.java +248 -0
  10. data/embulk-cli/src/main/sh/selfrun.sh +0 -103
  11. data/embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java +158 -143
  12. data/embulk-core/build.gradle +2 -2
  13. data/embulk-core/src/main/java/org/embulk/EmbulkVersion.java +109 -0
  14. data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java +11 -0
  15. data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java +29 -3
  16. data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java +47 -13
  17. data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java +6 -3
  18. data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java +385 -64
  19. data/embulk-core/src/main/java/org/embulk/spi/TempFileSpace.java +2 -1
  20. data/embulk-core/src/test/java/org/embulk/spi/TestPageBuilderReader.java +62 -0
  21. data/embulk-docs/src/built-in.rst +59 -21
  22. data/embulk-docs/src/customization.rst +8 -8
  23. data/embulk-docs/src/developers/index.rst +45 -0
  24. data/embulk-docs/src/index.rst +11 -7
  25. data/embulk-docs/src/recipe.rst +1 -1
  26. data/embulk-docs/src/recipe/{scheduled-csv-load-to-elasticsearch-kibana4.rst → scheduled-csv-load-to-elasticsearch-kibana5.rst} +26 -24
  27. data/embulk-docs/src/release.rst +1 -0
  28. data/embulk-docs/src/release/release-0.4.0.rst +1 -1
  29. data/embulk-docs/src/release/release-0.5.0.rst +1 -1
  30. data/embulk-docs/src/release/release-0.6.0.rst +1 -1
  31. data/embulk-docs/src/release/release-0.6.20.rst +1 -1
  32. data/embulk-docs/src/release/release-0.8.19.rst +43 -0
  33. data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +2 -2
  34. data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java +30 -1
  35. data/embulk-standards/src/test/java/org/embulk/standards/guess/TestCsvGuessPlugin.java +10 -0
  36. data/embulk-standards/src/test/java/org/embulk/standards/preview/TestFilePreview.java +73 -0
  37. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records.csv +5 -0
  38. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_guessed.yml +2 -0
  39. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_seed.yml +1 -0
  40. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes.csv +5 -0
  41. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_exec.yml +1 -0
  42. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_load.yml +19 -0
  43. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_previewed.csv +1 -0
  44. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple.csv +5 -0
  45. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_load.yml +19 -0
  46. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_previewed.csv +4 -0
  47. data/embulk-test/src/main/java/org/embulk/test/PreviewResultInputPlugin.java +65 -0
  48. data/embulk-test/src/main/java/org/embulk/test/TestingBulkLoader.java +5 -0
  49. data/embulk-test/src/main/java/org/embulk/test/TestingEmbulk.java +59 -2
  50. data/embulk.gemspec +2 -1
  51. data/lib/embulk/command/embulk_run.rb +11 -49
  52. data/lib/embulk/data/new/README.md.vm +106 -0
  53. data/lib/embulk/data/new/{gitignore.erb → gitignore.vm} +3 -3
  54. data/lib/embulk/data/new/java/{build.gradle.erb → build.gradle.vm} +8 -8
  55. data/lib/embulk/data/new/java/{decoder.java.erb → decoder.java.vm} +6 -4
  56. data/lib/embulk/data/new/java/{encoder.java.erb → encoder.java.vm} +7 -5
  57. data/lib/embulk/data/new/java/{file_input.java.erb → file_input.java.vm} +9 -7
  58. data/lib/embulk/data/new/java/{file_output.java.erb → file_output.java.vm} +7 -5
  59. data/lib/embulk/data/new/java/{filter.java.erb → filter.java.vm} +4 -3
  60. data/lib/embulk/data/new/java/{formatter.java.erb → formatter.java.vm} +5 -4
  61. data/lib/embulk/data/new/java/{input.java.erb → input.java.vm} +6 -4
  62. data/lib/embulk/data/new/java/{output.java.erb → output.java.vm} +7 -5
  63. data/lib/embulk/data/new/java/{parser.java.erb → parser.java.vm} +5 -4
  64. data/lib/embulk/data/new/java/plugin_loader.rb.vm +3 -0
  65. data/lib/embulk/data/new/java/test.java.vm +5 -0
  66. data/lib/embulk/data/new/ruby/decoder_guess.rb.vm +25 -0
  67. data/lib/embulk/data/new/ruby/{filter.rb.erb → filter.rb.vm} +2 -2
  68. data/lib/embulk/data/new/ruby/{formatter.rb.erb → formatter.rb.vm} +2 -2
  69. data/lib/embulk/data/new/ruby/gemspec.vm +20 -0
  70. data/lib/embulk/data/new/ruby/{input.rb.erb → input.rb.vm} +10 -10
  71. data/lib/embulk/data/new/ruby/{output.rb.erb → output.rb.vm} +7 -7
  72. data/lib/embulk/data/new/ruby/{parser.rb.erb → parser.rb.vm} +2 -2
  73. data/lib/embulk/data/new/ruby/parser_guess.rb.vm +65 -0
  74. data/lib/embulk/guess/csv.rb +5 -0
  75. data/lib/embulk/version.rb +22 -1
  76. metadata +55 -35
  77. data/lib/embulk/command/embulk_example.rb +0 -33
  78. data/lib/embulk/command/embulk_generate_bin.rb +0 -62
  79. data/lib/embulk/command/embulk_migrate_plugin.rb +0 -244
  80. data/lib/embulk/command/embulk_new_plugin.rb +0 -126
  81. data/lib/embulk/command/embulk_selfupdate.rb +0 -121
  82. data/lib/embulk/data/new/README.md.erb +0 -111
  83. data/lib/embulk/data/new/java/plugin_loader.rb.erb +0 -3
  84. data/lib/embulk/data/new/java/test.java.erb +0 -5
  85. data/lib/embulk/data/new/ruby/decoder_guess.rb.erb +0 -25
  86. data/lib/embulk/data/new/ruby/gemspec.erb +0 -20
  87. data/lib/embulk/data/new/ruby/parser_guess.rb.erb +0 -65
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 1d36e9ca66a386370910265a070608eea08890a7
4
- data.tar.gz: 2b07320950266fc836bb08e801cb4ea325c24781
3
+ metadata.gz: d4b7ef6ed2afcd543225f24b8fdf5a1b3dd0b29b
4
+ data.tar.gz: 1a70e4bc0134a580cc4c132369464cc829eb18ce
5
5
  SHA512:
6
- metadata.gz: ea70fe0768119bd5b2e801868299c90d3925a13fcd637d5f72ad663889d0b27cd4efde7e99cc08be3bffaab1752a332f2972b81d45fdd9de4e0497eb87ef9c11
7
- data.tar.gz: ec23e1279135ca1412f49fe6d294d02b1a88b47c36a35bf6b5441dbf0e31739d5c5ae9e93d472f550cdc2ae288d7a237088ef5f485ff50ca599c0adbd51c9152
6
+ metadata.gz: 812807d7b71534a9eb8e58c39375f4700bd82b8ce5a123dfbfa18e3be9518938c1707940d9fc7e55878d2e24df75152d0270c65e7dd1e3714845f439c97b467d
7
+ data.tar.gz: e495f2c1488b5b97c2f3b0fb9da15e64d434ba635af3206a65496c3a9d88bede388a47b39709cf043b1a1fba0cbbcb47258e26db9fd5672267a1e08e3f216772
data/README.md CHANGED
@@ -176,6 +176,16 @@ Task `dependencies` shows dependency tree of embulk-core project:
176
176
  ./gradlew :embulk-core:dependencies
177
177
  ```
178
178
 
179
+ ### Update JRuby
180
+
181
+ Task `updateJRuby` updates JRuby version of embulk-core project.
182
+
183
+ This is an example to update JRuby to `9.1.5.0`.
184
+
185
+ ```
186
+ ./gradlew updateJRuby -Pto=9.1.5.0
187
+ ```
188
+
179
189
  ### Documents
180
190
 
181
191
  Embulk uses Sphinx, YARD (Ruby API) and JavaDoc (Java API) for document generation.
@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards"), p
16
16
 
17
17
  allprojects {
18
18
  group = 'org.embulk'
19
- version = '0.8.18'
19
+ version = '0.8.19'
20
20
 
21
21
  ext {
22
22
  jrubyVersion = '9.1.5.0'
@@ -213,6 +213,9 @@ task cli(dependsOn: ':embulk-cli:shadowJar') {
213
213
  file('pkg').mkdirs()
214
214
  File f = file("pkg/embulk-${project.version}.jar")
215
215
  f.write("")
216
+ f.append("\n: <<END_OF_EMBULK_SELFRUN_BATCH_PART\r\n")
217
+ f.append(file("embulk-cli/src/main/bat/selfrun.bat").readBytes())
218
+ f.append("\r\nEND_OF_EMBULK_SELFRUN_BATCH_PART\r\n\n")
216
219
  f.append(file("embulk-cli/src/main/sh/selfrun.sh").readBytes())
217
220
  f.append(file("embulk-cli/build/libs/embulk-cli-${project.version}-all.jar").readBytes())
218
221
  f.setExecutable(true)
@@ -224,6 +227,8 @@ project(':embulk-cli') {
224
227
  apply plugin: 'com.github.johnrengelman.shadow'
225
228
 
226
229
  shadowJar {
230
+ // NOTE: This 'Implementation-Version' in the manifest is referred to provide the Embulk version at runtime.
231
+ // See also: embulk-core/src/main/java/org/embulk/EmbulkVersion.java
227
232
  manifest {
228
233
  attributes 'Implementation-Title': project.name,
229
234
  'Implementation-Version': project.version,
@@ -329,8 +334,10 @@ task setVersion {
329
334
  File gradle_ver = file('build.gradle')
330
335
  gradle_ver.write(gradle_ver.getText().replaceFirst("version = '(\\d+)(\\.\\d+){2}'", "version = '${to}'"))
331
336
 
337
+ // TODO(v2)[#562]: Remove this part to update the version number in lib/embulk/version.rb.
338
+ // https://github.com/embulk/embulk/issues/562
332
339
  File ruby_ver = file('lib/embulk/version.rb')
333
- ruby_ver.write(ruby_ver.getText().replaceFirst("VERSION = '(\\d+)(\\.\\d+){2}'", "VERSION = '${to}'"))
340
+ ruby_ver.write(ruby_ver.getText().replaceFirst("VERSION_INTERNAL = '(\\d+)(\\.\\d+){2}'", "VERSION_INTERNAL = '${to}'"))
334
341
 
335
342
  List<String> docs = [
336
343
  'README.md',
@@ -359,7 +366,7 @@ task updateJRuby {
359
366
  File gemspec_ver = file('embulk.gemspec')
360
367
  gemspec_ver.write(gemspec_ver.getText().replaceFirst("gem.add_dependency \"jruby-jars\", '= (\\d+)(\\.\\d+){3}'", "gem.add_dependency \"jruby-jars\", '= ${to}'"))
361
368
 
362
- File migrate_plugin_ver = file('lib/embulk/command/embulk_migrate_plugin.rb')
369
+ File migrate_plugin_ver = file('embulk-cli/src/main/java/org/embulk/cli/EmbulkMigrate.java')
363
370
  migrate_plugin_ver.write(migrate_plugin_ver.getText().replaceFirst("\".ruby-version\", \"jruby-(\\d+)(\\.\\d+){3}\"", "\".ruby-version\", \"jruby-${to}\""))
364
371
 
365
372
  File gemfile_lock_ver = file('Gemfile.lock')
@@ -1,6 +1,8 @@
1
1
  dependencies {
2
2
  compile project(':embulk-core')
3
3
  compile project(':embulk-standards')
4
+ compile 'org.apache.maven:maven-artifact:3.3.9'
5
+ compile 'org.apache.velocity:velocity:1.7'
4
6
 
5
7
  testCompile project(':embulk-core').sourceSets.test.output
6
8
  }
@@ -0,0 +1,98 @@
1
+ @echo off
2
+
3
+ setlocal
4
+
5
+ set this=%~f0
6
+ set java_args=
7
+ set jruby_args=
8
+ set default_optimize=
9
+ set overwrite_optimize=
10
+ set status=
11
+ set error=
12
+ set args=
13
+
14
+ rem In jar file, cannot goto ahread for some reason.
15
+
16
+ for %%a in ( %* ) do (
17
+ call :check_arg %%a
18
+ )
19
+
20
+ if "%error%" == "true" exit /b 1
21
+
22
+ set optimize=false
23
+ if "%overwrite_optimize%" == "true" (
24
+ set optimize=true
25
+ ) else (
26
+ if "%default_optimize%" == "true" (
27
+ if not "%overwrite_optimize%" == "false" (
28
+ set optimize=true
29
+ )
30
+ )
31
+ )
32
+
33
+ if "%optimize%" == "true" (
34
+ set java_args=-XX:+AggressiveOpts -XX:+UseConcMarkSweepGC %java_args%
35
+ ) else (
36
+ set java_args=-XX:+AggressiveOpts -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none %java_args%
37
+ )
38
+
39
+ java %java_args% -jar %this% %jruby_args% %args%
40
+
41
+ endlocal
42
+
43
+ exit /b %ERRORLEVEL%
44
+
45
+ :check_arg
46
+ set arg=%*
47
+
48
+ rem Remove double quotations
49
+ set p1=%arg:~0,1%
50
+ set p1=%p1:"=%
51
+ set p2=%arg:~-1,1%
52
+ set p2=%p2:"=%
53
+ set arg=%p1%%arg:~1,-1%%p2%
54
+
55
+ if "%status%" == "rest" (
56
+ set args=%args% %arg%
57
+
58
+ ) else if "%status%" == "read" (
59
+ call :read_file %arg%
60
+
61
+ ) else if "%arg%" == "-J+O" (
62
+ set overwrite_optimize=true
63
+ set status=rest
64
+
65
+ ) else if "%arg%" == "-J-O" (
66
+ set overwrite_optimize=false
67
+ set status=rest
68
+
69
+ ) else if "%arg:~0,2%" == "-J" (
70
+ if not "%arg:~2%" == "" (
71
+ set java_args=%java_args% %arg:~2%
72
+ ) else (
73
+ set status=read
74
+ )
75
+
76
+ ) else if "%arg:~0,2%" == "-R" (
77
+ set jruby_args=%jruby_args% %arg%
78
+
79
+ ) else if "%arg%" == "run" (
80
+ set default_optimize=true
81
+ set args=%args% %arg%
82
+ set status=rest
83
+
84
+ ) else (
85
+ set args=%args% %arg%
86
+ set status=rest
87
+ )
88
+ exit /b
89
+
90
+ :read_file
91
+ if not exist "%~1" (
92
+ echo "failed to load java argument file."
93
+ set error=true
94
+ ) else (
95
+ for /f "delims=" %%i in (%~1) do set java_args=%java_args% %%i
96
+ )
97
+ set status=
98
+ exit /b
@@ -0,0 +1,82 @@
1
+ package org.embulk.cli;
2
+
3
+ import java.io.IOException;
4
+ import java.io.OutputStream;
5
+ import java.nio.charset.StandardCharsets;
6
+ import java.nio.file.Files;
7
+ import java.nio.file.Path;
8
+ import java.nio.file.Paths;
9
+ import java.util.zip.GZIPOutputStream;
10
+
11
+ public class EmbulkExample
12
+ {
13
+ public void createExample(final String basePathInString)
14
+ throws IOException
15
+ {
16
+ createExample(Paths.get(basePathInString));
17
+ }
18
+
19
+ public void createExample(final Path basePath)
20
+ throws IOException
21
+ {
22
+ // TODO(dmikurube): Log with a kind of loggers instead of |System.out|.
23
+ System.out.printf("Creating %s directory...\n", basePath.toString());
24
+
25
+ System.out.printf(" Creating %s/\n", basePath.toString());
26
+ final Path csvPath = basePath.resolve("csv");
27
+ Files.createDirectories(csvPath);
28
+ System.out.printf(" Creating %s/\n", csvPath.toString());
29
+
30
+ final Path csvSamplePath = csvPath.resolve("sample_01.csv.gz");
31
+ System.out.printf(" Creating %s\n", csvSamplePath.toString());
32
+ outputSampleCsv(csvSamplePath);
33
+
34
+ final Path ymlSamplePath = basePath.resolve("seed.yml");
35
+ System.out.printf(" Creating %s\n", ymlSamplePath.toString());
36
+ outputSampleYml(csvPath, ymlSamplePath);
37
+
38
+ System.out.println("");
39
+ System.out.println("Run following subcommands to try embulk:");
40
+ System.out.println("");
41
+ System.out.printf(" 1. embulk guess %s -o config.yml\n", ymlSamplePath.toString());
42
+ System.out.println(" 2. embulk preview config.yml");
43
+ System.out.println(" 3. embulk run config.yml");
44
+ System.out.println("");
45
+ }
46
+
47
+ private void outputSampleCsv(final Path csvSamplePath)
48
+ throws IOException
49
+ {
50
+ // TODO(dmikurube): Move the data into Java resources.
51
+ StringBuilder csvBuilder = new StringBuilder();
52
+ csvBuilder.append("id,account,time,purchase,comment\n");
53
+ csvBuilder.append("1,32864,2015-01-27 19:23:49,20150127,embulk\n");
54
+ csvBuilder.append("2,14824,2015-01-27 19:01:23,20150127,embulk jruby\n");
55
+ csvBuilder.append("3,27559,2015-01-28 02:20:02,20150128,\"Embulk \"\"csv\"\" parser plugin\"\n");
56
+ csvBuilder.append("4,11270,2015-01-29 11:54:36,20150129,NULL\n");
57
+ csvBuilder.append("\n");
58
+ byte[] csvSample = csvBuilder.toString().getBytes(StandardCharsets.UTF_8);
59
+
60
+ try (GZIPOutputStream output = new GZIPOutputStream(Files.newOutputStream(csvSamplePath))) {
61
+ output.write(csvSample);
62
+ }
63
+ }
64
+
65
+ private void outputSampleYml(final Path csvPath, final Path ymlSamplePath)
66
+ throws IOException
67
+ {
68
+ // TODO(dmikurube): Move the data into Java resources.
69
+ StringBuilder ymlBuilder = new StringBuilder();
70
+ ymlBuilder.append("in:\n");
71
+ ymlBuilder.append(" type: file\n");
72
+ ymlBuilder.append(String.format(" path_prefix: \"%s\"\n",
73
+ csvPath.toAbsolutePath().resolve("sample_").toString()));
74
+ ymlBuilder.append("out:\n");
75
+ ymlBuilder.append(" type: stdout\n");
76
+ byte[] ymlSample = ymlBuilder.toString().getBytes(StandardCharsets.UTF_8);
77
+
78
+ try (OutputStream output = Files.newOutputStream(ymlSamplePath)) {
79
+ output.write(ymlSample);
80
+ }
81
+ }
82
+ }
@@ -0,0 +1,458 @@
1
+ package org.embulk.cli;
2
+
3
+ import java.io.IOException;
4
+ import java.nio.charset.StandardCharsets;
5
+ import java.nio.file.Files;
6
+ import java.nio.file.FileSystems;
7
+ import java.nio.file.FileVisitResult;
8
+ import java.nio.file.NoSuchFileException;
9
+ import java.nio.file.Path;
10
+ import java.nio.file.PathMatcher;
11
+ import java.nio.file.Paths;
12
+ import java.nio.file.SimpleFileVisitor;
13
+ import java.nio.file.StandardOpenOption;
14
+ import java.nio.file.attribute.BasicFileAttributes;
15
+ import java.util.ArrayList;
16
+ import java.util.Arrays;
17
+ import java.util.List;
18
+ import java.util.HashSet;
19
+ import java.util.Set;
20
+ import java.util.regex.Matcher;
21
+ import java.util.regex.Pattern;
22
+
23
+ import com.google.common.base.Joiner;
24
+ import com.google.common.collect.ImmutableList;
25
+
26
+ import org.apache.maven.artifact.versioning.ComparableVersion;
27
+
28
+ public class EmbulkMigrate
29
+ {
30
+ public void migratePlugin(final String pathInString, final String thisEmbulkVersion)
31
+ throws IOException
32
+ {
33
+ migratePlugin(Paths.get(pathInString), thisEmbulkVersion);
34
+ }
35
+
36
+ public void migratePlugin(final Path path, final String thisEmbulkVersion)
37
+ throws IOException
38
+ {
39
+ final Migrator migrator = new Migrator(path);
40
+
41
+ List<Matcher> matchedEmbulkCoreInGradle = migrator.matchersRecursive("build.gradle", EMBULK_CORE_IN_GRADLE);
42
+ List<Matcher> matchedNewEmbulkInGemspec = migrator.matchersRecursive("*.gemspec", NEW_EMBULK_IN_GEMSPEC);
43
+ List<Matcher> matchedOldEmbulkInGemspec = migrator.matchersRecursive("*.gemspec", OLD_EMBULK_IN_GEMSPEC);
44
+
45
+ final Language language;
46
+ final ComparableVersion fromVersion;
47
+ if (!matchedEmbulkCoreInGradle.isEmpty()) {
48
+ language = Language.JAVA;
49
+ fromVersion = new ComparableVersion(matchedEmbulkCoreInGradle.get(0).group(1).replace("+", "0"));
50
+ System.out.printf("Detected Java plugin for Embulk %s...\n", fromVersion.toString());
51
+ }
52
+ else if (!matchedNewEmbulkInGemspec.isEmpty()) {
53
+ language = Language.RUBY;
54
+ fromVersion = new ComparableVersion(matchedNewEmbulkInGemspec.get(0).group(1));
55
+ System.out.printf("Detected Ruby plugin for Embulk %s...\n", fromVersion.toString());
56
+ }
57
+ else if (!matchedOldEmbulkInGemspec.isEmpty()) {
58
+ language = Language.RUBY;
59
+ fromVersion = new ComparableVersion("0.1.0");
60
+ System.out.println("Detected Ruby plugin for unknown Embulk version...");
61
+ }
62
+ else {
63
+ throw new RuntimeException("Failed to detect plugin language and dependency version");
64
+ }
65
+
66
+ switch (language) {
67
+ case JAVA:
68
+ migrateJavaPlugin(migrator, fromVersion, thisEmbulkVersion);
69
+ break;
70
+ case RUBY:
71
+ migrateRubyPlugin(migrator, fromVersion, thisEmbulkVersion);
72
+ break;
73
+ }
74
+
75
+ if (migrator.getModifiedFiles().isEmpty()) {
76
+ System.out.println("Done. No files are modified.");
77
+ }
78
+ else {
79
+ System.out.println("Done. Please check modifieid files.");
80
+ }
81
+ }
82
+
83
+ private void migrateJavaPlugin(final Migrator migrator,
84
+ final ComparableVersion fromVersion,
85
+ final String thisEmbulkVersion)
86
+ throws IOException
87
+ {
88
+ if (fromVersion.compareTo(new ComparableVersion("0.7.0")) < 0) {
89
+ // rename CommitReport to TaskReport
90
+ migrator.replaceRecursive("*.java", Pattern.compile("(CommitReport)"), 1, "TaskReport");
91
+ migrator.replaceRecursive("*.java", Pattern.compile("(commitReport)"), 1, "taskReport");
92
+ }
93
+
94
+ // upgrade gradle version
95
+ if (migrator.match("gradle/wrapper/gradle-wrapper.properties", GRADLE_VERSION_IN_WRAPPER)) {
96
+ // gradle < 3.2.1
97
+ migrator.copy("embulk/data/new/java/gradle/wrapper/gradle-wrapper.properties",
98
+ "gradle/wrapper/gradle-wrapper.properties");
99
+ migrator.copy("embulk/data/new/java/gradle/wrapper/gradle-wrapper.jar",
100
+ "gradle/wrapper/gradle-wrapper.jar");
101
+ }
102
+
103
+ // Add a method |jsonColumn| before the method |timestampColumn| which should exist.
104
+ if (!migrator.matchRecursive("*.java", JSON_COLUMN_METHOD_IN_ALL_JAVA)) {
105
+ final List<Matcher> matchers = migrator.matchersRecursive("*.java", TIMESTAMP_COLUMN_METHOD_IN_ALL_JAVA);
106
+ if (!matchers.isEmpty()) {
107
+ final String indent = matchers.get(0).group(1);
108
+ final String JSON_COLUMN_METHOD = Joiner.on("\n").join(
109
+ "",
110
+ indent + "public void jsonColumn(Column column) {",
111
+ indent + " throw new UnsupportedOperationException(\"This plugin doesn't support json type. Please try to upgrade version of the plugin using 'embulk gem update' command. If the latest version still doesn't support json type, please contact plugin developers, or change configuration of input plugin not to use json type.\");",
112
+ indent + "}",
113
+ "",
114
+ indent + "@Override",
115
+ "");
116
+ migrator.replaceRecursive("*.java",
117
+ TIMESTAMP_COLUMN_METHOD_AFTER_NEWLINE_IN_ALL_JAVA,
118
+ 1,
119
+ JSON_COLUMN_METHOD);
120
+ }
121
+ }
122
+
123
+ // Add |sourceCompatibility| and |targetCompatibility| in build.gradle before |dependencies| existing.
124
+ if (!migrator.match("build.gradle", TARGET_COMPATIBILITY_IN_GRADLE)) {
125
+ migrator.insertLine("build.gradle", DEPENDENCIES_IN_GRADLE, new StringUpsert() {
126
+ @Override
127
+ public String getUpsertd(Matcher matcher) {
128
+ return String.format("%stargetCompatibility = 1.7\n", matcher.group(1));
129
+ }
130
+ });
131
+ }
132
+ if (!migrator.match("build.gradle", SOURCE_COMPATIBILITY_IN_GRADLE)) {
133
+ migrator.insertLine("build.gradle", TARGET_COMPATIBILITY_IN_GRADLE_WITH_INDENT, new StringUpsert() {
134
+ @Override
135
+ public String getUpsertd(Matcher matcher) {
136
+ return String.format("%ssourceCompatibility = 1.7\n", matcher.group(1));
137
+ }
138
+ });
139
+ }
140
+
141
+ // Add the |checkstyle| Gradle plugin before the |java| plugin.
142
+ if (!migrator.match("build.gradle", CHECKSTYLE_PLUGIN_IN_GRADLE)) {
143
+ migrator.insertLine("build.gradle", JAVA_PLUGIN_IN_GRADLE, new StringUpsert() {
144
+ @Override
145
+ public String getUpsertd(Matcher matcher) {
146
+ return String.format("%sid%s%scheckstyle%s",
147
+ matcher.group(1),
148
+ matcher.group(2),
149
+ matcher.group(3),
150
+ matcher.group(3));
151
+ }
152
+ });
153
+ migrator.copy("embulk/data/new/java/config/checkstyle/checkstyle.xml",
154
+ "config/checkstyle/checkstyle.xml");
155
+ }
156
+
157
+ // Add |checkstyle| settings before the |gem| task existing.
158
+ if (!migrator.match("build.gradle", CHECKSTYLE_CONFIGURATION_IN_GRADLE)) {
159
+ migrator.copy("embulk/data/new/java/config/checkstyle/default.xml",
160
+ "config/checkstyle/default.xml");
161
+ migrator.insertLine("build.gradle", GEM_TASK_IN_GRADLE, new StringUpsert() {
162
+ @Override
163
+ public String getUpsertd(Matcher matcher) {
164
+ final Joiner joiner = Joiner.on("\n");
165
+ final String indent = matcher.group(1);
166
+ return joiner.join(
167
+ indent + "checkstyle {",
168
+ indent + " configFile = file(\"${project.rootDir}/config/checkstyle/checkstyle.xml\")",
169
+ indent + " toolVersion = '6.14.1'",
170
+ indent + "}",
171
+ indent + "checkstyleMain {",
172
+ indent + " configFile = file(\"${project.rootDir}/config/checkstyle/default.xml\")",
173
+ indent + " ignoreFailures = true",
174
+ indent + "}",
175
+ indent + "checkstyleTest {",
176
+ indent + " configFile = file(\"${project.rootDir}/config/checkstyle/default.xml\")",
177
+ indent + " ignoreFailures = true",
178
+ indent + "}",
179
+ indent + "task checkstyle(type: Checkstyle) {",
180
+ indent + " classpath = sourceSets.main.output + sourceSets.test.output",
181
+ indent + " source = sourceSets.main.allJava + sourceSets.test.allJava",
182
+ indent + "}");
183
+ }
184
+ });
185
+ }
186
+
187
+ // Update |embulk-core| and |embulk-standards| versions depending.
188
+ migrator.replaceRecursive("build.gradle", EMBULK_CORE_OR_STANDARDS_IN_GRADLE, 1, thisEmbulkVersion);
189
+ }
190
+
191
+ private void migrateRubyPlugin(final Migrator migrator,
192
+ final ComparableVersion fromVersion,
193
+ final String thisEmbulkVersion)
194
+ throws IOException
195
+ {
196
+ migrator.write(".ruby-version", "jruby-9.1.5.0");
197
+
198
+ // Update |embulk| version depending.
199
+ if (fromVersion.compareTo(new ComparableVersion("0.1.0")) <= 0) {
200
+ // Add add_development_dependency.
201
+ migrator.insertLineRecursive("*.gemspec", DEVELOPMENT_DEPENDENCY_IN_GEMSPEC, new StringUpsert() {
202
+ @Override
203
+ public String getUpsertd(Matcher matcher) {
204
+ return String.format("%s.add_development_dependency 'embulk', ['>= %s']",
205
+ matcher.group(1),
206
+ thisEmbulkVersion);
207
+ }
208
+ });
209
+ }
210
+ else {
211
+ if (migrator.replaceRecursive("*.gemspec", EMBULK_DEPENDENCY_PRERELEASE_IN_GEMSPEC, 1,
212
+ ">= " + thisEmbulkVersion).isEmpty()) {
213
+ migrator.replaceRecursive("*.gemspec", EMBULK_DEPENDENCY_IN_GEMSPEC, 1, thisEmbulkVersion);
214
+ }
215
+ }
216
+ }
217
+
218
+ private class Migrator {
219
+ private Migrator(Path basePath) {
220
+ this.basePath = basePath;
221
+ this.modifiedFiles = new HashSet<Path>();
222
+ }
223
+
224
+ public Path getBasePath()
225
+ {
226
+ return this.basePath;
227
+ }
228
+
229
+ public Set<Path> getModifiedFiles()
230
+ {
231
+ return this.modifiedFiles;
232
+ }
233
+
234
+ public void copy(String sourceResourcePath, String destinationFileName)
235
+ throws IOException
236
+ {
237
+ Path destinationPath = this.basePath.resolve(destinationFileName);
238
+ Files.createDirectories(destinationPath.getParent());
239
+ Files.copy(EmbulkMigrate.class.getClassLoader().getResourceAsStream(sourceResourcePath), destinationPath);
240
+ }
241
+
242
+ public boolean match(String filePath, Pattern pattern)
243
+ throws IOException
244
+ {
245
+ final Matcher matcher = pattern.matcher(read(this.basePath.resolve(filePath)));
246
+ return matcher.find();
247
+ }
248
+
249
+ public boolean matchRecursive(String baseFileNameGlob, Pattern pattern)
250
+ throws IOException
251
+ {
252
+ return !matchersRecursive(baseFileNameGlob, pattern).isEmpty();
253
+ }
254
+
255
+ public List<Matcher> matchersRecursive(final String baseFileNameGlob, final Pattern pattern)
256
+ throws IOException
257
+ {
258
+ final ImmutableList.Builder<Matcher> matchers = ImmutableList.<Matcher>builder();
259
+ final PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher("glob:" + baseFileNameGlob);
260
+ Files.walkFileTree(this.basePath, new SimpleFileVisitor<Path>() {
261
+ @Override
262
+ public FileVisitResult visitFile(Path filePath, BasicFileAttributes attributes) throws IOException {
263
+ if (pathMatcher.matches(filePath.getFileName())) {
264
+ final Matcher matcher = pattern.matcher(read(filePath));
265
+ if (matcher.find()) {
266
+ matchers.add(matcher);
267
+ }
268
+ }
269
+ return FileVisitResult.CONTINUE;
270
+ }
271
+ });
272
+ return matchers.build();
273
+ }
274
+
275
+ public List<Matcher> replaceRecursive(final String baseFileNameGlob,
276
+ final Pattern pattern,
277
+ final int index,
278
+ final String immediate)
279
+ throws IOException
280
+ {
281
+ final ImmutableList.Builder<Matcher> matchers = ImmutableList.<Matcher>builder();
282
+ final PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher("glob:" + baseFileNameGlob);
283
+ Files.walkFileTree(this.basePath, new SimpleFileVisitor<Path>() {
284
+ @Override
285
+ public FileVisitResult visitFile(Path filePath, BasicFileAttributes attributes) throws IOException {
286
+ if (pathMatcher.matches(filePath.getFileName())) {
287
+ final String originalData = read(filePath);
288
+ Matcher first = null;
289
+ int position = 0;
290
+ String modifiedData = originalData;
291
+ while (position < modifiedData.length()) {
292
+ final Matcher matcher = pattern.matcher(modifiedData.substring(position));
293
+ if (!matcher.matches()) {
294
+ break;
295
+ }
296
+ if (first == null) {
297
+ first = matcher;
298
+ }
299
+ final String replacingString = immediate;
300
+ modifiedData =
301
+ modifiedData.substring(0, matcher.start(index) - 1) +
302
+ replacingString +
303
+ modifiedData.substring(matcher.end(index));
304
+ position =
305
+ matcher.start(index) +
306
+ replacingString.length() +
307
+ (matcher.end() - matcher.end(index));
308
+ }
309
+ if (first != null) {
310
+ modify(filePath, modifiedData);
311
+ matchers.add(first);
312
+ }
313
+ }
314
+ return FileVisitResult.CONTINUE;
315
+ }
316
+ });
317
+ return matchers.build();
318
+ }
319
+
320
+ public boolean insertLine(Path filePath, Pattern pattern, StringUpsert stringUpsert)
321
+ throws IOException
322
+ {
323
+ final String originalData = read(filePath);
324
+ final Matcher matcher = pattern.matcher(originalData);
325
+ if (matcher.find()) {
326
+ final String preMatch = originalData.substring(0, matcher.start());
327
+ final int lineNumber = preMatch.split("\n").length;
328
+ final String replacingString = stringUpsert.getUpsertd(matcher);
329
+ List<String> lines = new ArrayList<String>(Arrays.asList(originalData.split("\n")));
330
+ lines.add(lineNumber + 1, replacingString);
331
+ final String modifiedData = Joiner.on("\n").join(lines);
332
+ modify(filePath, modifiedData);
333
+ return true;
334
+ }
335
+ return false;
336
+ }
337
+
338
+ public boolean insertLine(String filePath, Pattern pattern, StringUpsert stringUpsert)
339
+ throws IOException
340
+ {
341
+ return insertLine(this.basePath.resolve(filePath), pattern, stringUpsert);
342
+ }
343
+
344
+ public void insertLineRecursive(final String baseFileNameGlob,
345
+ final Pattern pattern,
346
+ final StringUpsert stringUpsert)
347
+ throws IOException
348
+ {
349
+ final ImmutableList.Builder<Matcher> matchers = ImmutableList.<Matcher>builder();
350
+ final PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher("glob:" + baseFileNameGlob);
351
+ Files.walkFileTree(this.basePath, new SimpleFileVisitor<Path>() {
352
+ @Override
353
+ public FileVisitResult visitFile(Path filePath, BasicFileAttributes attributes) throws IOException {
354
+ if (pathMatcher.matches(filePath.getFileName())) {
355
+ insertLine(filePath, pattern, stringUpsert);
356
+ }
357
+ return FileVisitResult.CONTINUE;
358
+ }
359
+ });
360
+ }
361
+
362
+ public void write(String fileName, String writtenData)
363
+ throws IOException
364
+ {
365
+ Path destinationPath = this.basePath.resolve(fileName);
366
+ Files.createDirectories(destinationPath.getParent());
367
+ modify(destinationPath, writtenData);
368
+ }
369
+
370
+ private void modify(Path filePath, String modifiedData)
371
+ throws IOException
372
+ {
373
+ final String originalData = read(filePath);
374
+ if (!originalData.equals(modifiedData)) {
375
+ Files.write(filePath, modifiedData.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
376
+ if (!this.modifiedFiles.contains(filePath)) {
377
+ if (originalData.isEmpty()) {
378
+ System.out.printf(" Created %s\n", filePath.toString());
379
+ }
380
+ else {
381
+ System.out.printf(" Modified %s\n", filePath.toString());
382
+ }
383
+ this.modifiedFiles.add(filePath);
384
+ }
385
+ }
386
+ }
387
+
388
+ private String read(Path filePath)
389
+ throws IOException
390
+ {
391
+ // assumes source code is written in UTF-8.
392
+ return new String(readBytes(filePath), StandardCharsets.UTF_8);
393
+ }
394
+
395
+ private byte[] readBytes(Path filePath)
396
+ throws IOException
397
+ {
398
+ try {
399
+ return Files.readAllBytes(filePath);
400
+ }
401
+ catch (NoSuchFileException ex) {
402
+ return new byte[0];
403
+ }
404
+ }
405
+
406
+ private final Path basePath;
407
+ private final Set<Path> modifiedFiles;
408
+ }
409
+
410
+ private interface StringUpsert {
411
+ public String getUpsertd(Matcher matcher);
412
+ }
413
+
414
+ private enum Language {
415
+ JAVA,
416
+ RUBY,
417
+ ;
418
+ }
419
+
420
+ private static final Pattern EMBULK_CORE_IN_GRADLE = Pattern.compile(
421
+ "org\\.embulk:embulk-core:([\\d\\.\\+]+)?");
422
+ private static final Pattern NEW_EMBULK_IN_GEMSPEC = Pattern.compile(
423
+ "add_(?:development_)?dependency\\s+\\W+embulk\\W+\\s+([\\d\\.]+)\\W+");
424
+ private static final Pattern OLD_EMBULK_IN_GEMSPEC = Pattern.compile(
425
+ "embulk-");
426
+ private static final Pattern GRADLE_VERSION_IN_WRAPPER = Pattern.compile(
427
+ "gradle-[23]\\.\\d+(\\.\\d+)?-/");
428
+ private static final Pattern JSON_COLUMN_METHOD_IN_ALL_JAVA = Pattern.compile(
429
+ "void\\s+jsonColumn");
430
+ private static final Pattern TIMESTAMP_COLUMN_METHOD_IN_ALL_JAVA = Pattern.compile(
431
+ "^(\\W+).*?void\\s+timestampColumn", Pattern.MULTILINE);
432
+ private static final Pattern TIMESTAMP_COLUMN_METHOD_AFTER_NEWLINE_IN_ALL_JAVA = Pattern.compile(
433
+ "(\\r?\\n)(\\W+).*?void\\s+timestampColumn");
434
+ private static final Pattern TARGET_COMPATIBILITY_IN_GRADLE = Pattern.compile(
435
+ "targetCompatibility");
436
+ private static final Pattern SOURCE_COMPATIBILITY_IN_GRADLE = Pattern.compile(
437
+ "sourceCompatibility");
438
+ private static final Pattern DEPENDENCIES_IN_GRADLE = Pattern.compile(
439
+ "^([ \\t]*)dependencies\\s*\\{", Pattern.MULTILINE);
440
+ private static final Pattern TARGET_COMPATIBILITY_IN_GRADLE_WITH_INDENT = Pattern.compile(
441
+ "^([ \\t]*)targetCompatibility", Pattern.MULTILINE);
442
+ private static final Pattern CHECKSTYLE_PLUGIN_IN_GRADLE = Pattern.compile(
443
+ "id\\s+(?<quote>[\"\'])checkstyle\\k<quote>");
444
+ private static final Pattern JAVA_PLUGIN_IN_GRADLE = Pattern.compile(
445
+ "^([ \t]*)id( +)([\"\'])java[\"\']", Pattern.MULTILINE);
446
+ private static final Pattern CHECKSTYLE_CONFIGURATION_IN_GRADLE = Pattern.compile(
447
+ "checkstyle\\s+\\{");
448
+ private static final Pattern GEM_TASK_IN_GRADLE = Pattern.compile(
449
+ "^([ \\t]*)task\\s+gem\\W.*\\{", Pattern.MULTILINE);
450
+ private static final Pattern EMBULK_CORE_OR_STANDARDS_IN_GRADLE = Pattern.compile(
451
+ "org\\.embulk:embulk-(?:core|standards):([\\d\\.\\+]+)?");
452
+ private static final Pattern DEVELOPMENT_DEPENDENCY_IN_GEMSPEC = Pattern.compile(
453
+ "([ \\t]*\\w+)\\.add_development_dependency");
454
+ private static final Pattern EMBULK_DEPENDENCY_PRERELEASE_IN_GEMSPEC = Pattern.compile(
455
+ "add_(?:development_)?dependency\\s+\\W+embulk\\W+\\s*(\\~\\>\\s*[\\d\\.]+)\\W+");
456
+ private static final Pattern EMBULK_DEPENDENCY_IN_GEMSPEC = Pattern.compile(
457
+ "add_(?:development_)?dependency\\s+\\W+embulk\\W+\\s*([\\d\\.]+)\\W+");
458
+ }