embulk 0.8.27-java → 0.8.28-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +11 -1
  3. data/appveyor.yml +4 -4
  4. data/bin/embulk +33 -9
  5. data/build.gradle +17 -4
  6. data/embulk-cli/build.gradle +1 -0
  7. data/embulk-cli/src/main/bat/{selfrun.bat → selfrun.bat.template} +1 -2
  8. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkArguments.java +54 -0
  9. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkBundle.java +44 -0
  10. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkCommandLine.java +256 -0
  11. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkCommandLineException.java +25 -0
  12. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkGlobalJRubyScriptingContainer.java +195 -0
  13. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkMigrate.java +9 -5
  14. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkRun.java +769 -0
  15. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkSelfUpdate.java +1 -14
  16. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkSubcommand.java +47 -0
  17. data/embulk-cli/src/main/java/org/embulk/cli/Main.java +12 -21
  18. data/embulk-cli/src/main/java/org/embulk/cli/parse/AbstractHelpLineDefinition.java +15 -0
  19. data/embulk-cli/src/main/java/org/embulk/cli/parse/CliHelpFormatterWithHelpMessages.java +141 -0
  20. data/embulk-cli/src/main/java/org/embulk/cli/parse/CliOptionsWithHelpMessages.java +45 -0
  21. data/embulk-cli/src/main/java/org/embulk/cli/parse/EmbulkCommandLineHelpRequired.java +10 -0
  22. data/embulk-cli/src/main/java/org/embulk/cli/parse/EmbulkCommandLineParseException.java +25 -0
  23. data/embulk-cli/src/main/java/org/embulk/cli/parse/EmbulkCommandLineParser.java +183 -0
  24. data/embulk-cli/src/main/java/org/embulk/cli/parse/HelpMessageAsCliOption.java +36 -0
  25. data/embulk-cli/src/main/java/org/embulk/cli/parse/HelpMessageLineDefinition.java +20 -0
  26. data/embulk-cli/src/main/java/org/embulk/cli/parse/OptionBehavior.java +39 -0
  27. data/embulk-cli/src/main/java/org/embulk/cli/parse/OptionDefinition.java +120 -0
  28. data/embulk-cli/src/main/sh/{selfrun.sh → selfrun.sh.template} +1 -1
  29. data/embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java +7 -3
  30. data/embulk-core/src/main/java/org/embulk/EmbulkRunner.java +526 -0
  31. data/embulk-core/src/main/java/org/embulk/EmbulkSetup.java +70 -0
  32. data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoader.java +3 -3
  33. data/embulk-docs/src/release.rst +1 -0
  34. data/embulk-docs/src/release/release-0.8.28.rst +14 -0
  35. data/lib/embulk.rb +21 -17
  36. data/lib/embulk/runner.rb +35 -166
  37. data/lib/embulk/version.rb +1 -1
  38. metadata +29 -11
  39. data/lib/embulk/command/embulk_bundle.rb +0 -47
  40. data/lib/embulk/command/embulk_main.rb +0 -2
  41. data/lib/embulk/command/embulk_run.rb +0 -418
@@ -0,0 +1,36 @@
1
+ package org.embulk.cli.parse;
2
+
3
+ import org.apache.commons.cli.Option;
4
+
5
+ final class HelpMessageAsCliOption
6
+ extends Option
7
+ {
8
+ public HelpMessageAsCliOption(final String message)
9
+ {
10
+ super("_", message);
11
+ }
12
+
13
+ @Override
14
+ public final String toString()
15
+ {
16
+ return this.getDescription();
17
+ }
18
+
19
+ @Override
20
+ public final boolean equals(Object other)
21
+ {
22
+ return this == other;
23
+ }
24
+
25
+ @Override
26
+ public final int hashCode()
27
+ {
28
+ return this.getDescription().hashCode();
29
+ }
30
+
31
+ @Override
32
+ public final Object clone()
33
+ {
34
+ return new HelpMessageAsCliOption(this.getDescription());
35
+ }
36
+ }
@@ -0,0 +1,20 @@
1
+ package org.embulk.cli.parse;
2
+
3
+ import org.apache.commons.cli.Option;
4
+
5
+ class HelpMessageLineDefinition
6
+ extends AbstractHelpLineDefinition
7
+ {
8
+ HelpMessageLineDefinition(final String message)
9
+ {
10
+ this.cliOption = new HelpMessageAsCliOption(message);
11
+ }
12
+
13
+ @Override
14
+ final Option getCliOption()
15
+ {
16
+ return this.cliOption;
17
+ }
18
+
19
+ private final Option cliOption;
20
+ }
@@ -0,0 +1,39 @@
1
+ package org.embulk.cli.parse;
2
+
3
+ import java.io.OutputStream;
4
+ import java.io.PrintWriter;
5
+ import org.embulk.cli.EmbulkCommandLine;
6
+
7
+ public abstract class OptionBehavior
8
+ {
9
+ public OptionBehavior()
10
+ {
11
+ this.helpWriter = SYSTEM_HELP_WRITER;
12
+ this.errorWriter = SYSTEM_ERROR_WRITER;
13
+ }
14
+
15
+ public OptionBehavior(final OutputStream helpStream, final OutputStream errorStream)
16
+ {
17
+ this.helpWriter = new PrintWriter(helpStream, true);
18
+ this.errorWriter = new PrintWriter(errorStream, true);
19
+ }
20
+
21
+ public abstract void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
22
+ throws EmbulkCommandLineParseException;
23
+
24
+ protected final PrintWriter helpWriter()
25
+ {
26
+ return this.helpWriter;
27
+ }
28
+
29
+ protected final PrintWriter errorWriter()
30
+ {
31
+ return this.errorWriter;
32
+ }
33
+
34
+ private static final PrintWriter SYSTEM_HELP_WRITER = new PrintWriter(System.out, true);
35
+ private static final PrintWriter SYSTEM_ERROR_WRITER = new PrintWriter(System.err, true);
36
+
37
+ private final PrintWriter helpWriter;
38
+ private final PrintWriter errorWriter;
39
+ }
@@ -0,0 +1,120 @@
1
+ package org.embulk.cli.parse;
2
+
3
+ import org.apache.commons.cli.Option;
4
+ import org.embulk.cli.EmbulkCommandLine;
5
+
6
+ public class OptionDefinition
7
+ extends AbstractHelpLineDefinition
8
+ {
9
+ private OptionDefinition(
10
+ final String shortOption,
11
+ final String longOption,
12
+ final boolean hasArgument,
13
+ final String argumentNameDisplayed,
14
+ final String description,
15
+ final boolean printsHelp,
16
+ final OptionBehavior behavior)
17
+ {
18
+ final Option.Builder builder;
19
+
20
+ if (shortOption == null) {
21
+ builder = Option.builder();
22
+ }
23
+ else {
24
+ builder = Option.builder(shortOption);
25
+ }
26
+
27
+ if (longOption != null) {
28
+ builder.longOpt(longOption);
29
+ }
30
+
31
+ if (hasArgument) {
32
+ builder.hasArg(true);
33
+ builder.argName(argumentNameDisplayed);
34
+ }
35
+ if (description != null) {
36
+ builder.desc(description);
37
+ }
38
+ this.cliOption = builder.build();
39
+ this.printsHelp = printsHelp;
40
+ this.behavior = behavior;
41
+ }
42
+
43
+ public static OptionDefinition defineOnlyLongOptionWithArgument(
44
+ final String longOption,
45
+ final String argumentNameDisplayed,
46
+ final String description,
47
+ final OptionBehavior behavior)
48
+ {
49
+ return new OptionDefinition(null, longOption, true, argumentNameDisplayed, description, false, behavior);
50
+ }
51
+
52
+ public static OptionDefinition defineOnlyShortOptionWithArgument(
53
+ final String shortOption,
54
+ final String argumentNameDisplayed,
55
+ final String description,
56
+ final OptionBehavior behavior)
57
+ {
58
+ return new OptionDefinition(shortOption, null, true, argumentNameDisplayed, description, false, behavior);
59
+ }
60
+
61
+ public static OptionDefinition defineOnlyShortOptionWithoutArgument(
62
+ final String shortOption,
63
+ final String description,
64
+ final OptionBehavior behavior)
65
+ {
66
+ return new OptionDefinition(shortOption, null, false, null, description, false, behavior);
67
+ }
68
+
69
+ public static OptionDefinition defineOptionWithoutArgument(
70
+ final String shortOption,
71
+ final String longOption,
72
+ final String description,
73
+ final OptionBehavior behavior)
74
+ {
75
+ return new OptionDefinition(shortOption, longOption, false, null, description, false, behavior);
76
+ }
77
+
78
+ public static OptionDefinition defineOptionWithArgument(
79
+ final String shortOption,
80
+ final String longOption,
81
+ final String argumentNameDisplayed,
82
+ final String description,
83
+ final OptionBehavior behavior)
84
+ {
85
+ return new OptionDefinition(shortOption, longOption, true, argumentNameDisplayed, description, false, behavior);
86
+ }
87
+
88
+ public static OptionDefinition defineHelpOption(
89
+ final String shortOption,
90
+ final String longOption,
91
+ final String description)
92
+ {
93
+ return new OptionDefinition(shortOption, longOption, false, null, description, true, new OptionBehavior() {
94
+ public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
95
+ {
96
+ }
97
+ });
98
+ }
99
+
100
+ @Override
101
+ final Option getCliOption()
102
+ {
103
+ return this.cliOption;
104
+ }
105
+
106
+ final void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
107
+ throws EmbulkCommandLineParseException
108
+ {
109
+ this.behavior.behave(commandLineBuilder, argument);
110
+ }
111
+
112
+ final boolean printsHelp()
113
+ {
114
+ return this.printsHelp;
115
+ }
116
+
117
+ private final Option cliOption;
118
+ private final boolean printsHelp;
119
+ private final OptionBehavior behavior;
120
+ }
@@ -64,7 +64,7 @@ done
64
64
  if test -z ${EMBULK_BUNDLE_PATH}; then
65
65
  unset EMBULK_BUNDLE_PATH
66
66
  unset BUNDLE_GEMFILE
67
- GEM_HOME="`cd && pwd`/.embulk/jruby/`java -cp $0 org.jruby.Main -e 'print RbConfig::CONFIG["ruby_version"]'`"
67
+ GEM_HOME="`cd && pwd`/.embulk/jruby/<%= ruby_version %>"
68
68
  export GEM_HOME
69
69
  GEM_PATH=""
70
70
  export GEM_PATH
@@ -40,7 +40,11 @@ public class SelfrunTest {
40
40
  .replaceAll("java ",
41
41
  "java -classpath "
42
42
  + classpath.getAbsolutePath().replaceAll("\\\\", "\\\\\\\\")
43
- + " org.embulk.cli.DummyMain ");
43
+ + " org.embulk.cli.DummyMain ")
44
+ // The following version string emulates RbConfig::CONFIG["ruby_version"].
45
+ // RbConfig::CONFIG["ruby_version"] is different from org.jruby.runtime.Constants.RUBY_VERSION.
46
+ // https://github.com/jruby/jruby/blob/9.1.5.0/core/src/main/java/org/jruby/ext/rbconfig/RbConfigLibrary.java#L229-L235
47
+ .replace("<%= ruby_version %>", org.jruby.runtime.Constants.RUBY_MAJOR_VERSION + ".0");
44
48
 
45
49
  // Modify selfrun so that arguments are written in 'args.txt' .
46
50
  Files.write(fileSystem.getPath(testSelfrunFile.getAbsolutePath()),
@@ -288,9 +292,9 @@ public class SelfrunTest {
288
292
  folder = new File(folder, "embulk-cli");
289
293
  }
290
294
  if (System.getProperty("file.separator").equals("\\")) {
291
- return new File(new File(new File(new File(folder, "src"), "main"), "bat"), "selfrun.bat");
295
+ return new File(new File(new File(new File(folder, "src"), "main"), "bat"), "selfrun.bat.template");
292
296
  } else {
293
- return new File(new File(new File(new File(folder, "src"), "main"), "sh"), "selfrun.sh");
297
+ return new File(new File(new File(new File(folder, "src"), "main"), "sh"), "selfrun.sh.template");
294
298
  }
295
299
  }
296
300
  }
@@ -0,0 +1,526 @@
1
+ package org.embulk;
2
+
3
+ import java.io.IOException;
4
+ import java.io.OutputStream;
5
+ import java.nio.charset.StandardCharsets;
6
+ import java.nio.file.Files;
7
+ import java.nio.file.Path;
8
+ import java.nio.file.Paths;
9
+ import java.nio.file.StandardOpenOption;
10
+ import java.util.Map;
11
+ import java.util.regex.Pattern;
12
+ import org.embulk.command.PreviewPrinter;
13
+ import org.embulk.command.TablePreviewPrinter;
14
+ import org.embulk.command.VerticalPreviewPrinter;
15
+ import org.embulk.config.ConfigDiff;
16
+ import org.embulk.config.ConfigException;
17
+ import org.embulk.config.ConfigSource;
18
+ import org.embulk.config.DataSource;
19
+ import org.embulk.config.ModelManager;
20
+ import org.embulk.exec.ExecutionResult;
21
+ import org.embulk.exec.PreviewResult;
22
+ import org.embulk.exec.ResumeState;
23
+ import org.embulk.exec.TransactionStage;
24
+ import org.jruby.RubyHash;
25
+ import org.jruby.embed.ScriptingContainer;
26
+ import org.slf4j.Logger;
27
+ import org.slf4j.LoggerFactory;
28
+
29
+ /**
30
+ * EmbulkRunner runs the guess, preview, or run subcommand.
31
+ *
32
+ * NOTE: Developers should not depend on this EmbulkRunner class. This class is created tentatively, and may be
33
+ * re-implemented again in a different style.
34
+ */
35
+ public class EmbulkRunner
36
+ {
37
+ // |EmbulkSetup.setup| initializes:
38
+ // new EmbulkRunner(embed, globalJRubyContainer)
39
+ public EmbulkRunner(final EmbulkEmbed embed, final ScriptingContainer globalJRubyContainer)
40
+ {
41
+ this.embed = embed; // org.embulk.EmbulkEmbed
42
+ this.globalJRubyContainer = globalJRubyContainer;
43
+ }
44
+
45
+ /**
46
+ * Runs the guess subcommand.
47
+ *
48
+ * It receives Java Paths to be called from Java EmbulkRun in embulk-cli.
49
+ */
50
+ public void guess(final Path configFilePath, final Path outputPath)
51
+ {
52
+ // TODO: Utilize |templateParams| and |templateIncludePath|.
53
+ // They have not been used from embulk-cli while |template_params| and |template_include_path| are implemented
54
+ // in Ruby Embulk::EmbulkRunner.
55
+ final ConfigSource configSource;
56
+ try {
57
+ configSource = readConfig(
58
+ configFilePath, new RubyHash(this.globalJRubyContainer.getProvider().getRuntime()), null);
59
+ }
60
+ catch (IOException ex) {
61
+ throw new RuntimeException(ex);
62
+ }
63
+
64
+ try {
65
+ guessInternal(configSource, outputPath);
66
+ }
67
+ catch (IOException ex) {
68
+ throw new RuntimeException(ex);
69
+ }
70
+ }
71
+
72
+ /**
73
+ * Runs the guess subcommand.
74
+ *
75
+ * It receives Strings as parameters to be called from Ruby (embulk/runner.rb).
76
+ */
77
+ public void guess(final String configFilePathString, final String outputPathString)
78
+ {
79
+ final Path outputPath = (outputPathString == null ? null : Paths.get(outputPathString));
80
+ guess(Paths.get(configFilePathString), outputPath);
81
+ }
82
+
83
+ /**
84
+ * Runs the guess subcommand.
85
+ *
86
+ * It receives a ConfigSource and a String as parameters to be called from Ruby (embulk/runner.rb).
87
+ */
88
+ public void guess(final ConfigSource configSource, final String outputPathString)
89
+ {
90
+ final Path outputPath = (outputPathString == null ? null : Paths.get(outputPathString));
91
+ try {
92
+ guessInternal(configSource, outputPath);
93
+ }
94
+ catch (IOException ex) {
95
+ throw new RuntimeException(ex);
96
+ }
97
+ }
98
+
99
+ /**
100
+ * Runs the guess subcommand.
101
+ *
102
+ * It receives a Java Path and a String to be called from Java's EmbulkRun in embulk-cli.
103
+ */
104
+ public void preview(final Path configFilePath, final String format)
105
+ {
106
+ // TODO: Utilize |templateParams| and |templateIncludePath|.
107
+ // They have not been used from embulk-cli while |template_params| and |template_include_path| are implemented
108
+ // in Ruby Embulk::EmbulkRunner.
109
+ final ConfigSource configSource;
110
+ try {
111
+ configSource = readConfig(
112
+ configFilePath, new RubyHash(this.globalJRubyContainer.getProvider().getRuntime()), null);
113
+ }
114
+ catch (IOException ex) {
115
+ throw new RuntimeException(ex);
116
+ }
117
+
118
+ try {
119
+ previewInternal(configSource, format);
120
+ }
121
+ catch (IOException ex) {
122
+ throw new RuntimeException(ex);
123
+ }
124
+ }
125
+
126
+ /**
127
+ * Runs the preview subcommand.
128
+ *
129
+ * It receives Strings as parameters to be called from Ruby (embulk/runner.rb).
130
+ */
131
+ public void preview(final String configFilePathString, final String format)
132
+ {
133
+ preview(Paths.get(configFilePathString), format);
134
+ }
135
+
136
+ /**
137
+ * Runs the preview subcommand.
138
+ *
139
+ * It receives a ConfigSource and a String as parameters to be called from Ruby (embulk/runner.rb).
140
+ */
141
+ public void preview(final ConfigSource configSource, final String format)
142
+ {
143
+ try {
144
+ previewInternal(configSource, format);
145
+ }
146
+ catch (IOException ex) {
147
+ throw new RuntimeException(ex);
148
+ }
149
+ }
150
+
151
+ /**
152
+ * Runs the run subcommand.
153
+ *
154
+ * It receives Java Paths to be called from Java EmbulkRun in embulk-cli.
155
+ */
156
+ public void run(
157
+ final Path configFilePath,
158
+ final Path configDiffPath,
159
+ final Path outputPath,
160
+ final Path resumeStatePath)
161
+ {
162
+ // TODO: Utilize |templateParams| and |templateIncludePath|.
163
+ // They have not been used from embulk-cli while |template_params| and |template_include_path| are implemented
164
+ // in Ruby Embulk::EmbulkRunner.
165
+ final ConfigSource configSource;
166
+ try {
167
+ configSource = readConfig(
168
+ configFilePath, new RubyHash(this.globalJRubyContainer.getProvider().getRuntime()), null);
169
+ }
170
+ catch (IOException ex) {
171
+ throw new RuntimeException(ex);
172
+ }
173
+
174
+ try {
175
+ runInternal(configSource, configDiffPath, outputPath, resumeStatePath);
176
+ }
177
+ catch (IOException ex) {
178
+ throw new RuntimeException(ex);
179
+ }
180
+ }
181
+
182
+ /**
183
+ * Runs the run subcommand.
184
+ *
185
+ * It receives Strings as parameters to be called from Ruby (embulk/runner.rb).
186
+ */
187
+ public void run(final String configFilePathString,
188
+ final String configDiffPathString,
189
+ final String outputPathString,
190
+ final String resumeStatePathString)
191
+ {
192
+ final Path configDiffPath = (configDiffPathString == null ? null : Paths.get(configDiffPathString));
193
+ final Path outputPath = (outputPathString == null ? null : Paths.get(outputPathString));
194
+ final Path resumeStatePath = (resumeStatePathString == null ? null : Paths.get(resumeStatePathString));
195
+ run(Paths.get(configFilePathString), configDiffPath, outputPath, resumeStatePath);
196
+ }
197
+
198
+ /**
199
+ * Runs the run subcommand.
200
+ *
201
+ * It receives a ConfigSource and a String as parameters to be called from Ruby (embulk/runner.rb).
202
+ */
203
+ public void run(final ConfigSource configSource,
204
+ final String configDiffPathString,
205
+ final String outputPathString,
206
+ final String resumeStatePathString)
207
+ {
208
+ final Path configDiffPath = (configDiffPathString == null ? null : Paths.get(configDiffPathString));
209
+ final Path outputPath = (outputPathString == null ? null : Paths.get(outputPathString));
210
+ final Path resumeStatePath = (resumeStatePathString == null ? null : Paths.get(resumeStatePathString));
211
+ try {
212
+ runInternal(configSource, configDiffPath, outputPath, resumeStatePath);
213
+ }
214
+ catch (IOException ex) {
215
+ throw new RuntimeException(ex);
216
+ }
217
+ }
218
+
219
+ private void guessInternal(final ConfigSource configSource, final Path outputPath)
220
+ throws IOException
221
+ {
222
+ try {
223
+ checkFileWritable(outputPath);
224
+ }
225
+ catch (IOException ex) {
226
+ throw new RuntimeException("Not writable: " + outputPath.toString());
227
+ }
228
+
229
+ final ConfigDiff configDiff = this.embed.guess(configSource);
230
+ final ConfigSource guessedConfigSource = configSource.merge(configDiff);
231
+ final String yaml = writeConfig(outputPath, guessedConfigSource);
232
+ System.err.println(yaml);
233
+ if (outputPath != null) {
234
+ System.out.println("Created '" + outputPath + "' file.");
235
+ }
236
+ else {
237
+ System.out.println("Use -o PATH option to write the guessed config file to a file.");
238
+ }
239
+ }
240
+
241
+ private void previewInternal(final ConfigSource configSource, final String format)
242
+ throws IOException
243
+ {
244
+ final PreviewResult previewResult = this.embed.preview(configSource);
245
+ final ModelManager modelManager = this.embed.getModelManager();
246
+
247
+ final PreviewPrinter printer;
248
+ switch (format != null ? format : "table") {
249
+ case "table":
250
+ printer = new TablePreviewPrinter(System.out, modelManager, previewResult.getSchema());
251
+ break;
252
+ case "vertical":
253
+ printer = new VerticalPreviewPrinter(System.out, modelManager, previewResult.getSchema());
254
+ break;
255
+ default:
256
+ throw new IllegalArgumentException(
257
+ "Unknown preview output format '" + format + "'. Supported formats: table, vertical");
258
+ }
259
+
260
+ printer.printAllPages(previewResult.getPages());
261
+ printer.finish();
262
+ }
263
+
264
+ private void runInternal(
265
+ final ConfigSource originalConfigSource,
266
+ final Path configDiffPath,
267
+ final Path outputPath, // deprecated
268
+ final Path resumeStatePath)
269
+ throws IOException
270
+ {
271
+ try {
272
+ checkFileWritable(outputPath);
273
+ }
274
+ catch (IOException ex) {
275
+ throw new RuntimeException("Not writable: " + outputPath.toString());
276
+ }
277
+ try {
278
+ checkFileWritable(configDiffPath);
279
+ }
280
+ catch (IOException ex) {
281
+ throw new RuntimeException("Not writable: " + configDiffPath.toString());
282
+ }
283
+ try {
284
+ checkFileWritable(resumeStatePath);
285
+ }
286
+ catch (IOException ex) {
287
+ throw new RuntimeException("Not writable: " + resumeStatePath.toString());
288
+ }
289
+
290
+ final ConfigSource configSource;
291
+ if (configDiffPath != null && Files.size(configDiffPath) > 0L) {
292
+ final ConfigSource lastConfigDiff =
293
+ readConfig(configDiffPath, new RubyHash(this.globalJRubyContainer.getProvider().getRuntime()), null);
294
+ configSource = originalConfigSource.merge(lastConfigDiff);
295
+ }
296
+ else {
297
+ configSource = originalConfigSource;
298
+ }
299
+
300
+ final ConfigSource resumeConfig;
301
+ if (resumeStatePath != null) {
302
+ ConfigSource resumeConfigTemp = null;
303
+ try {
304
+ resumeConfigTemp = readYamlConfigFile(resumeStatePath);
305
+ }
306
+ catch (Throwable ex) {
307
+ // TODO log?
308
+ resumeConfigTemp = null;
309
+ }
310
+ if (resumeConfigTemp == null || resumeConfigTemp.isEmpty()) {
311
+ resumeConfig = null;
312
+ }
313
+ else {
314
+ resumeConfig = resumeConfigTemp;
315
+ }
316
+ }
317
+ else {
318
+ resumeConfig = null;
319
+ }
320
+
321
+ final EmbulkEmbed.ResumableResult resumableResult;
322
+ final ExecutionResult executionResultTemp;
323
+ if (resumeConfig != null) {
324
+ resumableResult = this.embed.resumeState(configSource, resumeConfig).resume();
325
+ executionResultTemp = null;
326
+ }
327
+ else if (resumeStatePath != null) {
328
+ resumableResult = this.embed.runResumable(configSource);
329
+ executionResultTemp = null;
330
+ }
331
+ else {
332
+ resumableResult = null;
333
+ executionResultTemp = this.embed.run(configSource);
334
+ }
335
+
336
+ final ExecutionResult executionResult;
337
+ if (executionResultTemp == null) {
338
+ if (!resumableResult.isSuccessful()) {
339
+ if (resumableResult.getTransactionStage().isBefore(TransactionStage.RUN)) {
340
+ // retry without resume state file if no tasks started yet
341
+ // delete resume file
342
+ if (resumeStatePath != null) {
343
+ try {
344
+ Files.deleteIfExists(resumeStatePath);
345
+ }
346
+ catch (Throwable ex) {
347
+ System.err.println("Failed to delete: " + resumeStatePath.toString());
348
+ }
349
+ }
350
+ }
351
+ else {
352
+ rootLogger.info("Writing resume state to '" + resumeStatePath.toString() + "'");
353
+ try {
354
+ writeResumeState(resumeStatePath, resumableResult.getResumeState());
355
+ }
356
+ catch (IOException ex) {
357
+ throw new RuntimeException(ex);
358
+ }
359
+ rootLogger.info("Resume state is written. Run the transaction again with -r option to resume or use \"cleanup\" subcommand to delete intermediate data.");
360
+ }
361
+ throw new RuntimeException(resumableResult.getCause());
362
+ }
363
+ executionResult = resumableResult.getSuccessfulResult();
364
+ }
365
+ else {
366
+ executionResult = executionResultTemp;
367
+ }
368
+
369
+ // delete resume file
370
+ if (resumeStatePath != null) {
371
+ try {
372
+ Files.deleteIfExists(resumeStatePath);
373
+ }
374
+ catch (Throwable ex) {
375
+ System.err.println("Failed to delete: " + resumeStatePath.toString());
376
+ }
377
+ }
378
+
379
+ final ConfigDiff configDiff = executionResult.getConfigDiff();
380
+ rootLogger.info("Committed.");
381
+ rootLogger.info("Next config diff: " + configDiff.toString());
382
+
383
+ writeConfig(configDiffPath, configDiff);
384
+ writeConfig(outputPath, configSource.merge(configDiff)); // deprecated
385
+ }
386
+
387
+ // def resume_state(config, options={})
388
+ // configSource = read_config(config, options)
389
+ // Resumed.new(self, DataSource.from_java(configSource), options)
390
+ // end
391
+
392
+ private ConfigSource readConfig(
393
+ final Path configFilePath,
394
+ final RubyHash templateParams,
395
+ final String templateIncludePath)
396
+ throws IOException
397
+ {
398
+ final String configString = configFilePath.toString();
399
+ if (EXT_YAML_LIQUID.matcher(configFilePath.toString()).matches()) {
400
+ return this.embed.newConfigLoader().fromYamlString(
401
+ runLiquid(new String(Files.readAllBytes(configFilePath), StandardCharsets.UTF_8),
402
+ templateParams,
403
+ templateIncludePath));
404
+ }
405
+ else if (EXT_YAML.matcher(configFilePath.toString()).matches()) {
406
+ return this.embed.newConfigLoader().fromYamlString(
407
+ new String(Files.readAllBytes(configFilePath), StandardCharsets.UTF_8));
408
+ }
409
+ else {
410
+ throw new ConfigException(
411
+ "Unsupported file extension. Supported file extensions are .yml and .yml.liquid: " +
412
+ configFilePath.toString());
413
+ }
414
+ }
415
+
416
+ private ConfigSource readYamlConfigFile(final Path path)
417
+ throws IOException
418
+ {
419
+ return this.embed.newConfigLoader().fromYamlString(
420
+ new String(Files.readAllBytes(path), StandardCharsets.UTF_8));
421
+ }
422
+
423
+ private String runLiquid(
424
+ final String templateSource,
425
+ final RubyHash templateParams,
426
+ final String templateIncludePath)
427
+ {
428
+ this.globalJRubyContainer.runScriptlet("require 'liquid'");
429
+
430
+ this.globalJRubyContainer.put("__internal_liquid_template_source__", templateSource);
431
+ this.globalJRubyContainer.runScriptlet("template = Liquid::Template.parse(__internal_liquid_template_source__, :error_mode => :strict)");
432
+ this.globalJRubyContainer.remove("__internal_liquid_template_source__");
433
+
434
+ if (templateIncludePath != null) {
435
+ this.globalJRubyContainer.put("__internal_liquid_template_include_path_java__", templateIncludePath);
436
+ this.globalJRubyContainer.runScriptlet("__internal_liquid_template_include_path__ = File.expand_path(__internal_liquid_template_include_path_java__ || File.dirname(config)) unless __internal_liquid_template_include_path_java__ == false");
437
+ this.globalJRubyContainer.runScriptlet("template.registers[:file_system] = Liquid::LocalFileSystem.new(__internal_liquid_template_include_path__, \"_%s.yml.liquid\")");
438
+ this.globalJRubyContainer.remove("__internal_liquid_template_include_path__");
439
+ }
440
+
441
+ this.globalJRubyContainer.put("__internal_liquid_template_params__", templateParams);
442
+ this.globalJRubyContainer.runScriptlet("__internal_liquid_template_data__ = { 'env' => ENV.to_h }.merge(__internal_liquid_template_params__)");
443
+ this.globalJRubyContainer.remove("__internal_liquid_template_params__");
444
+
445
+ final Object renderedObject =
446
+ this.globalJRubyContainer.runScriptlet("template.render(__internal_liquid_template_data__)");
447
+ return renderedObject.toString();
448
+ }
449
+
450
+ private boolean checkFileWritable(final Path path)
451
+ throws IOException
452
+ {
453
+ if (path != null) {
454
+ // Open file with append mode and do nothing.
455
+ // If file is not writable, it throws an exception.
456
+ // NOTE: |Files.isWritable| does not work for the purpose as it expects the file exists.
457
+ // Using |Files.newOutputStream| for the binary mode.
458
+ try (final OutputStream output =
459
+ Files.newOutputStream(path, StandardOpenOption.APPEND, StandardOpenOption.CREATE)) {
460
+ ;
461
+ }
462
+ }
463
+ return true;
464
+ }
465
+
466
+ private String writeConfig(final Path path, final DataSource modelObject)
467
+ throws IOException
468
+ {
469
+ final String yamlString = dumpDataSourceInYaml(modelObject);
470
+ if (path != null) {
471
+ Files.write(path, yamlString.getBytes(), StandardOpenOption.CREATE, StandardOpenOption.WRITE);
472
+ }
473
+ return yamlString;
474
+ }
475
+
476
+ private String writeResumeState(final Path path, final ResumeState modelObject)
477
+ throws IOException
478
+ {
479
+ final String yamlString = dumpResumeStateInYaml(modelObject);
480
+ if (path != null) {
481
+ Files.write(path, yamlString.getBytes(), StandardOpenOption.CREATE, StandardOpenOption.WRITE);
482
+ }
483
+ return yamlString;
484
+ }
485
+
486
+ private String dumpDataSourceInYaml(final DataSource modelObject)
487
+ {
488
+ final ModelManager modelManager = this.embed.getModelManager();
489
+ final Object object = modelManager.readObject(Object.class, modelManager.writeObject(modelObject));
490
+ return (new org.yaml.snakeyaml.Yaml()).dump(object);
491
+ }
492
+
493
+ private String dumpResumeStateInYaml(final ResumeState modelObject)
494
+ {
495
+ final ModelManager modelManager = this.embed.getModelManager();
496
+ final Object object = modelManager.readObject(Object.class, modelManager.writeObject(modelObject));
497
+ return (new org.yaml.snakeyaml.Yaml()).dump(object);
498
+ }
499
+
500
+ // class Runnable
501
+ // def initialize(runner, config, options)
502
+ // @runner = runner
503
+ // @config = config
504
+ // @options = options
505
+ // end
506
+ //
507
+ // attr_reader :config
508
+ //
509
+ // def preview(options={})
510
+ // @runner.preview(@config, @options.merge(options))
511
+ // end
512
+ //
513
+ // def run(options={})
514
+ // @runner.run(@config, @options.merge(options))
515
+ // end
516
+ // end
517
+
518
+ // NOTE: The root logger directly from |LoggerFactory|, not from |Exec.getLogger| as it's outside of |Exec.doWith|.
519
+ private static final Logger rootLogger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
520
+
521
+ private final Pattern EXT_YAML = Pattern.compile(".*\\.ya?ml$");
522
+ private final Pattern EXT_YAML_LIQUID = Pattern.compile(".*\\.ya?ml\\.liquid$");
523
+
524
+ private final EmbulkEmbed embed;
525
+ private final ScriptingContainer globalJRubyContainer;
526
+ }