embulk 0.8.27-java → 0.8.28-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +11 -1
- data/appveyor.yml +4 -4
- data/bin/embulk +33 -9
- data/build.gradle +17 -4
- data/embulk-cli/build.gradle +1 -0
- data/embulk-cli/src/main/bat/{selfrun.bat → selfrun.bat.template} +1 -2
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkArguments.java +54 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkBundle.java +44 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkCommandLine.java +256 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkCommandLineException.java +25 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkGlobalJRubyScriptingContainer.java +195 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkMigrate.java +9 -5
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkRun.java +769 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkSelfUpdate.java +1 -14
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkSubcommand.java +47 -0
- data/embulk-cli/src/main/java/org/embulk/cli/Main.java +12 -21
- data/embulk-cli/src/main/java/org/embulk/cli/parse/AbstractHelpLineDefinition.java +15 -0
- data/embulk-cli/src/main/java/org/embulk/cli/parse/CliHelpFormatterWithHelpMessages.java +141 -0
- data/embulk-cli/src/main/java/org/embulk/cli/parse/CliOptionsWithHelpMessages.java +45 -0
- data/embulk-cli/src/main/java/org/embulk/cli/parse/EmbulkCommandLineHelpRequired.java +10 -0
- data/embulk-cli/src/main/java/org/embulk/cli/parse/EmbulkCommandLineParseException.java +25 -0
- data/embulk-cli/src/main/java/org/embulk/cli/parse/EmbulkCommandLineParser.java +183 -0
- data/embulk-cli/src/main/java/org/embulk/cli/parse/HelpMessageAsCliOption.java +36 -0
- data/embulk-cli/src/main/java/org/embulk/cli/parse/HelpMessageLineDefinition.java +20 -0
- data/embulk-cli/src/main/java/org/embulk/cli/parse/OptionBehavior.java +39 -0
- data/embulk-cli/src/main/java/org/embulk/cli/parse/OptionDefinition.java +120 -0
- data/embulk-cli/src/main/sh/{selfrun.sh → selfrun.sh.template} +1 -1
- data/embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java +7 -3
- data/embulk-core/src/main/java/org/embulk/EmbulkRunner.java +526 -0
- data/embulk-core/src/main/java/org/embulk/EmbulkSetup.java +70 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoader.java +3 -3
- data/embulk-docs/src/release.rst +1 -0
- data/embulk-docs/src/release/release-0.8.28.rst +14 -0
- data/lib/embulk.rb +21 -17
- data/lib/embulk/runner.rb +35 -166
- data/lib/embulk/version.rb +1 -1
- metadata +29 -11
- data/lib/embulk/command/embulk_bundle.rb +0 -47
- data/lib/embulk/command/embulk_main.rb +0 -2
- data/lib/embulk/command/embulk_run.rb +0 -418
@@ -0,0 +1,25 @@
|
|
1
|
+
package org.embulk.cli;
|
2
|
+
|
3
|
+
public final class EmbulkCommandLineException
|
4
|
+
extends RuntimeException
|
5
|
+
{
|
6
|
+
protected EmbulkCommandLineException()
|
7
|
+
{
|
8
|
+
super();
|
9
|
+
}
|
10
|
+
|
11
|
+
public EmbulkCommandLineException(final String message)
|
12
|
+
{
|
13
|
+
super(message);
|
14
|
+
}
|
15
|
+
|
16
|
+
public EmbulkCommandLineException(final Throwable cause)
|
17
|
+
{
|
18
|
+
super(cause);
|
19
|
+
}
|
20
|
+
|
21
|
+
public EmbulkCommandLineException(final String message, final Throwable cause)
|
22
|
+
{
|
23
|
+
super(message, cause);
|
24
|
+
}
|
25
|
+
}
|
@@ -0,0 +1,195 @@
|
|
1
|
+
package org.embulk.cli;
|
2
|
+
|
3
|
+
import java.io.PrintStream;
|
4
|
+
import java.net.URISyntaxException;
|
5
|
+
import java.net.URL;
|
6
|
+
import java.nio.file.Files;
|
7
|
+
import java.nio.file.Path;
|
8
|
+
import java.nio.file.Paths;
|
9
|
+
import java.security.CodeSource;
|
10
|
+
import java.security.ProtectionDomain;
|
11
|
+
import java.util.Collections;
|
12
|
+
import java.util.List;
|
13
|
+
import org.jruby.RubyInstanceConfig;
|
14
|
+
import org.jruby.embed.LocalContextScope;
|
15
|
+
import org.jruby.embed.LocalVariableBehavior;
|
16
|
+
import org.jruby.embed.PathType;
|
17
|
+
import org.jruby.embed.ScriptingContainer;
|
18
|
+
import org.jruby.util.cli.Options;
|
19
|
+
|
20
|
+
/**
|
21
|
+
* EmbulkGlobalJRubyScriptingContainer creates a ScriptingContainer instance for global use in Embulk.
|
22
|
+
*
|
23
|
+
* The creator method is static because the target instance is singleton by definition.
|
24
|
+
*/
|
25
|
+
public class EmbulkGlobalJRubyScriptingContainer
|
26
|
+
{
|
27
|
+
private EmbulkGlobalJRubyScriptingContainer()
|
28
|
+
{
|
29
|
+
// Do not instantiate.
|
30
|
+
}
|
31
|
+
|
32
|
+
/**
|
33
|
+
* Sets up a ScriptingContainer instance for global use in Embulk.
|
34
|
+
*/
|
35
|
+
public static ScriptingContainer setup(
|
36
|
+
final String[] embulkArgs,
|
37
|
+
final List<String> jrubyOptions,
|
38
|
+
final String bundlePath,
|
39
|
+
final PrintStream warning)
|
40
|
+
{
|
41
|
+
// The JRuby instance is a global singleton so that the settings here affects later execution.
|
42
|
+
// The local variable should be persistent so that local variables are set through ScriptingContainer.put.
|
43
|
+
final ScriptingContainer jrubyGlobalContainer =
|
44
|
+
new ScriptingContainer(LocalContextScope.SINGLETON, LocalVariableBehavior.PERSISTENT);
|
45
|
+
final RubyInstanceConfig jrubyGlobalConfig = jrubyGlobalContainer.getProvider().getRubyInstanceConfig();
|
46
|
+
|
47
|
+
for (final String jrubyOption : jrubyOptions) {
|
48
|
+
try {
|
49
|
+
processJRubyOption(jrubyOption, jrubyGlobalConfig);
|
50
|
+
}
|
51
|
+
catch (UnrecognizedJRubyOptionException ex) {
|
52
|
+
warning.println("[WARN] The \"-R\" option(s) are not recognized in Embulk: -R" + jrubyOption);
|
53
|
+
warning.println("[WARN] Please add your requests at: https://github.com/embulk/embulk/issues/707");
|
54
|
+
warning.println("");
|
55
|
+
}
|
56
|
+
catch (NotWorkingJRubyOptionException ex) {
|
57
|
+
warning.println("[WARN] The \"-R\" option(s) do not work in Embulk: -R" + jrubyOption);
|
58
|
+
warning.println("");
|
59
|
+
}
|
60
|
+
}
|
61
|
+
|
62
|
+
if (bundlePath != null) {
|
63
|
+
/* Environment variables are set in the selfrun script or bin/embulk:
|
64
|
+
* ENV['EMBULK_BUNDLE_PATH']: set through '-b' | '--bundle', or inherit from the runtime environment
|
65
|
+
* ENV['BUNDLE_GEMFILE']: set for "ENV['EMBULK_BUNDLE_PATH']/Gemfile"
|
66
|
+
* ENV['GEM_HOME']: unset
|
67
|
+
* ENV['GEM_PATH']: unset
|
68
|
+
*/
|
69
|
+
|
70
|
+
// bundler is included in embulk-core.jar
|
71
|
+
jrubyGlobalContainer.runScriptlet("Gem.clear_paths");
|
72
|
+
jrubyGlobalContainer.runScriptlet("require 'bundler'");
|
73
|
+
|
74
|
+
jrubyGlobalContainer.runScriptlet("Bundler.load.setup_environment");
|
75
|
+
jrubyGlobalContainer.runScriptlet("require 'bundler/setup'");
|
76
|
+
// since here, `require` may load files of different (newer) embulk versions
|
77
|
+
// especially following 'embulk/command/embulk_main'.
|
78
|
+
|
79
|
+
// NOTE: It is intentionally not done by building a Ruby statement string from |bundlePath|.
|
80
|
+
// It can cause insecure injections.
|
81
|
+
//
|
82
|
+
// add bundle directory path to load local plugins at ./embulk
|
83
|
+
jrubyGlobalContainer.put("__internal_bundle_path__", bundlePath);
|
84
|
+
jrubyGlobalContainer.runScriptlet("$LOAD_PATH << File.expand_path(__internal_bundle_path__)");
|
85
|
+
jrubyGlobalContainer.remove("__internal_bundle_path__");
|
86
|
+
|
87
|
+
return jrubyGlobalContainer;
|
88
|
+
}
|
89
|
+
else {
|
90
|
+
/* Environment variables are set in the selfrun script or bin/embulk:
|
91
|
+
* ENV['EMBULK_BUNDLE_PATH']: unset
|
92
|
+
* ENV['BUNDLE_GEMFILE']: unset
|
93
|
+
* ENV['GEM_HOME']: set for "~/.embulk/jruby/${ruby-version}"
|
94
|
+
* ENV['GEM_PATH']: set for ""
|
95
|
+
*/
|
96
|
+
|
97
|
+
jrubyGlobalContainer.runScriptlet("Gem.clear_paths"); // force rubygems to reload GEM_HOME
|
98
|
+
|
99
|
+
// NOTE: The path from |getEmbulkJRubyLoadPath()| is added in $LOAD_PATH just in case.
|
100
|
+
// Though it is not mandatory just to run "embulk_main.rb", it may be required in later steps.
|
101
|
+
//
|
102
|
+
// NOTE: It is intentionally not done by building a Ruby statement string from |getEmbulkJRubyLoadPath()|.
|
103
|
+
// It can cause insecure injections.
|
104
|
+
//
|
105
|
+
// NOTE: It was written in Ruby as follows:
|
106
|
+
// $LOAD_PATH << File.expand_path('../../', File.dirname(__FILE__))
|
107
|
+
jrubyGlobalContainer.put("__internal_load_path__", getEmbulkJRubyLoadPath());
|
108
|
+
jrubyGlobalContainer.runScriptlet("$LOAD_PATH << File.expand_path(__internal_load_path__)");
|
109
|
+
jrubyGlobalContainer.remove("__internal_load_path__");
|
110
|
+
|
111
|
+
return jrubyGlobalContainer;
|
112
|
+
}
|
113
|
+
}
|
114
|
+
|
115
|
+
private static final class UnrecognizedJRubyOptionException extends Exception {}
|
116
|
+
private static final class NotWorkingJRubyOptionException extends Exception {}
|
117
|
+
|
118
|
+
private static void processJRubyOption(final String jrubyOption, final RubyInstanceConfig jrubyGlobalConfig)
|
119
|
+
throws UnrecognizedJRubyOptionException, NotWorkingJRubyOptionException
|
120
|
+
{
|
121
|
+
if (jrubyOption.charAt(0) != '-') {
|
122
|
+
throw new UnrecognizedJRubyOptionException();
|
123
|
+
}
|
124
|
+
|
125
|
+
for (int index = 1; index < jrubyOption.length(); ++index) {
|
126
|
+
switch (jrubyOption.charAt(index)) {
|
127
|
+
case '-':
|
128
|
+
if (jrubyOption.equals("--dev")) {
|
129
|
+
// They are not all of "--dev", but they are most possible configurations after JVM boot.
|
130
|
+
Options.COMPILE_INVOKEDYNAMIC.force("false"); // NOTE: Options is global.
|
131
|
+
jrubyGlobalConfig.setCompileMode(RubyInstanceConfig.CompileMode.OFF);
|
132
|
+
return;
|
133
|
+
}
|
134
|
+
else if (jrubyOption.equals("--client")) {
|
135
|
+
throw new NotWorkingJRubyOptionException();
|
136
|
+
}
|
137
|
+
else if (jrubyOption.equals("--server")) {
|
138
|
+
throw new NotWorkingJRubyOptionException();
|
139
|
+
}
|
140
|
+
throw new UnrecognizedJRubyOptionException();
|
141
|
+
default:
|
142
|
+
throw new UnrecognizedJRubyOptionException();
|
143
|
+
}
|
144
|
+
}
|
145
|
+
}
|
146
|
+
|
147
|
+
/**
|
148
|
+
* Returns a path to be added in JRuby's $LOAD_PATH.
|
149
|
+
*
|
150
|
+
* In case Embulk runs from the Embulk JAR file (normal case):
|
151
|
+
* "file:/some/directory/embulk.jar!"
|
152
|
+
*
|
153
|
+
* In case Embulk runs out of a JAR file (irregular case):
|
154
|
+
* "/some/directory"
|
155
|
+
*/
|
156
|
+
private static String getEmbulkJRubyLoadPath()
|
157
|
+
{
|
158
|
+
final ProtectionDomain protectionDomain;
|
159
|
+
try {
|
160
|
+
protectionDomain = EmbulkGlobalJRubyScriptingContainer.class.getProtectionDomain();
|
161
|
+
}
|
162
|
+
catch (SecurityException ex) {
|
163
|
+
throw new EmbulkCommandLineException("Failed to achieve ProtectionDomain", ex);
|
164
|
+
}
|
165
|
+
|
166
|
+
final CodeSource codeSource = protectionDomain.getCodeSource();
|
167
|
+
if (codeSource == null) {
|
168
|
+
throw new EmbulkCommandLineException("Failed to achieve CodeSource");
|
169
|
+
}
|
170
|
+
|
171
|
+
final URL locationUrl = codeSource.getLocation();
|
172
|
+
if (locationUrl == null) {
|
173
|
+
throw new EmbulkCommandLineException("Failed to achieve location");
|
174
|
+
}
|
175
|
+
else if (!locationUrl.getProtocol().equals("file")) {
|
176
|
+
throw new EmbulkCommandLineException("Invalid location: " + locationUrl.toString());
|
177
|
+
}
|
178
|
+
|
179
|
+
final Path locationPath;
|
180
|
+
try {
|
181
|
+
locationPath = Paths.get(locationUrl.toURI().getPath());
|
182
|
+
}
|
183
|
+
catch (URISyntaxException ex) {
|
184
|
+
throw new EmbulkCommandLineException("Invalid location: " + locationUrl.toString(), ex);
|
185
|
+
}
|
186
|
+
|
187
|
+
if (Files.isDirectory(locationPath)) { // Out of a JAR file
|
188
|
+
System.err.println("Warning: Embulk looks running out of the Embulk jar file. It is unsupported.");
|
189
|
+
return locationPath.toString();
|
190
|
+
}
|
191
|
+
|
192
|
+
// TODO: Consider checking the file is really a JAR file.
|
193
|
+
return locationUrl.toString() + "!"; // Inside the Embulk JAR file
|
194
|
+
}
|
195
|
+
}
|
@@ -292,8 +292,10 @@ public class EmbulkMigrate
|
|
292
292
|
int position = 0;
|
293
293
|
String modifiedData = originalData;
|
294
294
|
while (position < modifiedData.length()) {
|
295
|
-
final
|
296
|
-
|
295
|
+
final String formerModifiedData = modifiedData.substring(0, position);
|
296
|
+
final String latterModifiedData = modifiedData.substring(position);
|
297
|
+
final Matcher matcher = pattern.matcher(latterModifiedData);
|
298
|
+
if (!matcher.find()) {
|
297
299
|
break;
|
298
300
|
}
|
299
301
|
if (first == null) {
|
@@ -301,10 +303,12 @@ public class EmbulkMigrate
|
|
301
303
|
}
|
302
304
|
final String replacingString = immediate;
|
303
305
|
modifiedData =
|
304
|
-
|
306
|
+
formerModifiedData +
|
307
|
+
latterModifiedData.substring(0, matcher.start(index)) +
|
305
308
|
replacingString +
|
306
|
-
|
309
|
+
latterModifiedData.substring(matcher.end(index));
|
307
310
|
position =
|
311
|
+
formerModifiedData.length() +
|
308
312
|
matcher.start(index) +
|
309
313
|
replacingString.length() +
|
310
314
|
(matcher.end() - matcher.end(index));
|
@@ -451,7 +455,7 @@ public class EmbulkMigrate
|
|
451
455
|
private static final Pattern GEM_TASK_IN_GRADLE = Pattern.compile(
|
452
456
|
"^([ \\t]*)task\\s+gem\\W.*\\{", Pattern.MULTILINE);
|
453
457
|
private static final Pattern EMBULK_CORE_OR_STANDARDS_IN_GRADLE = Pattern.compile(
|
454
|
-
"org\\.embulk:embulk-(?:core|standards):([\\d\\.\\+]+)?");
|
458
|
+
"org\\.embulk:embulk-(?:core|standards|test):([\\d\\.\\+]+)?");
|
455
459
|
private static final Pattern DEVELOPMENT_DEPENDENCY_IN_GEMSPEC = Pattern.compile(
|
456
460
|
"([ \\t]*\\w+)\\.add_development_dependency");
|
457
461
|
private static final Pattern EMBULK_DEPENDENCY_PRERELEASE_IN_GEMSPEC = Pattern.compile(
|
@@ -0,0 +1,769 @@
|
|
1
|
+
package org.embulk.cli;
|
2
|
+
|
3
|
+
import java.io.IOException;
|
4
|
+
import java.io.PrintStream;
|
5
|
+
import java.io.PrintWriter;
|
6
|
+
import java.net.URISyntaxException;
|
7
|
+
import java.nio.file.Files;
|
8
|
+
import java.nio.file.FileVisitResult;
|
9
|
+
import java.nio.file.Path;
|
10
|
+
import java.nio.file.Paths;
|
11
|
+
import java.nio.file.SimpleFileVisitor;
|
12
|
+
import java.nio.file.attribute.BasicFileAttributes;
|
13
|
+
import java.util.ArrayList;
|
14
|
+
import java.util.Arrays;
|
15
|
+
import java.util.List;
|
16
|
+
import org.embulk.EmbulkRunner;
|
17
|
+
import org.embulk.EmbulkSetup;
|
18
|
+
import org.embulk.cli.parse.EmbulkCommandLineHelpRequired;
|
19
|
+
import org.embulk.cli.parse.EmbulkCommandLineParseException;
|
20
|
+
import org.embulk.cli.parse.EmbulkCommandLineParser;
|
21
|
+
import org.embulk.cli.parse.OptionBehavior;
|
22
|
+
import org.embulk.cli.parse.OptionDefinition;
|
23
|
+
// TODO: Replace org.joda.time with java.time when Embulk goes to Java 8.
|
24
|
+
import org.joda.time.DateTime;
|
25
|
+
import org.joda.time.format.DateTimeFormat;
|
26
|
+
import org.joda.time.format.DateTimeFormatter;
|
27
|
+
import org.jruby.embed.ScriptingContainer;
|
28
|
+
|
29
|
+
public class EmbulkRun
|
30
|
+
{
|
31
|
+
public EmbulkRun(final String embulkVersion, final ScriptingContainer jrubyContainer)
|
32
|
+
{
|
33
|
+
this.embulkVersion = embulkVersion;
|
34
|
+
this.jrubyContainer = jrubyContainer;
|
35
|
+
}
|
36
|
+
|
37
|
+
public int run(final List<String> argsEmbulk, final List<String> argsJRuby)
|
38
|
+
{
|
39
|
+
final EmbulkArguments arguments;
|
40
|
+
try {
|
41
|
+
arguments = EmbulkArguments.extract(argsEmbulk);
|
42
|
+
}
|
43
|
+
catch (EmbulkCommandLineException ex) {
|
44
|
+
printGeneralUsage(System.err);
|
45
|
+
System.err.println("");
|
46
|
+
System.err.println("error: " + ex.getMessage());
|
47
|
+
return 1;
|
48
|
+
}
|
49
|
+
|
50
|
+
final EmbulkSubcommand subcommand = arguments.getSubcommand();
|
51
|
+
if (subcommand == null) {
|
52
|
+
printGeneralUsage(System.err);
|
53
|
+
System.err.println("");
|
54
|
+
System.err.println("Use `<command> --help` to see description of the commands.");
|
55
|
+
return 1;
|
56
|
+
}
|
57
|
+
|
58
|
+
final List<String> subcommandArguments = arguments.getSubcommandArguments();
|
59
|
+
|
60
|
+
switch (subcommand) {
|
61
|
+
case VERSION_OUT:
|
62
|
+
// TODO(v2)[#723]: Consider capitalizing this "embulk".
|
63
|
+
// https://github.com/embulk/embulk/issues/723
|
64
|
+
System.out.println("embulk " + this.embulkVersion);
|
65
|
+
return 0;
|
66
|
+
case VERSION_ERR:
|
67
|
+
// TODO(v2)[#723]: Consider capitalizing this "embulk".
|
68
|
+
// https://github.com/embulk/embulk/issues/723
|
69
|
+
System.err.println("embulk " + this.embulkVersion);
|
70
|
+
return 0;
|
71
|
+
}
|
72
|
+
|
73
|
+
printEmbulkVersionHeader(System.out);
|
74
|
+
|
75
|
+
switch (subcommand) {
|
76
|
+
case BUNDLE:
|
77
|
+
case EXEC:
|
78
|
+
case GEM:
|
79
|
+
case IRB:
|
80
|
+
return runSubcommand(subcommand, subcommandArguments, null);
|
81
|
+
default:
|
82
|
+
final EmbulkCommandLineParser parser = buildCommandLineParser(subcommand);
|
83
|
+
final EmbulkCommandLine commandLine;
|
84
|
+
try {
|
85
|
+
commandLine = parser.parse(
|
86
|
+
subcommandArguments, new PrintWriter(System.out), new PrintWriter(System.err));
|
87
|
+
}
|
88
|
+
catch (EmbulkCommandLineParseException ex) {
|
89
|
+
parser.printHelp(System.err);
|
90
|
+
System.err.println("");
|
91
|
+
System.err.println(ex.getMessage());
|
92
|
+
return 1;
|
93
|
+
}
|
94
|
+
catch (EmbulkCommandLineHelpRequired ex) {
|
95
|
+
parser.printHelp(System.err);
|
96
|
+
return 1;
|
97
|
+
}
|
98
|
+
return runSubcommand(subcommand, subcommandArguments, commandLine);
|
99
|
+
}
|
100
|
+
}
|
101
|
+
|
102
|
+
private EmbulkCommandLineParser buildCommandLineParser(final EmbulkSubcommand subcommand)
|
103
|
+
{
|
104
|
+
final EmbulkCommandLineParser.Builder parserBuilder = EmbulkCommandLineParser.builder();
|
105
|
+
|
106
|
+
// TODO: Revisit the width. JLine may help. https://github.com/jline
|
107
|
+
parserBuilder
|
108
|
+
.setWidth(160)
|
109
|
+
.addHelpMessageLine(" Help:")
|
110
|
+
.addOptionDefinition(OptionDefinition.defineHelpOption("h", "help", "Print help."))
|
111
|
+
.addHelpMessageLine("");
|
112
|
+
|
113
|
+
switch (subcommand) {
|
114
|
+
case RUN:
|
115
|
+
parserBuilder
|
116
|
+
.setMainUsage("embulk run <config.yml>")
|
117
|
+
.addHelpMessageLine(" Options:")
|
118
|
+
// op.on('-r', '--resume-state PATH', 'Path to a file to write or read resume state') do |path|
|
119
|
+
// options[:resume_state_path] = path
|
120
|
+
// end
|
121
|
+
.addOptionDefinition(OptionDefinition.defineOptionWithArgument(
|
122
|
+
"r", "resume-state", "PATH", "Path to a file to write or read resume state",
|
123
|
+
new OptionBehavior()
|
124
|
+
{
|
125
|
+
public void behave(final EmbulkCommandLine.Builder commandLineBuilder,
|
126
|
+
final String argument)
|
127
|
+
{
|
128
|
+
commandLineBuilder.setResumeState(argument);
|
129
|
+
}
|
130
|
+
}))
|
131
|
+
// op.on('-o', '--output PATH', '(deprecated)') do |path|
|
132
|
+
// STDERR.puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: Run with -o option is deprecated. Please use -c option instead. For example,"
|
133
|
+
// STDERR.puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: "
|
134
|
+
// STDERR.puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: $ embulk run config.yml -c diff.yml"
|
135
|
+
// STDERR.puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: "
|
136
|
+
// STDERR.puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: This -c option stores only diff of the next configuration."
|
137
|
+
// STDERR.puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: The diff will be merged to the original config.yml file."
|
138
|
+
// STDERR.puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: "
|
139
|
+
// options[:next_config_output_path] = path
|
140
|
+
// end
|
141
|
+
.addOptionDefinition(OptionDefinition.defineOptionWithArgument(
|
142
|
+
"o", "output", "PATH", "(deprecated)",
|
143
|
+
new OptionBehavior()
|
144
|
+
{
|
145
|
+
public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
|
146
|
+
{
|
147
|
+
final DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS Z");
|
148
|
+
final String now = DateTime.now().toString(formatter);
|
149
|
+
errorWriter().println(now + ": Run with -o option is deprecated. Please use -c option instead. For example,");
|
150
|
+
errorWriter().println(now + ": ");
|
151
|
+
errorWriter().println(now + ": $ embulk run config.yml -c diff.yml");
|
152
|
+
errorWriter().println(now + ": ");
|
153
|
+
errorWriter().println(now + ": This -c option stores only diff of the next configuration.");
|
154
|
+
errorWriter().println(now + ": The diff will be merged to the original config.yml file.");
|
155
|
+
errorWriter().println(now + ": ");
|
156
|
+
commandLineBuilder.setOutput(argument);
|
157
|
+
}
|
158
|
+
}))
|
159
|
+
// op.on('-c', '--config-diff PATH', 'Path to a file to read & write the next configuration diff') do |path|
|
160
|
+
// options[:next_config_diff_path] = path
|
161
|
+
// end
|
162
|
+
.addOptionDefinition(OptionDefinition.defineOptionWithArgument(
|
163
|
+
"c", "config-diff", "PATH", "Path to a file to read & write the next configuration diff",
|
164
|
+
new OptionBehavior()
|
165
|
+
{
|
166
|
+
public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
|
167
|
+
{
|
168
|
+
commandLineBuilder.setConfigDiff(argument);
|
169
|
+
}
|
170
|
+
}))
|
171
|
+
.setArgumentsRange(1, 1);
|
172
|
+
addPluginLoadOptionDefinitions(parserBuilder);
|
173
|
+
addOtherOptionDefinitions(parserBuilder);
|
174
|
+
break;
|
175
|
+
case CLEANUP:
|
176
|
+
parserBuilder
|
177
|
+
.setMainUsage("embulk cleanup <config.yml>")
|
178
|
+
.addHelpMessageLine(" Options:")
|
179
|
+
// op.on('-r', '--resume-state PATH', 'Path to a file to cleanup resume state') do |path|
|
180
|
+
// options[:resume_state_path] = path
|
181
|
+
// end
|
182
|
+
.addOptionDefinition(OptionDefinition.defineOptionWithArgument(
|
183
|
+
"r", "resume-state", "PATH", "Path to a file to cleanup resume state",
|
184
|
+
new OptionBehavior()
|
185
|
+
{
|
186
|
+
public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
|
187
|
+
{
|
188
|
+
commandLineBuilder.setResumeState(argument);
|
189
|
+
}
|
190
|
+
}))
|
191
|
+
.setArgumentsRange(1, 1);
|
192
|
+
addPluginLoadOptionDefinitions(parserBuilder);
|
193
|
+
addOtherOptionDefinitions(parserBuilder);
|
194
|
+
break;
|
195
|
+
case PREVIEW:
|
196
|
+
parserBuilder
|
197
|
+
.setMainUsage("embulk preview <config.yml>")
|
198
|
+
.addHelpMessageLine(" Options:")
|
199
|
+
// op.on('-G', '--vertical', "Use vertical output format", TrueClass) do |b|
|
200
|
+
// options[:format] = "vertical"
|
201
|
+
// end
|
202
|
+
.addOptionDefinition(OptionDefinition.defineOptionWithoutArgument(
|
203
|
+
"G", "vertical", "Use vertical output format",
|
204
|
+
new OptionBehavior()
|
205
|
+
{
|
206
|
+
public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
|
207
|
+
{
|
208
|
+
commandLineBuilder.setFormat("vertical");
|
209
|
+
}
|
210
|
+
}))
|
211
|
+
.setArgumentsRange(1, 1);
|
212
|
+
addPluginLoadOptionDefinitions(parserBuilder);
|
213
|
+
addOtherOptionDefinitions(parserBuilder);
|
214
|
+
break;
|
215
|
+
case GUESS:
|
216
|
+
parserBuilder
|
217
|
+
.setMainUsage("embulk guess <partial-config.yml>")
|
218
|
+
.addHelpMessageLine(" Options:")
|
219
|
+
// op.on('-o', '--output PATH', 'Path to a file to write the guessed configuration') do |path|
|
220
|
+
// options[:next_config_output_path] = path
|
221
|
+
// end
|
222
|
+
.addOptionDefinition(OptionDefinition.defineOptionWithArgument(
|
223
|
+
"o", "output", "PATH", "Path to a file to write the guessed configuration",
|
224
|
+
new OptionBehavior()
|
225
|
+
{
|
226
|
+
public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
|
227
|
+
{
|
228
|
+
commandLineBuilder.setOutput(argument);
|
229
|
+
}
|
230
|
+
}))
|
231
|
+
// op.on('-g', '--guess NAMES', "Comma-separated list of guess plugin names") do |names|
|
232
|
+
// (options[:system_config][:guess_plugins] ||= []).concat names.split(",") # TODO
|
233
|
+
// end
|
234
|
+
.addOptionDefinition(OptionDefinition.defineOptionWithArgument(
|
235
|
+
"g", "guess", "NAMES", "Comma-separated list of guess plugin names",
|
236
|
+
new OptionBehavior()
|
237
|
+
{
|
238
|
+
public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
|
239
|
+
{
|
240
|
+
for (final String guess : argument.split(",")) {
|
241
|
+
commandLineBuilder.addSystemConfig("guess_plugins", guess);
|
242
|
+
}
|
243
|
+
}
|
244
|
+
}))
|
245
|
+
.setArgumentsRange(1, 1);
|
246
|
+
addPluginLoadOptionDefinitions(parserBuilder);
|
247
|
+
addOtherOptionDefinitions(parserBuilder);
|
248
|
+
break;
|
249
|
+
case MKBUNDLE:
|
250
|
+
parserBuilder
|
251
|
+
.setMainUsage("embulk mkbundle <directory> [--path PATH]")
|
252
|
+
.addHelpMessageLine(" Options:")
|
253
|
+
// op.on('--path PATH', 'Relative path from <directory> for the location to install gems to (e.g. --path shared/bundle).') do |path|
|
254
|
+
// options[:bundle_path] = path
|
255
|
+
// end
|
256
|
+
.addOptionDefinition(OptionDefinition.defineOnlyLongOptionWithArgument(
|
257
|
+
"path", "PATH",
|
258
|
+
"Relative path from <directory> for the location to install gems to (e.g. --path shared/bundle).",
|
259
|
+
new OptionBehavior()
|
260
|
+
{
|
261
|
+
public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
|
262
|
+
{
|
263
|
+
commandLineBuilder.setBundlePath(argument);
|
264
|
+
}
|
265
|
+
}))
|
266
|
+
.addHelpMessageLine("")
|
267
|
+
.addHelpMessageLine(" \"mkbundle\" creates a new a plugin bundle directory. You can install")
|
268
|
+
.addHelpMessageLine(" plugins (gems) to the directory instead of ~/.embulk.")
|
269
|
+
.addHelpMessageLine("")
|
270
|
+
.addHelpMessageLine(" See generated <directory>/Gemfile to install plugins to the directory.")
|
271
|
+
.addHelpMessageLine(" Use -b, --bundle BUNDLE_DIR option to use it:")
|
272
|
+
.addHelpMessageLine("")
|
273
|
+
.addHelpMessageLine(" $ embulk mkbundle ./dir # create bundle directory")
|
274
|
+
.addHelpMessageLine(" $ (cd dir && vi Gemfile && embulk bundle) # update plugin list")
|
275
|
+
.addHelpMessageLine(" $ embulk guess -b ./dir ... # guess using bundled plugins")
|
276
|
+
.addHelpMessageLine(" $ embulk run -b ./dir ... # run using bundled plugins")
|
277
|
+
.setArgumentsRange(1, 1);
|
278
|
+
break;
|
279
|
+
case NEW:
|
280
|
+
parserBuilder
|
281
|
+
.setMainUsage("embulk new <category> <name>")
|
282
|
+
.addUsage("")
|
283
|
+
.addUsage("categories:")
|
284
|
+
.addUsage("")
|
285
|
+
.addUsage(" ruby-input Ruby record input plugin (like \"mysql\")")
|
286
|
+
.addUsage(" ruby-output Ruby record output plugin (like \"mysql\")")
|
287
|
+
.addUsage(" ruby-filter Ruby record filter plugin (like \"add-hostname\")")
|
288
|
+
.addUsage(" #ruby-file-input Ruby file input plugin (like \"ftp\") # not implemented yet [#21]")
|
289
|
+
.addUsage(" #ruby-file-output Ruby file output plugin (like \"ftp\") # not implemented yet [#22]")
|
290
|
+
.addUsage(" ruby-parser Ruby file parser plugin (like \"csv\")")
|
291
|
+
.addUsage(" ruby-formatter Ruby file formatter plugin (like \"csv\")")
|
292
|
+
.addUsage(" #ruby-decoder Ruby file decoder plugin (like \"gzip\") # not implemented yet [#31]")
|
293
|
+
.addUsage(" #ruby-encoder Ruby file encoder plugin (like \"gzip\") # not implemented yet [#32]")
|
294
|
+
.addUsage(" java-input Java record input plugin (like \"mysql\")")
|
295
|
+
.addUsage(" java-output Java record output plugin (like \"mysql\")")
|
296
|
+
.addUsage(" java-filter Java record filter plugin (like \"add-hostname\")")
|
297
|
+
.addUsage(" java-file-input Java file input plugin (like \"ftp\")")
|
298
|
+
.addUsage(" java-file-output Java file output plugin (like \"ftp\")")
|
299
|
+
.addUsage(" java-parser Java file parser plugin (like \"csv\")")
|
300
|
+
.addUsage(" java-formatter Java file formatter plugin (like \"csv\")")
|
301
|
+
.addUsage(" java-decoder Java file decoder plugin (like \"gzip\")")
|
302
|
+
.addUsage(" java-encoder Java file encoder plugin (like \"gzip\")")
|
303
|
+
.addUsage("")
|
304
|
+
.addUsage("examples:")
|
305
|
+
.addUsage(" new ruby-output hbase")
|
306
|
+
.addUsage(" new ruby-filter int-to-string")
|
307
|
+
.setArgumentsRange(2, 2);
|
308
|
+
break;
|
309
|
+
case MIGRATE:
|
310
|
+
parserBuilder
|
311
|
+
.setMainUsage("embulk migrate <directory>")
|
312
|
+
.setArgumentsRange(1, 1);
|
313
|
+
break;
|
314
|
+
case SELFUPDATE:
|
315
|
+
parserBuilder
|
316
|
+
.setMainUsage("embulk selfupdate")
|
317
|
+
// op.on('-f', "Skip corruption check", TrueClass) do |b|
|
318
|
+
// options[:force] = true
|
319
|
+
// end
|
320
|
+
.addOptionDefinition(OptionDefinition.defineOnlyShortOptionWithoutArgument(
|
321
|
+
"f", "Skip corruption check",
|
322
|
+
new OptionBehavior()
|
323
|
+
{
|
324
|
+
public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
|
325
|
+
{
|
326
|
+
commandLineBuilder.setForce(true);
|
327
|
+
}
|
328
|
+
}))
|
329
|
+
.setArgumentsRange(0, 1);
|
330
|
+
break;
|
331
|
+
case EXAMPLE:
|
332
|
+
parserBuilder
|
333
|
+
.setMainUsage("embulk example [directory]")
|
334
|
+
.setArgumentsRange(0, 1);
|
335
|
+
break;
|
336
|
+
default:
|
337
|
+
parserBuilder.setMainUsage("[FATAL] Unknown subcommand: " + subcommand);
|
338
|
+
}
|
339
|
+
|
340
|
+
return parserBuilder.build();
|
341
|
+
}
|
342
|
+
|
343
|
+
private int runSubcommand(final EmbulkSubcommand subcommand,
|
344
|
+
final List<String> subcommandArguments,
|
345
|
+
final EmbulkCommandLine commandLine)
|
346
|
+
{
|
347
|
+
switch (subcommand) {
|
348
|
+
case EXAMPLE:
|
349
|
+
final EmbulkExample embulkExample = new EmbulkExample();
|
350
|
+
try {
|
351
|
+
embulkExample.createExample(commandLine.getArguments().isEmpty()
|
352
|
+
? "embulk-example"
|
353
|
+
: commandLine.getArguments().get(0));
|
354
|
+
}
|
355
|
+
catch (IOException ex) {
|
356
|
+
ex.printStackTrace(System.err);
|
357
|
+
return 1;
|
358
|
+
}
|
359
|
+
return 0;
|
360
|
+
case NEW:
|
361
|
+
final String categoryWithLanguage = commandLine.getArguments().get(0);
|
362
|
+
final String nameGiven = commandLine.getArguments().get(1);
|
363
|
+
try {
|
364
|
+
final EmbulkNew embulkNew = new EmbulkNew(categoryWithLanguage, nameGiven, this.embulkVersion);
|
365
|
+
embulkNew.newPlugin();
|
366
|
+
}
|
367
|
+
catch (IOException ex) {
|
368
|
+
ex.printStackTrace();
|
369
|
+
return 1;
|
370
|
+
}
|
371
|
+
return 0;
|
372
|
+
case MIGRATE:
|
373
|
+
final String path = commandLine.getArguments().get(0);
|
374
|
+
final EmbulkMigrate embulkMigrate = new EmbulkMigrate();
|
375
|
+
try {
|
376
|
+
embulkMigrate.migratePlugin(path, this.embulkVersion);
|
377
|
+
}
|
378
|
+
catch (IOException ex) {
|
379
|
+
ex.printStackTrace();
|
380
|
+
return 1;
|
381
|
+
}
|
382
|
+
return 0;
|
383
|
+
case SELFUPDATE:
|
384
|
+
final String specifiedVersionString;
|
385
|
+
if (commandLine.getArguments().isEmpty()) {
|
386
|
+
specifiedVersionString = null;
|
387
|
+
}
|
388
|
+
else {
|
389
|
+
specifiedVersionString = commandLine.getArguments().get(0);
|
390
|
+
}
|
391
|
+
final EmbulkSelfUpdate embulkSelfUpdate = new EmbulkSelfUpdate();
|
392
|
+
try {
|
393
|
+
embulkSelfUpdate.updateSelf(this.embulkVersion, specifiedVersionString, commandLine.getForce());
|
394
|
+
}
|
395
|
+
catch (IOException | URISyntaxException ex) {
|
396
|
+
ex.printStackTrace();
|
397
|
+
return 1;
|
398
|
+
}
|
399
|
+
return 0;
|
400
|
+
case BUNDLE:
|
401
|
+
if (!commandLine.getArguments().isEmpty() && commandLine.getArguments().get(0).equals("new")) {
|
402
|
+
if (commandLine.getArguments().size() != 2) {
|
403
|
+
printGeneralUsage(System.err);
|
404
|
+
System.err.println("");
|
405
|
+
System.err.println("Use `<command> --help` to see description of the commands.");
|
406
|
+
return 1;
|
407
|
+
}
|
408
|
+
newBundle(commandLine.getArguments().get(1), null);
|
409
|
+
System.err.println("'embulk bundle new' is deprecated. This will be removed in future release. Please use 'embulk mkbundle' instead.");
|
410
|
+
}
|
411
|
+
else {
|
412
|
+
runBundler(subcommandArguments, null);
|
413
|
+
}
|
414
|
+
return 0;
|
415
|
+
case GEM:
|
416
|
+
this.jrubyContainer.runScriptlet("require 'rubygems/gem_runner'");
|
417
|
+
this.jrubyContainer.put("__internal_argv_java__", subcommandArguments);
|
418
|
+
this.jrubyContainer.runScriptlet("__internal_argv__ = Array.new(__internal_argv_java__)");
|
419
|
+
this.jrubyContainer.runScriptlet("Gem::GemRunner.new.run __internal_argv__");
|
420
|
+
this.jrubyContainer.remove("__internal_argv_java__");
|
421
|
+
return 0;
|
422
|
+
case MKBUNDLE:
|
423
|
+
newBundle(commandLine.getArguments().get(0), commandLine.getBundlePath());
|
424
|
+
break;
|
425
|
+
case EXEC:
|
426
|
+
this.jrubyContainer.put("__internal_argv_java__", subcommandArguments);
|
427
|
+
this.jrubyContainer.runScriptlet("__internal_argv__ = Array.new(__internal_argv_java__)");
|
428
|
+
this.jrubyContainer.runScriptlet("exec(*__internal_argv__)");
|
429
|
+
this.jrubyContainer.remove("__internal_argv_java__");
|
430
|
+
return 127;
|
431
|
+
case IRB:
|
432
|
+
this.jrubyContainer.runScriptlet("require 'irb'");
|
433
|
+
this.jrubyContainer.runScriptlet("IRB.start");
|
434
|
+
return 0;
|
435
|
+
case RUN:
|
436
|
+
case CLEANUP:
|
437
|
+
case PREVIEW:
|
438
|
+
case GUESS:
|
439
|
+
// NOTE: When it was in Ruby "require 'embulk'" was required on top for Ruby |Embulk::setup|.
|
440
|
+
// Ruby |Embulk::setup| is now replaced with Java |org.embulk.EmbulkSetup.setup|.
|
441
|
+
|
442
|
+
// TODO: Move this to initial JRuby instantiation.
|
443
|
+
// reset context class loader set by org.jruby.Main.main to nil. embulk manages
|
444
|
+
// multiple classloaders. default classloader should be Plugin.class.getClassloader().
|
445
|
+
Thread.currentThread().setContextClassLoader(null);
|
446
|
+
|
447
|
+
// NOTE: When it was in Ruby ""require 'json'" was required.
|
448
|
+
|
449
|
+
setupLoadPaths(commandLine.getLoadPath(), commandLine.getLoad());
|
450
|
+
setupClasspaths(commandLine.getClasspath());
|
451
|
+
|
452
|
+
// call |EmbulkSetup.setup| after setup_classpaths to allow users to overwrite
|
453
|
+
// embulk classes
|
454
|
+
// NOTE: |EmbulkSetup.setup| returns |EmbulkEmbed| while it stores Ruby |Embulk::EmbulkRunner(EmbulkEmbed)|
|
455
|
+
// into Ruby |Embulk::Runner|.
|
456
|
+
final EmbulkRunner runner = EmbulkSetup.setup(commandLine.getSystemConfig(), this.jrubyContainer);
|
457
|
+
|
458
|
+
final Path configDiffPath =
|
459
|
+
(commandLine.getConfigDiff() == null ? null : Paths.get(commandLine.getConfigDiff()));
|
460
|
+
final Path outputPath =
|
461
|
+
(commandLine.getOutput() == null ? null : Paths.get(commandLine.getOutput()));
|
462
|
+
final Path resumeStatePath =
|
463
|
+
(commandLine.getResumeState() == null ? null : Paths.get(commandLine.getResumeState()));
|
464
|
+
|
465
|
+
try {
|
466
|
+
switch (subcommand) {
|
467
|
+
case GUESS:
|
468
|
+
runner.guess(Paths.get(commandLine.getArguments().get(0)), outputPath);
|
469
|
+
break;
|
470
|
+
case PREVIEW:
|
471
|
+
runner.preview(Paths.get(commandLine.getArguments().get(0)), commandLine.getFormat());
|
472
|
+
break;
|
473
|
+
case RUN:
|
474
|
+
runner.run(Paths.get(commandLine.getArguments().get(0)),
|
475
|
+
configDiffPath,
|
476
|
+
outputPath,
|
477
|
+
resumeStatePath);
|
478
|
+
break;
|
479
|
+
}
|
480
|
+
}
|
481
|
+
catch (Throwable ex) {
|
482
|
+
ex.printStackTrace(System.err);
|
483
|
+
System.err.println("");
|
484
|
+
System.err.println("Error: " + ex.getMessage());
|
485
|
+
return 1;
|
486
|
+
}
|
487
|
+
}
|
488
|
+
return 0;
|
489
|
+
}
|
490
|
+
|
491
|
+
private int newBundle(final String pathString, final String bundlePath)
|
492
|
+
{
|
493
|
+
this.jrubyContainer.runScriptlet("require 'embulk'");
|
494
|
+
|
495
|
+
final Path path = Paths.get(pathString);
|
496
|
+
this.jrubyContainer.runScriptlet("require 'fileutils'");
|
497
|
+
this.jrubyContainer.runScriptlet("require 'rubygems/gem_runner'");
|
498
|
+
|
499
|
+
if (Files.exists(path)) {
|
500
|
+
System.err.println("'" + pathString + "' already exists.");
|
501
|
+
return 1;
|
502
|
+
}
|
503
|
+
|
504
|
+
System.out.println("Initializing " + pathString + "...");
|
505
|
+
try {
|
506
|
+
Files.createDirectories(path);
|
507
|
+
}
|
508
|
+
catch (IOException ex) {
|
509
|
+
ex.printStackTrace();
|
510
|
+
return 1;
|
511
|
+
}
|
512
|
+
boolean success = false;
|
513
|
+
try {
|
514
|
+
// copy embulk/data/bundle/ contents
|
515
|
+
this.jrubyContainer.runScriptlet("require 'embulk/data/package_data'");
|
516
|
+
this.jrubyContainer.put("__internal_path__", pathString);
|
517
|
+
this.jrubyContainer.runScriptlet("pkg = Embulk::PackageData.new('bundle', __internal_path__)");
|
518
|
+
this.jrubyContainer.remove("__internal_path__");
|
519
|
+
this.jrubyContainer.runScriptlet("%w[Gemfile .ruby-version .bundle/config embulk/input/example.rb embulk/output/example.rb embulk/filter/example.rb].each { |file| pkg.cp(file, file) }");
|
520
|
+
// run the first bundle-install
|
521
|
+
final ArrayList<String> bundlerArguments = new ArrayList<String>();
|
522
|
+
bundlerArguments.add("install");
|
523
|
+
bundlerArguments.add("--path");
|
524
|
+
runBundler(Arrays.asList("install", "--path", bundlePath != null ? bundlePath : "."), path);
|
525
|
+
success = true;
|
526
|
+
}
|
527
|
+
catch (Exception ex) {
|
528
|
+
ex.printStackTrace();
|
529
|
+
throw ex;
|
530
|
+
// success = true;
|
531
|
+
}
|
532
|
+
finally {
|
533
|
+
if (!success) {
|
534
|
+
try {
|
535
|
+
Files.walkFileTree(path, new SimpleFileVisitor<Path>()
|
536
|
+
{
|
537
|
+
@Override
|
538
|
+
public FileVisitResult visitFile(Path file, BasicFileAttributes attributes)
|
539
|
+
{
|
540
|
+
try {
|
541
|
+
Files.deleteIfExists(file);
|
542
|
+
}
|
543
|
+
catch (IOException ex) {
|
544
|
+
// Ignore.
|
545
|
+
}
|
546
|
+
return FileVisitResult.CONTINUE;
|
547
|
+
}
|
548
|
+
|
549
|
+
@Override
|
550
|
+
public FileVisitResult postVisitDirectory(Path dir, IOException exception)
|
551
|
+
{
|
552
|
+
try {
|
553
|
+
Files.deleteIfExists(dir);
|
554
|
+
}
|
555
|
+
catch (IOException ex) {
|
556
|
+
// Ignore.
|
557
|
+
}
|
558
|
+
return FileVisitResult.CONTINUE;
|
559
|
+
}
|
560
|
+
});
|
561
|
+
}
|
562
|
+
catch (IOException ex) {
|
563
|
+
ex.printStackTrace();
|
564
|
+
return 1;
|
565
|
+
}
|
566
|
+
}
|
567
|
+
}
|
568
|
+
return 0;
|
569
|
+
}
|
570
|
+
|
571
|
+
private void runBundler(final List<String> arguments, final Path path)
|
572
|
+
{
|
573
|
+
this.jrubyContainer.runScriptlet("require 'bundler'"); // bundler is included in embulk-core.jar
|
574
|
+
|
575
|
+
// this hack is necessary to make --help working
|
576
|
+
this.jrubyContainer.runScriptlet("Bundler.define_singleton_method(:which_orig, Bundler.method(:which))");
|
577
|
+
this.jrubyContainer.runScriptlet("Bundler.define_singleton_method(:which) { |executable| (executable == 'man' ? false : which_orig(executable)) }");
|
578
|
+
|
579
|
+
this.jrubyContainer.runScriptlet("require 'bundler/friendly_errors'");
|
580
|
+
this.jrubyContainer.runScriptlet("require 'bundler/cli'");
|
581
|
+
|
582
|
+
this.jrubyContainer.put("__internal_argv_java__", arguments);
|
583
|
+
this.jrubyContainer.runScriptlet("__internal_argv__ = Array.new(__internal_argv_java__)");
|
584
|
+
if (path == null) {
|
585
|
+
this.jrubyContainer.runScriptlet("Bundler.with_friendly_errors { Bundler::CLI.start(__internal_argv__, debug: true) }");
|
586
|
+
}
|
587
|
+
else {
|
588
|
+
this.jrubyContainer.put("__internal_working_dir__", path.toString());
|
589
|
+
this.jrubyContainer.runScriptlet("Dir.chdir(__internal_working_dir__) { Bundler.with_friendly_errors { Bundler::CLI.start(__internal_argv__, debug: true) } }");
|
590
|
+
this.jrubyContainer.remove("__internal_working_dir__");
|
591
|
+
}
|
592
|
+
this.jrubyContainer.remove("__internal_argv_java__");
|
593
|
+
}
|
594
|
+
|
595
|
+
private void addPluginLoadOptionDefinitions(final EmbulkCommandLineParser.Builder parserBuilder)
|
596
|
+
{
|
597
|
+
parserBuilder.addHelpMessageLine("");
|
598
|
+
parserBuilder.addHelpMessageLine(" Plugin load options:");
|
599
|
+
// op.on('-L', '--load PATH', 'Add a local plugin path') do |plugin_path|
|
600
|
+
// plugin_paths << plugin_path
|
601
|
+
// end
|
602
|
+
parserBuilder.addOptionDefinition(OptionDefinition.defineOptionWithArgument(
|
603
|
+
"L", "load", "PATH", "Add a local plugin path",
|
604
|
+
new OptionBehavior()
|
605
|
+
{
|
606
|
+
public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
|
607
|
+
{
|
608
|
+
commandLineBuilder.addLoad(argument);
|
609
|
+
}
|
610
|
+
}));
|
611
|
+
// op.on('-I', '--load-path PATH', 'Add ruby script directory path ($LOAD_PATH)') do |load_path|
|
612
|
+
// load_paths << load_path
|
613
|
+
// end
|
614
|
+
parserBuilder.addOptionDefinition(OptionDefinition.defineOptionWithArgument(
|
615
|
+
"I", "load-path", "PATH", "Add ruby script directory path ($LOAD_PATH)",
|
616
|
+
new OptionBehavior()
|
617
|
+
{
|
618
|
+
public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
|
619
|
+
{
|
620
|
+
commandLineBuilder.addLoadPath(argument);
|
621
|
+
}
|
622
|
+
}));
|
623
|
+
// op.on('-C', '--classpath PATH', "Add java classpath separated by #{classpath_separator} (CLASSPATH)") do |classpath|
|
624
|
+
// classpaths.concat classpath.split(classpath_separator)
|
625
|
+
// end
|
626
|
+
parserBuilder.addOptionDefinition(OptionDefinition.defineOptionWithArgument(
|
627
|
+
"C", "classpath", "PATH", "Add java classpath separated by " + java.io.File.pathSeparator + " (CLASSPATH)",
|
628
|
+
new OptionBehavior()
|
629
|
+
{
|
630
|
+
public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
|
631
|
+
{
|
632
|
+
final String[] classpaths = argument.split("\\" + java.io.File.pathSeparator);
|
633
|
+
for (final String classpath : classpaths) {
|
634
|
+
commandLineBuilder.addClasspath(classpath);
|
635
|
+
}
|
636
|
+
}
|
637
|
+
}));
|
638
|
+
// op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory (create one using "embulk mkbundle" command)') do |path|
|
639
|
+
// # only for help message. implemented at lib/embulk/command/embulk_bundle.rb
|
640
|
+
// end
|
641
|
+
parserBuilder.addOptionDefinition(OptionDefinition.defineOptionWithArgument(
|
642
|
+
"b", "bundle", "BUNDLE_DIR", "Path to a Gemfile directory (create one using \"embulk mkbundle\" command)",
|
643
|
+
new OptionBehavior()
|
644
|
+
{
|
645
|
+
public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
|
646
|
+
{
|
647
|
+
commandLineBuilder.setBundle(argument);
|
648
|
+
}
|
649
|
+
}));
|
650
|
+
}
|
651
|
+
|
652
|
+
private void addOtherOptionDefinitions(final EmbulkCommandLineParser.Builder parserBuilder)
|
653
|
+
{
|
654
|
+
parserBuilder.addHelpMessageLine("");
|
655
|
+
parserBuilder.addHelpMessageLine(" Other options:");
|
656
|
+
// op.on('-l', '--log PATH', 'Output log messages to a file (default: -)') do |path|
|
657
|
+
// options[:system_config][:log_path] = path
|
658
|
+
// end
|
659
|
+
parserBuilder.addOptionDefinition(OptionDefinition.defineOnlyLongOptionWithArgument(
|
660
|
+
"log", "PATH", "Output log messages to a file (default: -)",
|
661
|
+
new OptionBehavior()
|
662
|
+
{
|
663
|
+
public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
|
664
|
+
{
|
665
|
+
commandLineBuilder.setSystemConfig("log_path", argument);
|
666
|
+
}
|
667
|
+
}));
|
668
|
+
// op.on('-l', '--log-level LEVEL', 'Log level (error, warn, info, debug or trace)') do |level|
|
669
|
+
// options[:system_config][:log_level] = level
|
670
|
+
// end
|
671
|
+
parserBuilder.addOptionDefinition(OptionDefinition.defineOptionWithArgument(
|
672
|
+
"l", "log-level", "LEVEL", "Log level (error, warn, info, debug or trace)",
|
673
|
+
new OptionBehavior()
|
674
|
+
{
|
675
|
+
public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
|
676
|
+
{
|
677
|
+
commandLineBuilder.setSystemConfig("log_level", argument);
|
678
|
+
}
|
679
|
+
}));
|
680
|
+
// op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
|
681
|
+
// k, v = kv.split('=', 2)
|
682
|
+
// v ||= "true"
|
683
|
+
// options[:system_config][k] = v
|
684
|
+
// end
|
685
|
+
parserBuilder.addOptionDefinition(OptionDefinition.defineOnlyShortOptionWithArgument(
|
686
|
+
"X", "KEY=VALUE", "Add a performance system config",
|
687
|
+
new OptionBehavior()
|
688
|
+
{
|
689
|
+
public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
|
690
|
+
throws EmbulkCommandLineParseException
|
691
|
+
{
|
692
|
+
try {
|
693
|
+
final String[] keyValue = argument.split("=", 2);
|
694
|
+
commandLineBuilder.setSystemConfig(keyValue[0], keyValue[1]);
|
695
|
+
}
|
696
|
+
catch (Throwable ex) {
|
697
|
+
throw new EmbulkCommandLineParseException(ex);
|
698
|
+
}
|
699
|
+
}
|
700
|
+
}));
|
701
|
+
}
|
702
|
+
|
703
|
+
private void setupLoadPaths(final List<String> loadPaths, final List<String> pluginPaths)
|
704
|
+
{
|
705
|
+
// first $LOAD_PATH has highet priority. later load_paths should have highest priority.
|
706
|
+
for (final String loadPath : loadPaths) {
|
707
|
+
// ruby script directory (use unshift to make it highest priority)
|
708
|
+
this.jrubyContainer.put("__internal_load_path__", loadPath);
|
709
|
+
this.jrubyContainer.runScriptlet("$LOAD_PATH.unshift File.expand_path(__internal_load_path__)");
|
710
|
+
this.jrubyContainer.remove("__internal_load_path__");
|
711
|
+
}
|
712
|
+
|
713
|
+
// # Gem::StubSpecification is an internal API that seems chainging often.
|
714
|
+
// # Gem::Specification.add_spec is deprecated also. Therefore, here makes
|
715
|
+
// # -L <path> option alias of -I <path>/lib by assuming that *.gemspec file
|
716
|
+
// # always has require_paths = ["lib"].
|
717
|
+
for (final String pluginPath : pluginPaths) {
|
718
|
+
this.jrubyContainer.put("__internal_plugin_path__", pluginPath);
|
719
|
+
this.jrubyContainer.runScriptlet("$LOAD_PATH.unshift File.expand_path(File.join(__internal_plugin_path__, 'lib')");
|
720
|
+
this.jrubyContainer.remove("__internal_plugin_path__");
|
721
|
+
}
|
722
|
+
}
|
723
|
+
|
724
|
+
private void setupClasspaths(final List<String> classpaths)
|
725
|
+
{
|
726
|
+
for (final String classpath : classpaths) {
|
727
|
+
this.jrubyContainer.put("__internal_classpath__", classpath);
|
728
|
+
// $CLASSPATH object doesn't have concat method
|
729
|
+
this.jrubyContainer.runScriptlet("$CLASSPATH << __internal_classpath__");
|
730
|
+
this.jrubyContainer.remove("__internal_classpath__");
|
731
|
+
}
|
732
|
+
}
|
733
|
+
|
734
|
+
private void printGeneralUsage(final PrintStream out)
|
735
|
+
{
|
736
|
+
final String gemHomeEnv = System.getenv("GEM_HOME");
|
737
|
+
out.println("Embulk v" + this.embulkVersion);
|
738
|
+
out.println("Usage: embulk [-vm-options] <command> [--options]");
|
739
|
+
out.println("Commands:");
|
740
|
+
out.println(" mkbundle <directory> # create a new plugin bundle environment.");
|
741
|
+
out.println(" bundle [directory] # update a plugin bundle environment.");
|
742
|
+
out.println(" run <config.yml> # run a bulk load transaction.");
|
743
|
+
out.println(" cleanup <config.yml> # cleanup resume state.");
|
744
|
+
out.println(" preview <config.yml> # dry-run the bulk load without output and show preview.");
|
745
|
+
out.println(" guess <partial-config.yml> -o <output.yml> # guess missing parameters to create a complete configuration file.");
|
746
|
+
out.println(" gem <install | list | help> # install a plugin or show installed plugins.");
|
747
|
+
out.println(" # plugin path is " + (gemHomeEnv == null ? "(empty)" : gemHomeEnv));
|
748
|
+
out.println(" new <category> <name> # generates new plugin template");
|
749
|
+
out.println(" migrate <path> # modify plugin code to use the latest Embulk plugin API");
|
750
|
+
out.println(" example [path] # creates an example config file and csv file to try embulk.");
|
751
|
+
out.println(" selfupdate [version] # upgrades embulk to the latest released version or to the specified version.");
|
752
|
+
out.println("");
|
753
|
+
out.println("VM options:");
|
754
|
+
out.println(" -J-O Disable JVM optimizations to speed up startup time (enabled by default if command is 'run')");
|
755
|
+
out.println(" -J+O Enable JVM optimizations to speed up throughput");
|
756
|
+
out.println(" -J... Set JVM options (use -J-help to see available options)");
|
757
|
+
out.println(" -R... Set JRuby options (use -R--help to see available options)");
|
758
|
+
}
|
759
|
+
|
760
|
+
private void printEmbulkVersionHeader(final PrintStream out)
|
761
|
+
{
|
762
|
+
final DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS Z");
|
763
|
+
final String now = DateTime.now().toString(formatter);
|
764
|
+
out.println(now + ": Embulk v" + this.embulkVersion);
|
765
|
+
}
|
766
|
+
|
767
|
+
private final String embulkVersion;
|
768
|
+
private final ScriptingContainer jrubyContainer;
|
769
|
+
}
|