embulk 0.7.11-java → 0.8.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +3 -3
- data/README.md +1 -1
- data/build.gradle +2 -2
- data/embulk-core/build.gradle +2 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigLoader.java +11 -3
- data/embulk-core/src/main/java/org/embulk/config/YamlTagResolver.java +53 -0
- data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java +0 -1
- data/embulk-core/src/main/java/org/embulk/exec/LocalExecutorPlugin.java +479 -69
- data/embulk-core/src/main/java/org/embulk/spi/Column.java +3 -0
- data/embulk-core/src/main/java/org/embulk/spi/ColumnVisitor.java +2 -0
- data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java +12 -5
- data/embulk-core/src/main/java/org/embulk/spi/Page.java +19 -0
- data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java +26 -5
- data/embulk-core/src/main/java/org/embulk/spi/PageReader.java +13 -0
- data/embulk-core/src/main/java/org/embulk/spi/json/JsonParseException.java +17 -0
- data/embulk-core/src/main/java/org/embulk/spi/json/JsonParser.java +125 -0
- data/embulk-core/src/main/java/org/embulk/spi/json/RubyValueApi.java +55 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/JsonType.java +14 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/TypeDeserializer.java +1 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/Types.java +2 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetterFactory.java +6 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/PagePrinter.java +5 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Pages.java +10 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java +3 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/BooleanColumnSetter.java +7 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DefaultValueSetter.java +2 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DoubleColumnSetter.java +7 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/JsonColumnSetter.java +73 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/LongColumnSetter.java +11 -2
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/NullDefaultValueSetter.java +5 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/SkipColumnSetter.java +5 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/StringColumnSetter.java +7 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/TimestampColumnSetter.java +9 -1
- data/embulk-core/src/test/java/org/embulk/spi/MockFormatterPlugin.java +7 -0
- data/embulk-docs/src/built-in.rst +40 -3
- data/embulk-docs/src/conf.py +2 -2
- data/embulk-docs/src/release.rst +1 -1
- data/embulk-docs/src/release/release-0.8.0.rst +68 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java +12 -1
- data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +18 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java +1 -1
- data/embulk.gemspec +1 -1
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/gradlew +3 -7
- data/lib/embulk/column.rb +2 -0
- data/lib/embulk/command/embulk_migrate_plugin.rb +76 -10
- data/lib/embulk/command/embulk_new_plugin.rb +2 -0
- data/lib/embulk/command/embulk_run.rb +17 -10
- data/lib/embulk/data/bundle/.ruby-version +1 -1
- data/lib/embulk/data/new/java/build.gradle.erb +21 -0
- data/lib/embulk/data/new/java/config/checkstyle/checkstyle.xml +128 -0
- data/lib/embulk/data/new/java/config/checkstyle/default.xml +108 -0
- data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/lib/embulk/data/new/java/gradlew +3 -7
- data/lib/embulk/data/new/ruby/.ruby-version +1 -1
- data/lib/embulk/guess/csv.rb +1 -1
- data/lib/embulk/guess/schema_guess.rb +6 -0
- data/lib/embulk/guess_plugin.rb +1 -1
- data/lib/embulk/java/imports.rb +4 -0
- data/lib/embulk/plugin_registry.rb +8 -12
- data/lib/embulk/schema.rb +6 -0
- data/lib/embulk/version.rb +1 -1
- data/test/guess/test_csv_guess.rb +170 -0
- data/test/helper.rb +2 -0
- metadata +17 -15
- data/embulk-core/src/main/java/org/embulk/exec/LocalThreadExecutor.java +0 -34
- data/embulk-core/src/main/java/org/embulk/guice/Bootstrap.java +0 -157
- data/embulk-core/src/main/java/org/embulk/guice/CloseableInjector.java +0 -22
- data/embulk-core/src/main/java/org/embulk/guice/InjectorProxy.java +0 -145
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleInjector.java +0 -26
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleInjectorProxy.java +0 -61
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleManager.java +0 -187
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleMethods.java +0 -89
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleMethodsMap.java +0 -38
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleModule.java +0 -97
- data/embulk-docs/src/release/release-0.7.11.rst +0 -13
|
@@ -18,8 +18,8 @@ import org.embulk.spi.PageReader;
|
|
|
18
18
|
import org.embulk.spi.FileOutput;
|
|
19
19
|
import org.embulk.spi.util.LineEncoder;
|
|
20
20
|
import org.embulk.spi.util.Timestamps;
|
|
21
|
-
|
|
22
21
|
import org.embulk.spi.util.Newline;
|
|
22
|
+
import org.msgpack.value.Value;
|
|
23
23
|
import java.util.Map;
|
|
24
24
|
|
|
25
25
|
public class CsvFormatterPlugin
|
|
@@ -180,6 +180,17 @@ public class CsvFormatterPlugin
|
|
|
180
180
|
}
|
|
181
181
|
}
|
|
182
182
|
|
|
183
|
+
public void jsonColumn(Column column)
|
|
184
|
+
{
|
|
185
|
+
addDelimiter(column);
|
|
186
|
+
if (!pageReader.isNull(column)) {
|
|
187
|
+
Value value = pageReader.getJson(column);
|
|
188
|
+
addValue(value.toJson());
|
|
189
|
+
} else {
|
|
190
|
+
addNullString();
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
183
194
|
private void addDelimiter(Column column)
|
|
184
195
|
{
|
|
185
196
|
if (column.getIndex() != 0) {
|
|
@@ -13,6 +13,8 @@ import org.embulk.config.ConfigException;
|
|
|
13
13
|
import org.embulk.config.TaskSource;
|
|
14
14
|
import org.embulk.spi.time.TimestampParser;
|
|
15
15
|
import org.embulk.spi.time.TimestampParseException;
|
|
16
|
+
import org.embulk.spi.json.JsonParser;
|
|
17
|
+
import org.embulk.spi.json.JsonParseException;
|
|
16
18
|
import org.embulk.spi.Column;
|
|
17
19
|
import org.embulk.spi.Schema;
|
|
18
20
|
import org.embulk.spi.SchemaConfig;
|
|
@@ -229,6 +231,7 @@ public class CsvParserPlugin
|
|
|
229
231
|
{
|
|
230
232
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
|
231
233
|
final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, task.getSchemaConfig());
|
|
234
|
+
final JsonParser jsonParser = new JsonParser();
|
|
232
235
|
final CsvTokenizer tokenizer = new CsvTokenizer(new LineDecoder(input, task), task);
|
|
233
236
|
final String nullStringOrNull = task.getNullString().orNull();
|
|
234
237
|
final boolean allowOptionalColumns = task.getAllowOptionalColumns();
|
|
@@ -320,6 +323,21 @@ public class CsvParserPlugin
|
|
|
320
323
|
}
|
|
321
324
|
}
|
|
322
325
|
|
|
326
|
+
public void jsonColumn(Column column)
|
|
327
|
+
{
|
|
328
|
+
String v = nextColumn();
|
|
329
|
+
if (v == null) {
|
|
330
|
+
pageBuilder.setNull(column);
|
|
331
|
+
} else {
|
|
332
|
+
try {
|
|
333
|
+
pageBuilder.setJson(column, jsonParser.parse(v));
|
|
334
|
+
} catch (JsonParseException e) {
|
|
335
|
+
// TODO support default value
|
|
336
|
+
throw new CsvRecordValidateException(e);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
|
|
323
341
|
private String nextColumn()
|
|
324
342
|
{
|
|
325
343
|
if (allowOptionalColumns && !tokenizer.hasNextColumn()) {
|
data/embulk.gemspec
CHANGED
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
#
|
|
1
|
+
#Wed Jan 13 12:41:02 JST 2016
|
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
|
3
3
|
distributionPath=wrapper/dists
|
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
|
5
5
|
zipStorePath=wrapper/dists
|
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
|
data/gradlew
CHANGED
|
@@ -42,11 +42,6 @@ case "`uname`" in
|
|
|
42
42
|
;;
|
|
43
43
|
esac
|
|
44
44
|
|
|
45
|
-
# For Cygwin, ensure paths are in UNIX format before anything is touched.
|
|
46
|
-
if $cygwin ; then
|
|
47
|
-
[ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
|
|
48
|
-
fi
|
|
49
|
-
|
|
50
45
|
# Attempt to set APP_HOME
|
|
51
46
|
# Resolve links: $0 may be a link
|
|
52
47
|
PRG="$0"
|
|
@@ -61,9 +56,9 @@ while [ -h "$PRG" ] ; do
|
|
|
61
56
|
fi
|
|
62
57
|
done
|
|
63
58
|
SAVED="`pwd`"
|
|
64
|
-
cd "`dirname \"$PRG\"`/"
|
|
59
|
+
cd "`dirname \"$PRG\"`/" >/dev/null
|
|
65
60
|
APP_HOME="`pwd -P`"
|
|
66
|
-
cd "$SAVED"
|
|
61
|
+
cd "$SAVED" >/dev/null
|
|
67
62
|
|
|
68
63
|
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
|
|
69
64
|
|
|
@@ -114,6 +109,7 @@ fi
|
|
|
114
109
|
if $cygwin ; then
|
|
115
110
|
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
|
|
116
111
|
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
|
|
112
|
+
JAVACMD=`cygpath --unix "$JAVACMD"`
|
|
117
113
|
|
|
118
114
|
# We build the pattern for arguments to be converted via cygpath
|
|
119
115
|
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
|
data/lib/embulk/column.rb
CHANGED
|
@@ -57,6 +57,8 @@ module Embulk
|
|
|
57
57
|
Java::Types::STRING
|
|
58
58
|
when :timestamp
|
|
59
59
|
Java::Types::TIMESTAMP
|
|
60
|
+
when :json
|
|
61
|
+
Java::Types::JSON
|
|
60
62
|
else
|
|
61
63
|
raise ArgumentError, "Unknown type #{ruby_type.inspect}: supported types are :boolean, :long, :double, :string and :timestamp"
|
|
62
64
|
end
|
|
@@ -2,9 +2,9 @@ module Embulk
|
|
|
2
2
|
def self.migrate_plugin(path)
|
|
3
3
|
migrator = Migrator.new(path)
|
|
4
4
|
|
|
5
|
-
if ms = migrator.match("**/build.gradle", /org\.embulk:embulk-core:([\d
|
|
5
|
+
if ms = migrator.match("**/build.gradle", /org\.embulk:embulk-core:([\d\.\+]+)?/)
|
|
6
6
|
lang = :java
|
|
7
|
-
from_ver = version(ms[0][1])
|
|
7
|
+
from_ver = version(ms[0][1].gsub(/\++/, '0')) # replaces "0.8.+" to "0.8.0"
|
|
8
8
|
puts "Detected Java plugin for Embulk #{from_ver}..."
|
|
9
9
|
|
|
10
10
|
elsif ms = migrator.match("**/*.gemspec", /add_(?:development_)?dependency\s+\W+embulk\W+\s+([\d\.]+)\W+/)
|
|
@@ -42,12 +42,71 @@ module Embulk
|
|
|
42
42
|
migrator.replace("**/*.java", /(commitReport)/, "taskReport")
|
|
43
43
|
end
|
|
44
44
|
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
# upgrade gradle version
|
|
46
|
+
if migrator.match("gradle/wrapper/gradle-wrapper.properties", /gradle-2\.\d-/)
|
|
47
|
+
# gradle < 2.10 (\d matches one digit)
|
|
47
48
|
require 'embulk/data/package_data'
|
|
48
49
|
data = PackageData.new("new", migrator.path)
|
|
49
|
-
migrator.write "gradle/wrapper/gradle-wrapper.jar", data.bincontent("java/gradle/wrapper/gradle-wrapper.jar")
|
|
50
50
|
migrator.write "gradle/wrapper/gradle-wrapper.properties", data.content("java/gradle/wrapper/gradle-wrapper.properties")
|
|
51
|
+
migrator.write "gradle/wrapper/gradle-wrapper.jar", data.bincontent("java/gradle/wrapper/gradle-wrapper.jar")
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# add jsonColumn method
|
|
55
|
+
if !migrator.match("**/*.java", /void\s+jsonColumn/) && ms = migrator.match("**/*.java", /^(\W+).*?void\s+timestampColumn/)
|
|
56
|
+
indent = ms.first[1]
|
|
57
|
+
replace = <<EOF
|
|
58
|
+
|
|
59
|
+
#{indent}public void jsonColumn(Column column) {
|
|
60
|
+
#{indent} throw new UnsupportedOperationException("This plugin doesn't support json type. Please try to upgrade version of the plugin using 'embulk gem update' command. If the latest version still doesn't support json type, please contact plugin developers, or change configuration of input plugin not to use json type.");
|
|
61
|
+
#{indent}}
|
|
62
|
+
|
|
63
|
+
#{indent}@Override
|
|
64
|
+
EOF
|
|
65
|
+
migrator.replace("**/*.java", /(\r?\n)(\W+).*?void\s+timestampColumn/, replace)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# add sourceCompatibility and targetCompatibility
|
|
69
|
+
unless migrator.match("build.gradle", /targetCompatibility/)
|
|
70
|
+
migrator.insert_line("build.gradle", /^([ \t]*)dependencies\s*{/) {|m|
|
|
71
|
+
"#{m[1]}targetCompatibility = 1.7\n"
|
|
72
|
+
}
|
|
73
|
+
end
|
|
74
|
+
unless migrator.match("build.gradle", /sourceCompatibility/)
|
|
75
|
+
migrator.insert_line("build.gradle", /^([ \t]*)targetCompatibility/) {|m|
|
|
76
|
+
"#{m[1]}sourceCompatibility = 1.7"
|
|
77
|
+
}
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# add checkstyle
|
|
81
|
+
unless migrator.match("build.gradle", /id\s+(?<quote>["'])checkstyle\k<quote>/)
|
|
82
|
+
migrator.insert_line("build.gradle", /^([ \t]*)id( +)(["'])java["']/) {|m|
|
|
83
|
+
"#{m[1]}id#{m[2]}#{m[3]}checkstyle#{m[3]}"
|
|
84
|
+
}
|
|
85
|
+
migrator.write "config/checkstyle/checkstyle.xml", migrator.new_data.content("java/config/checkstyle/checkstyle.xml")
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
unless migrator.match("build.gradle", /checkstyle\s+{/)
|
|
89
|
+
migrator.write "config/checkstyle/default.xml", migrator.new_data.content("java/config/checkstyle/default.xml")
|
|
90
|
+
migrator.insert_line("build.gradle", /^([ \t]*)task\s+gem\W.*{/) {|m|
|
|
91
|
+
<<-EOF
|
|
92
|
+
#{m[1]}checkstyle {
|
|
93
|
+
#{m[1]} configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
|
|
94
|
+
#{m[1]} toolVersion = '6.14.1'
|
|
95
|
+
#{m[1]}}
|
|
96
|
+
#{m[1]}checkstyleMain {
|
|
97
|
+
#{m[1]} configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
98
|
+
#{m[1]} ignoreFailures = true
|
|
99
|
+
#{m[1]}}
|
|
100
|
+
#{m[1]}checkstyleTest {
|
|
101
|
+
#{m[1]} configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
102
|
+
#{m[1]} ignoreFailures = true
|
|
103
|
+
#{m[1]}}
|
|
104
|
+
#{m[1]}task checkstyle(type: Checkstyle) {
|
|
105
|
+
#{m[1]} classpath = sourceSets.main.output + sourceSets.test.output
|
|
106
|
+
#{m[1]} source = sourceSets.main.allJava + sourceSets.test.allJava
|
|
107
|
+
#{m[1]}}
|
|
108
|
+
EOF
|
|
109
|
+
}
|
|
51
110
|
end
|
|
52
111
|
|
|
53
112
|
#
|
|
@@ -55,7 +114,7 @@ module Embulk
|
|
|
55
114
|
##
|
|
56
115
|
|
|
57
116
|
# update version at the end
|
|
58
|
-
migrator.replace("**/build.gradle", /org\.embulk:embulk-(?:core|standards):([\d
|
|
117
|
+
migrator.replace("**/build.gradle", /org\.embulk:embulk-(?:core|standards):([\d\.\+]+)?/, Embulk::VERSION)
|
|
59
118
|
end
|
|
60
119
|
|
|
61
120
|
def self.migrate_ruby_plugin(migrator, from_ver)
|
|
@@ -63,7 +122,7 @@ module Embulk
|
|
|
63
122
|
# add rules...
|
|
64
123
|
##
|
|
65
124
|
|
|
66
|
-
migrator.write(".ruby-version", "jruby-9.0.
|
|
125
|
+
migrator.write(".ruby-version", "jruby-9.0.4.0")
|
|
67
126
|
|
|
68
127
|
# update version at the end
|
|
69
128
|
if from_ver <= version("0.1.0")
|
|
@@ -86,9 +145,14 @@ module Embulk
|
|
|
86
145
|
def initialize(path)
|
|
87
146
|
@path = path
|
|
88
147
|
@modified_files = {}
|
|
148
|
+
|
|
149
|
+
require 'fileutils'
|
|
150
|
+
require 'embulk/data/package_data'
|
|
151
|
+
@new_data = PackageData.new("new", path)
|
|
89
152
|
end
|
|
90
153
|
|
|
91
154
|
attr_reader :path
|
|
155
|
+
attr_reader :new_data
|
|
92
156
|
|
|
93
157
|
def modified_files
|
|
94
158
|
@modified_files.keys
|
|
@@ -124,13 +188,13 @@ module Embulk
|
|
|
124
188
|
ms
|
|
125
189
|
end
|
|
126
190
|
|
|
127
|
-
def insert_line(glob, pattern, text
|
|
191
|
+
def insert_line(glob, pattern, text: nil)
|
|
128
192
|
ms = Dir[File.join(@path, glob)].map do |file|
|
|
129
193
|
data = read(file)
|
|
130
194
|
if m = data.match(pattern)
|
|
131
195
|
ln = m.pre_match.split("\n").count
|
|
132
196
|
replace = text || yield(m)
|
|
133
|
-
lines = data.split("\n")
|
|
197
|
+
lines = data.split("\n", -1) # preserve the last empty line
|
|
134
198
|
lines.insert(ln + 1, replace)
|
|
135
199
|
data = lines.join("\n")
|
|
136
200
|
modify(file, data)
|
|
@@ -146,7 +210,9 @@ module Embulk
|
|
|
146
210
|
end
|
|
147
211
|
|
|
148
212
|
def write(file, data)
|
|
149
|
-
|
|
213
|
+
dst = File.join(@path, file)
|
|
214
|
+
FileUtils.mkdir_p File.dirname(dst)
|
|
215
|
+
modify(dst, data)
|
|
150
216
|
end
|
|
151
217
|
|
|
152
218
|
private
|
|
@@ -87,6 +87,8 @@ module Embulk
|
|
|
87
87
|
pkg.cp("java/gradlew.bat", "gradlew.bat")
|
|
88
88
|
pkg.cp("java/gradlew", "gradlew")
|
|
89
89
|
pkg.set_executable("gradlew")
|
|
90
|
+
pkg.cp("java/config/checkstyle/checkstyle.xml","config/checkstyle/checkstyle.xml")
|
|
91
|
+
pkg.cp("java/config/checkstyle/default.xml","config/checkstyle/default.xml")
|
|
90
92
|
pkg.cp_erb("java/build.gradle.erb", "build.gradle")
|
|
91
93
|
pkg.cp_erb("java/plugin_loader.rb.erb", plugin_path)
|
|
92
94
|
pkg.cp_erb("java/#{category}.java.erb", "src/main/java/#{java_package_name.gsub(/\./, '/')}/#{java_class_name}.java")
|
|
@@ -73,7 +73,7 @@ module Embulk
|
|
|
73
73
|
|
|
74
74
|
case subcmd
|
|
75
75
|
when :run
|
|
76
|
-
op.banner = "Usage: run <config.yml>"
|
|
76
|
+
op.banner = "Usage: embulk run <config.yml>"
|
|
77
77
|
op.separator " Options:"
|
|
78
78
|
op.on('-r', '--resume-state PATH', 'Path to a file to write or read resume state') do |path|
|
|
79
79
|
options[:resume_state_path] = path
|
|
@@ -86,9 +86,9 @@ module Embulk
|
|
|
86
86
|
args = 1..1
|
|
87
87
|
|
|
88
88
|
when :cleanup
|
|
89
|
-
op.banner = "Usage: cleanup <config.yml>"
|
|
89
|
+
op.banner = "Usage: embulk cleanup <config.yml>"
|
|
90
90
|
op.separator " Options:"
|
|
91
|
-
op.on('-r', '--resume-state PATH', 'Path to a file to
|
|
91
|
+
op.on('-r', '--resume-state PATH', 'Path to a file to cleanup resume state') do |path|
|
|
92
92
|
options[:resume_state_path] = path
|
|
93
93
|
end
|
|
94
94
|
plugin_load_ops.call
|
|
@@ -96,7 +96,7 @@ module Embulk
|
|
|
96
96
|
args = 1..1
|
|
97
97
|
|
|
98
98
|
when :preview
|
|
99
|
-
op.banner = "Usage: preview <config.yml>"
|
|
99
|
+
op.banner = "Usage: embulk preview <config.yml>"
|
|
100
100
|
op.separator " Options:"
|
|
101
101
|
op.on('-G', '--vertical', "Use vertical output format", TrueClass) do |b|
|
|
102
102
|
options[:format] = "vertical"
|
|
@@ -106,7 +106,7 @@ module Embulk
|
|
|
106
106
|
args = 1..1
|
|
107
107
|
|
|
108
108
|
when :guess
|
|
109
|
-
op.banner = "Usage: guess <partial-config.yml>"
|
|
109
|
+
op.banner = "Usage: embulk guess <partial-config.yml>"
|
|
110
110
|
op.separator " Options:"
|
|
111
111
|
op.on('-o', '--output PATH', 'Path to a file to write the guessed configuration') do |path|
|
|
112
112
|
options[:next_config_output_path] = path
|
|
@@ -119,7 +119,7 @@ module Embulk
|
|
|
119
119
|
args = 1..1
|
|
120
120
|
|
|
121
121
|
when :mkbundle
|
|
122
|
-
op.banner = "Usage: mkbundle <directory> [--path PATH]"
|
|
122
|
+
op.banner = "Usage: embulk mkbundle <directory> [--path PATH]"
|
|
123
123
|
op.separator " Options:"
|
|
124
124
|
op.on('--path PATH', 'Relative path from <directory> for the location to install gems to (e.g. --path shared/bundle).') do |path|
|
|
125
125
|
options[:bundle_path] = path
|
|
@@ -156,7 +156,7 @@ module Embulk
|
|
|
156
156
|
system_exit_success
|
|
157
157
|
|
|
158
158
|
when :new
|
|
159
|
-
op.banner = "Usage: new <category> <name>" + %[
|
|
159
|
+
op.banner = "Usage: embulk new <category> <name>" + %[
|
|
160
160
|
categories:
|
|
161
161
|
ruby-input Ruby record input plugin (like "mysql")
|
|
162
162
|
ruby-output Ruby record output plugin (like "mysql")
|
|
@@ -184,7 +184,7 @@ examples:
|
|
|
184
184
|
args = 2..2
|
|
185
185
|
|
|
186
186
|
when :migrate
|
|
187
|
-
op.banner = "Usage: migrate <directory>"
|
|
187
|
+
op.banner = "Usage: embulk migrate <directory>"
|
|
188
188
|
args = 1..1
|
|
189
189
|
|
|
190
190
|
when :selfupdate
|
|
@@ -429,11 +429,12 @@ examples:
|
|
|
429
429
|
|
|
430
430
|
def self.usage(message)
|
|
431
431
|
STDERR.puts "Embulk v#{Embulk::VERSION}"
|
|
432
|
-
STDERR.puts "
|
|
433
|
-
STDERR.puts "
|
|
432
|
+
STDERR.puts "Usage: embulk [-vm-options] <command> [--options]"
|
|
433
|
+
STDERR.puts "Commands:"
|
|
434
434
|
STDERR.puts " mkbundle <directory> # create a new plugin bundle environment."
|
|
435
435
|
STDERR.puts " bundle [directory] # update a plugin bundle environment."
|
|
436
436
|
STDERR.puts " run <config.yml> # run a bulk load transaction."
|
|
437
|
+
STDERR.puts " cleanup <config.yml> # cleanup resume state."
|
|
437
438
|
STDERR.puts " preview <config.yml> # dry-run the bulk load without output and show preview."
|
|
438
439
|
STDERR.puts " guess <partial-config.yml> -o <output.yml> # guess missing parameters to create a complete configuration file."
|
|
439
440
|
STDERR.puts " gem <install | list | help> # install a plugin or show installed plugins."
|
|
@@ -443,6 +444,12 @@ examples:
|
|
|
443
444
|
STDERR.puts " example [path] # creates an example config file and csv file to try embulk."
|
|
444
445
|
STDERR.puts " selfupdate [version] # upgrades embulk to the latest released version or to the specified version."
|
|
445
446
|
STDERR.puts ""
|
|
447
|
+
STDERR.puts "VM options:"
|
|
448
|
+
STDERR.puts " -J-O Disable JVM optimizations to speed up startup time (enabled by default if command is 'run')"
|
|
449
|
+
STDERR.puts " -J+O Enable JVM optimizations to speed up throughput"
|
|
450
|
+
STDERR.puts " -J... Set JVM options (use -J-help to see available options)"
|
|
451
|
+
STDERR.puts " -R... Set JRuby options (use -R--help to see available options)"
|
|
452
|
+
STDERR.puts ""
|
|
446
453
|
if message
|
|
447
454
|
system_exit "error: #{message}"
|
|
448
455
|
else
|
|
@@ -1 +1 @@
|
|
|
1
|
-
jruby-9.0.
|
|
1
|
+
jruby-9.0.4.0
|
|
@@ -2,6 +2,7 @@ plugins {
|
|
|
2
2
|
id "com.jfrog.bintray" version "1.1"
|
|
3
3
|
id "com.github.jruby-gradle.base" version "0.1.5"
|
|
4
4
|
id "java"
|
|
5
|
+
id "checkstyle"
|
|
5
6
|
}
|
|
6
7
|
import com.github.jrubygradle.JRubyExec
|
|
7
8
|
repositories {
|
|
@@ -14,6 +15,9 @@ configurations {
|
|
|
14
15
|
|
|
15
16
|
version = "0.1.0"
|
|
16
17
|
|
|
18
|
+
sourceCompatibility = 1.7
|
|
19
|
+
targetCompatibility = 1.7
|
|
20
|
+
|
|
17
21
|
dependencies {
|
|
18
22
|
compile "org.embulk:embulk-core:<%= Embulk::VERSION %>"
|
|
19
23
|
provided "org.embulk:embulk-core:<%= Embulk::VERSION %>"
|
|
@@ -28,6 +32,23 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
|
28
32
|
}
|
|
29
33
|
clean { delete "classpath" }
|
|
30
34
|
|
|
35
|
+
checkstyle {
|
|
36
|
+
configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
|
|
37
|
+
toolVersion = '6.14.1'
|
|
38
|
+
}
|
|
39
|
+
checkstyleMain {
|
|
40
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
41
|
+
ignoreFailures = true
|
|
42
|
+
}
|
|
43
|
+
checkstyleTest {
|
|
44
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
45
|
+
ignoreFailures = true
|
|
46
|
+
}
|
|
47
|
+
task checkstyle(type: Checkstyle) {
|
|
48
|
+
classpath = sourceSets.main.output + sourceSets.test.output
|
|
49
|
+
source = sourceSets.main.allJava + sourceSets.test.allJava
|
|
50
|
+
}
|
|
51
|
+
|
|
31
52
|
task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
|
|
32
53
|
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
|
|
33
54
|
script "${project.name}.gemspec"
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<module name="Checker">
|
|
6
|
+
<!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
|
|
7
|
+
<module name="FileTabCharacter"/>
|
|
8
|
+
<module name="NewlineAtEndOfFile">
|
|
9
|
+
<property name="lineSeparator" value="lf"/>
|
|
10
|
+
</module>
|
|
11
|
+
<module name="RegexpMultiline">
|
|
12
|
+
<property name="format" value="\r"/>
|
|
13
|
+
<property name="message" value="Line contains carriage return"/>
|
|
14
|
+
</module>
|
|
15
|
+
<module name="RegexpMultiline">
|
|
16
|
+
<property name="format" value=" \n"/>
|
|
17
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
18
|
+
</module>
|
|
19
|
+
<module name="RegexpMultiline">
|
|
20
|
+
<property name="format" value="\{\n\n"/>
|
|
21
|
+
<property name="message" value="Blank line after opening brace"/>
|
|
22
|
+
</module>
|
|
23
|
+
<module name="RegexpMultiline">
|
|
24
|
+
<property name="format" value="\n\n\s*\}"/>
|
|
25
|
+
<property name="message" value="Blank line before closing brace"/>
|
|
26
|
+
</module>
|
|
27
|
+
<module name="RegexpMultiline">
|
|
28
|
+
<property name="format" value="\n\n\n"/>
|
|
29
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
30
|
+
</module>
|
|
31
|
+
<module name="RegexpMultiline">
|
|
32
|
+
<property name="format" value="\n\n\Z"/>
|
|
33
|
+
<property name="message" value="Blank line before end of file"/>
|
|
34
|
+
</module>
|
|
35
|
+
<module name="RegexpMultiline">
|
|
36
|
+
<property name="format" value="Preconditions\.checkNotNull"/>
|
|
37
|
+
<property name="message" value="Use of checkNotNull"/>
|
|
38
|
+
</module>
|
|
39
|
+
|
|
40
|
+
<module name="TreeWalker">
|
|
41
|
+
<module name="EmptyBlock">
|
|
42
|
+
<property name="option" value="text"/>
|
|
43
|
+
<property name="tokens" value="
|
|
44
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
45
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
46
|
+
</module>
|
|
47
|
+
<module name="EmptyStatement"/>
|
|
48
|
+
<module name="EmptyForInitializerPad"/>
|
|
49
|
+
<module name="EmptyForIteratorPad">
|
|
50
|
+
<property name="option" value="space"/>
|
|
51
|
+
</module>
|
|
52
|
+
<module name="MethodParamPad">
|
|
53
|
+
<property name="allowLineBreaks" value="true"/>
|
|
54
|
+
<property name="option" value="nospace"/>
|
|
55
|
+
</module>
|
|
56
|
+
<module name="ParenPad"/>
|
|
57
|
+
<module name="TypecastParenPad"/>
|
|
58
|
+
<module name="NeedBraces"/>
|
|
59
|
+
<module name="LeftCurly">
|
|
60
|
+
<property name="option" value="nl"/>
|
|
61
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
62
|
+
</module>
|
|
63
|
+
<module name="LeftCurly">
|
|
64
|
+
<property name="option" value="eol"/>
|
|
65
|
+
<property name="tokens" value="
|
|
66
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
67
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
68
|
+
</module>
|
|
69
|
+
<module name="RightCurly">
|
|
70
|
+
<property name="option" value="alone"/>
|
|
71
|
+
</module>
|
|
72
|
+
<module name="GenericWhitespace"/>
|
|
73
|
+
<module name="WhitespaceAfter"/>
|
|
74
|
+
<module name="NoWhitespaceBefore"/>
|
|
75
|
+
|
|
76
|
+
<module name="UpperEll"/>
|
|
77
|
+
<module name="DefaultComesLast"/>
|
|
78
|
+
<module name="ArrayTypeStyle"/>
|
|
79
|
+
<module name="MultipleVariableDeclarations"/>
|
|
80
|
+
<module name="ModifierOrder"/>
|
|
81
|
+
<module name="OneStatementPerLine"/>
|
|
82
|
+
<module name="StringLiteralEquality"/>
|
|
83
|
+
<module name="MutableException"/>
|
|
84
|
+
<module name="EqualsHashCode"/>
|
|
85
|
+
<module name="InnerAssignment"/>
|
|
86
|
+
<module name="InterfaceIsType"/>
|
|
87
|
+
<module name="HideUtilityClassConstructor"/>
|
|
88
|
+
|
|
89
|
+
<module name="MemberName"/>
|
|
90
|
+
<module name="LocalVariableName"/>
|
|
91
|
+
<module name="LocalFinalVariableName"/>
|
|
92
|
+
<module name="TypeName"/>
|
|
93
|
+
<module name="PackageName"/>
|
|
94
|
+
<module name="ParameterName"/>
|
|
95
|
+
<module name="StaticVariableName"/>
|
|
96
|
+
<module name="ClassTypeParameterName">
|
|
97
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
98
|
+
</module>
|
|
99
|
+
<module name="MethodTypeParameterName">
|
|
100
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
101
|
+
</module>
|
|
102
|
+
|
|
103
|
+
<module name="AvoidStarImport"/>
|
|
104
|
+
<module name="RedundantImport"/>
|
|
105
|
+
<module name="UnusedImports"/>
|
|
106
|
+
<module name="ImportOrder">
|
|
107
|
+
<property name="groups" value="*,javax,java"/>
|
|
108
|
+
<property name="separated" value="true"/>
|
|
109
|
+
<property name="option" value="bottom"/>
|
|
110
|
+
<property name="sortStaticImportsAlphabetically" value="true"/>
|
|
111
|
+
</module>
|
|
112
|
+
|
|
113
|
+
<module name="WhitespaceAround">
|
|
114
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
115
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
116
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
117
|
+
<property name="tokens" value="
|
|
118
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
119
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
120
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
121
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
122
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
123
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
124
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
125
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
126
|
+
</module>
|
|
127
|
+
</module>
|
|
128
|
+
</module>
|