embulk 0.7.7-java → 0.7.8-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +5 -5
- data/bin/embulk +7 -3
- data/build.gradle +1 -1
- data/embulk-cli/src/main/java/org/embulk/cli/Main.java +1 -1
- data/embulk-core/build.gradle +1 -0
- data/embulk-core/src/main/java/org/embulk/spi/AbortTransactionResource.java +36 -0
- data/embulk-core/src/main/java/org/embulk/spi/CloseResource.java +42 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java +9 -15
- data/embulk-core/src/main/java/org/embulk/spi/FileOutputRunner.java +17 -25
- data/embulk-core/src/main/java/org/embulk/spi/util/Executors.java +21 -23
- data/embulk-docs/src/release.rst +1 -0
- data/embulk-docs/src/release/release-0.7.8.rst +14 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java +1 -1
- data/embulk.gemspec +1 -1
- data/lib/embulk.rb +2 -2
- data/lib/embulk/command/embulk_bundle.rb +6 -10
- data/lib/embulk/command/embulk_generate_bin.rb +1 -1
- data/lib/embulk/command/embulk_run.rb +34 -44
- data/lib/embulk/data/bundle/Gemfile +16 -11
- data/lib/embulk/version.rb +1 -1
- metadata +19 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bbf761efe988b3d306299a56b251209e2d30d473
|
4
|
+
data.tar.gz: 2ffff3b608e07887cfa5988513b30e65bc4efb4c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d9d4ca224cdff843099f630f430995737b45b6c53b618e6b05d7f677648fe61ac1b849774884cc42afbf064c62d50838870e37e6c78b5e7ab4793da10d245606
|
7
|
+
data.tar.gz: 3f02b050a1d3552663563b1460898c2b2e969cbc5d905f771d95ffd9d77eec740c1f64b8a36eefc284aa491076fdb0595d8c3def0ba96a3e1bbe14c575467f3f
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -104,15 +104,15 @@ embulk cleanup config.yml -r resume-state.yml
|
|
104
104
|
|
105
105
|
### Using plugin bundle
|
106
106
|
|
107
|
-
`embulk
|
108
|
-
|
107
|
+
`embulk mkbundle` subcommand creates a isolated bundle of plugins. You can install plugins (gems) to the bundle directory instead of ~/.embulk directory. This makes it easy to manage versions of plugins.
|
108
|
+
To use the bundle, add `-b <bundle_dir>` option to `guess`, `preview`, or `run` subcommand. `embulk mkbundle` also generates some example plugins to \<bundle_dir>/embulk/\*.rb directory.
|
109
109
|
|
110
110
|
See the generated \<bundle_dir>/Gemfile file how to plugin bundles work.
|
111
111
|
|
112
112
|
```
|
113
|
-
embulk
|
114
|
-
embulk guess
|
115
|
-
embulk run
|
113
|
+
embulk mkbundle ./embulk_bundle
|
114
|
+
embulk guess -b ./embulk_bundle ...
|
115
|
+
embulk run -b ./embulk_bundle ...
|
116
116
|
```
|
117
117
|
|
118
118
|
## Use cases
|
data/bin/embulk
CHANGED
@@ -1,12 +1,14 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
3
|
if RUBY_PLATFORM =~ /java/i
|
4
|
-
# Enable embulk_bundle if run by CRuby.
|
5
|
-
# Disable embulk_bundle if run by JRuby.
|
6
4
|
if ENV['EMBULK_BIN_ENABLE_BUNDLE'] == File.expand_path(__FILE__)
|
5
|
+
# bin/embulk is run by CRuby (embulk gem for CRuby is installed). enable embulk_bundle.
|
7
6
|
ENV.delete('EMBULK_BIN_ENABLE_BUNDLE')
|
7
|
+
# include -cp CLASSPATH to LOAD_PATH so that embulk_bundle.rb can load bundler included in embulk-core.jar
|
8
|
+
$LOAD_PATH << "uri:classloader:/"
|
8
9
|
require_relative '../lib/embulk/command/embulk_bundle'
|
9
10
|
else
|
11
|
+
# bin/embulk is run by JRuby (embulk gem for JRuby is installed). disable embulk_bundle not to bother the JRuby's bundler
|
10
12
|
$LOAD_PATH << File.expand_path('../lib', File.dirname(__FILE__))
|
11
13
|
require 'embulk/command/embulk_main'
|
12
14
|
end
|
@@ -93,11 +95,13 @@ rescue LoadError => e
|
|
93
95
|
raise e
|
94
96
|
end
|
95
97
|
jruby_cp = "#{File.dirname(JRubyJars.core_jar_path)}/*"
|
98
|
+
embulk_cp = "#{File.expand_path('../../classpath', __FILE__)}/*" # bundler is included in embulk-core.jar
|
96
99
|
|
97
100
|
# java ... -jar ruby-complete.jar bin/embulk "$@"
|
98
101
|
cmdline = [java_cmd]
|
99
102
|
cmdline.concat java_args
|
100
|
-
cmdline << '-cp' << jruby_cp
|
103
|
+
cmdline << '-cp' << [jruby_cp, embulk_cp].join(File::PATH_SEPARATOR)
|
104
|
+
cmdline << 'org.jruby.Main'
|
101
105
|
cmdline.concat jruby_args
|
102
106
|
cmdline << __FILE__
|
103
107
|
cmdline.concat ARGV
|
data/build.gradle
CHANGED
@@ -30,7 +30,7 @@ public class Main
|
|
30
30
|
resourcePath = Main.class.getProtectionDomain().getCodeSource().getLocation().toURI().toString() + "!/";
|
31
31
|
}
|
32
32
|
catch (URISyntaxException ex) {
|
33
|
-
resourcePath = "
|
33
|
+
resourcePath = "uri:classloader:/";
|
34
34
|
}
|
35
35
|
return resourcePath + "embulk/command/embulk_bundle.rb";
|
36
36
|
}
|
data/embulk-core/build.gradle
CHANGED
@@ -0,0 +1,36 @@
|
|
1
|
+
package org.embulk.spi;
|
2
|
+
|
3
|
+
public class AbortTransactionResource
|
4
|
+
implements AutoCloseable
|
5
|
+
{
|
6
|
+
private Transactional tran;
|
7
|
+
|
8
|
+
public AbortTransactionResource()
|
9
|
+
{
|
10
|
+
this(null);
|
11
|
+
}
|
12
|
+
|
13
|
+
public AbortTransactionResource(Transactional tran)
|
14
|
+
{
|
15
|
+
this.tran = tran;
|
16
|
+
}
|
17
|
+
|
18
|
+
public void abortThis(Transactional tran)
|
19
|
+
{
|
20
|
+
this.tran = tran;
|
21
|
+
}
|
22
|
+
|
23
|
+
public void dontAbort()
|
24
|
+
{
|
25
|
+
this.tran = null;
|
26
|
+
}
|
27
|
+
|
28
|
+
@Override
|
29
|
+
public void close()
|
30
|
+
{
|
31
|
+
if (tran != null) {
|
32
|
+
tran.abort();
|
33
|
+
}
|
34
|
+
}
|
35
|
+
}
|
36
|
+
|
@@ -0,0 +1,42 @@
|
|
1
|
+
package org.embulk.spi;
|
2
|
+
|
3
|
+
import com.google.common.base.Throwables;
|
4
|
+
|
5
|
+
public class CloseResource
|
6
|
+
implements AutoCloseable
|
7
|
+
{
|
8
|
+
private AutoCloseable resource;
|
9
|
+
|
10
|
+
public CloseResource()
|
11
|
+
{
|
12
|
+
this(null);
|
13
|
+
}
|
14
|
+
|
15
|
+
public CloseResource(AutoCloseable resource)
|
16
|
+
{
|
17
|
+
this.resource = resource;
|
18
|
+
}
|
19
|
+
|
20
|
+
public void closeThis(AutoCloseable resource)
|
21
|
+
{
|
22
|
+
this.resource = resource;
|
23
|
+
}
|
24
|
+
|
25
|
+
public void dontClose()
|
26
|
+
{
|
27
|
+
this.resource = null;
|
28
|
+
}
|
29
|
+
|
30
|
+
public void close()
|
31
|
+
{
|
32
|
+
if (resource != null) {
|
33
|
+
try {
|
34
|
+
resource.close();
|
35
|
+
}
|
36
|
+
catch (Exception ex) {
|
37
|
+
throw Throwables.propagate(ex);
|
38
|
+
}
|
39
|
+
}
|
40
|
+
}
|
41
|
+
}
|
42
|
+
|
@@ -147,21 +147,15 @@ public class FileInputRunner
|
|
147
147
|
|
148
148
|
TransactionalFileInput tran = PluginWrappers.transactionalFileInput(
|
149
149
|
fileInputPlugin.open(task.getFileInputTaskSource(), taskIndex));
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
try {
|
160
|
-
if (tran != null) {
|
161
|
-
tran.abort();
|
162
|
-
}
|
163
|
-
} finally {
|
164
|
-
fileInput.close();
|
150
|
+
try (CloseResource closer = new CloseResource(tran)) {
|
151
|
+
try (AbortTransactionResource aborter = new AbortTransactionResource(tran)) {
|
152
|
+
FileInput fileInput = Decoders.open(decoderPlugins, task.getDecoderTaskSources(), tran);
|
153
|
+
closer.closeThis(fileInput);
|
154
|
+
parserPlugin.run(task.getParserTaskSource(), schema, fileInput, output);
|
155
|
+
|
156
|
+
TaskReport report = tran.commit(); // TODO check output.finish() is called. wrap
|
157
|
+
aborter.dontAbort();
|
158
|
+
return report;
|
165
159
|
}
|
166
160
|
}
|
167
161
|
}
|
@@ -127,31 +127,23 @@ public class FileOutputRunner
|
|
127
127
|
List<EncoderPlugin> encoderPlugins = newEncoderPlugins(task);
|
128
128
|
FormatterPlugin formatterPlugin = newFormatterPlugin(task);
|
129
129
|
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
if (output != null) {
|
148
|
-
output.close();
|
149
|
-
}
|
150
|
-
if (fileOutput != null) {
|
151
|
-
fileOutput.close();
|
152
|
-
}
|
153
|
-
if (tran != null) {
|
154
|
-
tran.abort();
|
130
|
+
try (AbortTransactionResource aborter = new AbortTransactionResource()) {
|
131
|
+
try (CloseResource closer = new CloseResource()) {
|
132
|
+
TransactionalFileOutput finalOutput = PluginWrappers.transactionalFileOutput(
|
133
|
+
fileOutputPlugin.open(task.getFileOutputTaskSource(), taskIndex));
|
134
|
+
aborter.abortThis(finalOutput);
|
135
|
+
closer.closeThis(finalOutput);
|
136
|
+
|
137
|
+
FileOutput encodedOutput = Encoders.open(encoderPlugins, task.getEncoderTaskSources(), finalOutput);
|
138
|
+
closer.closeThis(encodedOutput);
|
139
|
+
|
140
|
+
PageOutput output = formatterPlugin.open(task.getFormatterTaskSource(), schema, encodedOutput);
|
141
|
+
closer.closeThis(output);
|
142
|
+
|
143
|
+
TransactionalPageOutput ret = new DelegateTransactionalPageOutput(finalOutput, output);
|
144
|
+
aborter.dontAbort();
|
145
|
+
closer.dontClose(); // ownership of output is transferred to caller (input plugin). the owner will close output.
|
146
|
+
return ret;
|
155
147
|
}
|
156
148
|
}
|
157
149
|
}
|
@@ -12,6 +12,8 @@ import org.embulk.spi.InputPlugin;
|
|
12
12
|
import org.embulk.spi.FilterPlugin;
|
13
13
|
import org.embulk.spi.OutputPlugin;
|
14
14
|
import org.embulk.spi.ProcessTask;
|
15
|
+
import org.embulk.spi.AbortTransactionResource;
|
16
|
+
import org.embulk.spi.CloseResource;
|
15
17
|
import org.embulk.plugin.compat.PluginWrappers;
|
16
18
|
|
17
19
|
public abstract class Executors
|
@@ -53,32 +55,28 @@ public abstract class Executors
|
|
53
55
|
TransactionalPageOutput tran = PluginWrappers.transactionalPageOutput(
|
54
56
|
outputPlugin.open(outputTaskSource, outputSchema, taskIndex));
|
55
57
|
|
56
|
-
PageOutput closeThis = tran;
|
57
58
|
callback.started();
|
58
|
-
try
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
}
|
73
|
-
callback.outputCommitted(outputTaskReport); // TODO check output.finish() is called. wrap or abstract
|
59
|
+
// here needs to use try-with-resource to add exception happend at close() or abort()
|
60
|
+
// to suppressed exception. otherwise exception happend at close() or abort() overwrites
|
61
|
+
// essential exception.
|
62
|
+
try (CloseResource closer = new CloseResource(tran)) {
|
63
|
+
try (AbortTransactionResource aborter = new AbortTransactionResource(tran)) {
|
64
|
+
PageOutput filtered = Filters.open(filterPlugins, filterTaskSources, filterSchemas, tran);
|
65
|
+
closer.closeThis(filtered);
|
66
|
+
|
67
|
+
TaskReport inputTaskReport = inputPlugin.run(inputTaskSource, inputSchema, taskIndex, filtered);
|
68
|
+
|
69
|
+
if (inputTaskReport == null) {
|
70
|
+
inputTaskReport = exec.newTaskReport();
|
71
|
+
}
|
72
|
+
callback.inputCommitted(inputTaskReport);
|
74
73
|
|
75
|
-
|
76
|
-
|
77
|
-
if (
|
78
|
-
|
74
|
+
TaskReport outputTaskReport = tran.commit();
|
75
|
+
aborter.dontAbort();
|
76
|
+
if (outputTaskReport == null) {
|
77
|
+
outputTaskReport = exec.newTaskReport();
|
79
78
|
}
|
80
|
-
|
81
|
-
closeThis.close();
|
79
|
+
callback.outputCommitted(outputTaskReport); // TODO check output.finish() is called. wrap or abstract
|
82
80
|
}
|
83
81
|
}
|
84
82
|
}
|
data/embulk-docs/src/release.rst
CHANGED
@@ -0,0 +1,14 @@
|
|
1
|
+
Release 0.7.8
|
2
|
+
==================================
|
3
|
+
|
4
|
+
General Changes
|
5
|
+
------------------
|
6
|
+
|
7
|
+
* Added "mkbundle" subcommand. Its help messae shows how to use plugin blundes with examples.
|
8
|
+
* When a transaction throws an exception and cleanup code (close() method) also throws an exception, the exception of transaction was hidden. Now the exception of transaction is the primary error and exception of cleanup code is also included as a suppressed exception.
|
9
|
+
* CSV parser plugin doesn't show lines twice with "Unexpected extra character" error.
|
10
|
+
|
11
|
+
|
12
|
+
Release Date
|
13
|
+
------------------
|
14
|
+
2015-11-08
|
@@ -314,7 +314,7 @@ public class CsvTokenizer
|
|
314
314
|
// column has trailing spaces and quoted. TODO should this be rejected?
|
315
315
|
|
316
316
|
} else {
|
317
|
-
throw new InvalidValueException(String.format("Unexpected extra character '%c' after
|
317
|
+
throw new InvalidValueException(String.format("Unexpected extra character '%c' after a value quoted by '%c'", c, quote));
|
318
318
|
}
|
319
319
|
break;
|
320
320
|
|
data/embulk.gemspec
CHANGED
@@ -20,6 +20,7 @@ Gem::Specification.new do |gem|
|
|
20
20
|
gem.has_rdoc = false
|
21
21
|
|
22
22
|
if RUBY_PLATFORM =~ /java/i
|
23
|
+
gem.add_dependency "bundler", '~> 1.10.6'
|
23
24
|
gem.add_dependency "liquid", '~> 3.0.6'
|
24
25
|
|
25
26
|
# For embulk/guess/charset.rb. See also embulk-core/build.gradle
|
@@ -31,7 +32,6 @@ Gem::Specification.new do |gem|
|
|
31
32
|
gem.add_dependency "jruby-jars", '= 9.0.0.0'
|
32
33
|
end
|
33
34
|
|
34
|
-
gem.add_development_dependency "bundler", [">= 1.0"]
|
35
35
|
gem.add_development_dependency "rake", [">= 0.10.0"]
|
36
36
|
gem.add_development_dependency "test-unit", ["~> 3.0.9"]
|
37
37
|
gem.add_development_dependency "yard", ["~> 0.8.7"]
|
data/lib/embulk.rb
CHANGED
@@ -8,7 +8,7 @@ module Embulk
|
|
8
8
|
if resource
|
9
9
|
lib = resource.split("/")[0..-2].join("/")
|
10
10
|
"#{jar}!#{lib}/#{path}"
|
11
|
-
elsif __FILE__ =~ /^classpath:/
|
11
|
+
elsif __FILE__ =~ /^(?:classpath|uri:classloader):/
|
12
12
|
lib = __FILE__.split("/")[0..-2].join("/")
|
13
13
|
"#{lib}/#{path}"
|
14
14
|
else
|
@@ -22,7 +22,7 @@ module Embulk
|
|
22
22
|
# single jar. __FILE__ should point path/to/embulk.jar!/embulk.rb
|
23
23
|
# which means that embulk.jar is already loaded in this JVM.
|
24
24
|
|
25
|
-
elsif __FILE__ =~ /^classpath:/
|
25
|
+
elsif __FILE__ =~ /^(?:classpath|uri:classloader):/
|
26
26
|
# already in classpath
|
27
27
|
|
28
28
|
else
|
@@ -9,19 +9,15 @@ if bundle_path_index
|
|
9
9
|
end
|
10
10
|
|
11
11
|
if bundle_path
|
12
|
-
# use bundler installed at bundle_path
|
13
12
|
ENV['EMBULK_BUNDLE_PATH'] = bundle_path
|
14
|
-
ENV['GEM_HOME'] = File.expand_path File.join(bundle_path, Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
|
15
|
-
ENV['GEM_PATH'] = ''
|
16
|
-
Gem.clear_paths # force rubygems to reload GEM_HOME
|
17
|
-
|
18
13
|
ENV['BUNDLE_GEMFILE'] = File.expand_path File.join(bundle_path, "Gemfile")
|
19
14
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
15
|
+
# bundler is included in embulk-core.jar
|
16
|
+
ENV.delete('GEM_HOME')
|
17
|
+
ENV.delete('GEM_PATH')
|
18
|
+
Gem.clear_paths
|
19
|
+
require 'bundler'
|
20
|
+
|
25
21
|
Bundler.load.setup_environment
|
26
22
|
require 'bundler/setup'
|
27
23
|
# since here, `require` may load files of different (newer) embulk versions
|
@@ -1,7 +1,7 @@
|
|
1
1
|
module Embulk
|
2
2
|
def self.generate_bin(options={})
|
3
3
|
jruby_jar_path = org.jruby.Main.java_class.protection_domain.code_source.location.to_s
|
4
|
-
if __FILE__ =~ /^classpath:/ || __FILE__.include?('!/')
|
4
|
+
if __FILE__ =~ /^(?:classpath|uri:classloader):/ || __FILE__.include?('!/')
|
5
5
|
resource_class = org.embulk.command.Runner.java_class
|
6
6
|
ruby_script_path = resource_class.resource("/embulk/command/embulk.rb").to_s
|
7
7
|
else
|
@@ -66,7 +66,7 @@ module Embulk
|
|
66
66
|
op.on('-C', '--classpath PATH', "Add java classpath separated by #{classpath_separator} (CLASSPATH)") do |classpath|
|
67
67
|
classpaths.concat classpath.split(classpath_separator)
|
68
68
|
end
|
69
|
-
op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory (create one using "embulk
|
69
|
+
op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory (create one using "embulk mkbundle" command)') do |path|
|
70
70
|
# only for help message. implemented at lib/embulk/command/embulk_bundle.rb
|
71
71
|
end
|
72
72
|
end
|
@@ -118,21 +118,34 @@ module Embulk
|
|
118
118
|
java_embed_ops.call
|
119
119
|
args = 1..1
|
120
120
|
|
121
|
+
when :mkbundle
|
122
|
+
op.banner = "Usage: mkbundle <directory> [--path PATH]"
|
123
|
+
op.separator " Options:"
|
124
|
+
op.on('--path PATH', 'Relative path from <directory> for the location to install gems to (e.g. --path shared/bundle).') do |path|
|
125
|
+
options[:bundle_path] = path
|
126
|
+
end
|
127
|
+
op.separator <<-EOF
|
128
|
+
|
129
|
+
"mkbundle" creates a new a plugin bundle directory. You can install
|
130
|
+
plugins (gems) to the directory instead of ~/.embulk.
|
131
|
+
|
132
|
+
See generated <directory>/Gemfile to install plugins to the directory.
|
133
|
+
Use -b, --bundle BUNDLE_DIR option to use it:
|
134
|
+
|
135
|
+
$ embulk mkbundle ./dir # create bundle directory
|
136
|
+
$ (cd dir && vi Gemfile && embulk bundle) # update plugin list
|
137
|
+
$ embulk guess -b ./dir ... # guess using bundled plugins
|
138
|
+
$ embulk run -b ./dir ... # run using bundled plugins
|
139
|
+
|
140
|
+
EOF
|
141
|
+
args = 1..1
|
142
|
+
|
121
143
|
when :bundle
|
122
144
|
if argv[0] == 'new'
|
123
145
|
usage nil if argv.length != 2
|
124
|
-
new_bundle(argv[1])
|
146
|
+
new_bundle(argv[1], nil)
|
147
|
+
STDERR.puts "'embulk bundle new' is deprecated. This will be removed in future release. Please use 'embulk mkbundle' instead."
|
125
148
|
else
|
126
|
-
gemfile_path = ENV['BUNDLE_GEMFILE'].to_s
|
127
|
-
|
128
|
-
if !gemfile_path.empty?
|
129
|
-
bundle_path = File.dirname(gemfile_path)
|
130
|
-
elsif File.exists?('Gemfile')
|
131
|
-
bundle_path = '.'
|
132
|
-
else
|
133
|
-
system_exit "'#{gemfile_path}' already exists. You already ran 'embulk bundle new'. Please remove it, or run \"cd #{gemfile_path}\" and \"embulk bundle\" instead"
|
134
|
-
end
|
135
|
-
|
136
149
|
run_bundler(argv)
|
137
150
|
end
|
138
151
|
system_exit_success
|
@@ -259,6 +272,9 @@ examples:
|
|
259
272
|
options[:version] = ARGV[0]
|
260
273
|
Embulk.selfupdate(options)
|
261
274
|
|
275
|
+
when :mkbundle
|
276
|
+
new_bundle(argv[0], options[:bundle_path])
|
277
|
+
|
262
278
|
else
|
263
279
|
require 'json'
|
264
280
|
|
@@ -355,12 +371,12 @@ examples:
|
|
355
371
|
end
|
356
372
|
end
|
357
373
|
|
358
|
-
def self.new_bundle(path)
|
374
|
+
def self.new_bundle(path, bundle_path)
|
359
375
|
require 'fileutils'
|
360
376
|
require 'rubygems/gem_runner'
|
361
377
|
|
362
378
|
if File.exists?(path)
|
363
|
-
error = "'#{path}' already exists.
|
379
|
+
error = "'#{path}' already exists."
|
364
380
|
STDERR.puts error
|
365
381
|
raise SystemExit.new(1, error)
|
366
382
|
end
|
@@ -379,7 +395,7 @@ examples:
|
|
379
395
|
|
380
396
|
# run the first bundle-install
|
381
397
|
Dir.chdir(path) do
|
382
|
-
run_bundler(['install', '--path',
|
398
|
+
run_bundler(['install', '--path', bundle_path || "."])
|
383
399
|
end
|
384
400
|
|
385
401
|
success = true
|
@@ -392,34 +408,7 @@ examples:
|
|
392
408
|
end
|
393
409
|
|
394
410
|
def self.run_bundler(argv)
|
395
|
-
#
|
396
|
-
begin
|
397
|
-
require 'bundler'
|
398
|
-
bundler_provided = true
|
399
|
-
rescue LoadError
|
400
|
-
ENV['GEM_HOME'] = File.expand_path File.join(".", Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
|
401
|
-
ENV['GEM_PATH'] = ''
|
402
|
-
ENV.delete('BUNDLE_GEMFILE')
|
403
|
-
Gem.clear_paths # force rubygems to reload GEM_HOME
|
404
|
-
begin
|
405
|
-
require 'bundler'
|
406
|
-
bundler_provided = true
|
407
|
-
rescue LoadError
|
408
|
-
end
|
409
|
-
end
|
410
|
-
|
411
|
-
unless bundler_provided
|
412
|
-
# install bundler to the path (bundler is not included in embulk-core.jar)
|
413
|
-
require 'rubygems/gem_runner'
|
414
|
-
begin
|
415
|
-
puts "Installing bundler..."
|
416
|
-
Gem::GemRunner.new.run %w[install bundler]
|
417
|
-
rescue Gem::SystemExitException => e
|
418
|
-
raise e if e.exit_code != 0
|
419
|
-
end
|
420
|
-
Gem.clear_paths
|
421
|
-
require 'bundler'
|
422
|
-
end
|
411
|
+
require 'bundler' # bundler is included in embulk-core.jar
|
423
412
|
|
424
413
|
require 'bundler/friendly_errors'
|
425
414
|
require 'bundler/cli'
|
@@ -432,7 +421,8 @@ examples:
|
|
432
421
|
STDERR.puts "Embulk v#{Embulk::VERSION}"
|
433
422
|
STDERR.puts "usage: <command> [--options]"
|
434
423
|
STDERR.puts "commands:"
|
435
|
-
STDERR.puts "
|
424
|
+
STDERR.puts " mkbundle <directory> # create a new plugin bundle environment."
|
425
|
+
STDERR.puts " bundle [directory] # update a plugin bundle environment."
|
436
426
|
STDERR.puts " run <config.yml> # run a bulk load transaction."
|
437
427
|
STDERR.puts " preview <config.yml> # dry-run the bulk load without output and show preview."
|
438
428
|
STDERR.puts " guess <partial-config.yml> -o <output.yml> # guess missing parameters to create a complete configuration file."
|
@@ -2,25 +2,30 @@ source 'https://rubygems.org/'
|
|
2
2
|
gem 'embulk', '~> 0.7.0'
|
3
3
|
|
4
4
|
#
|
5
|
-
#
|
5
|
+
# 1. Use following syntax to specify versions of plugins
|
6
|
+
# to install this bundle directory:
|
6
7
|
#
|
7
|
-
#
|
8
|
-
#
|
8
|
+
#gem 'embulk-output-mysql' # the latest version
|
9
|
+
#gem 'embulk-input-baz', '= 0.2.0' # specific version
|
10
|
+
#gem 'embulk-input-xyz', '~> 0.3.0' # latest major version
|
11
|
+
#gem 'embulk-output-postgresql', '>= 0.1.0' # newer than specific version
|
12
|
+
#
|
13
|
+
#gem 'embulk-output-awesome', git: 'https://github.com/you/embulk-output-awesome.git', branch: 'master'
|
9
14
|
#
|
10
15
|
|
11
16
|
#
|
12
|
-
#
|
17
|
+
# 2. When you modify this file, run following command to
|
18
|
+
# install plugins:
|
13
19
|
#
|
14
|
-
#
|
15
|
-
#
|
16
|
-
#gem 'embulk-input-baz', '= 0.2.0'
|
17
|
-
#gem 'embulk-input-xyz', '~> 0.3.0'
|
20
|
+
# $ cd this_directory
|
21
|
+
# $ embulk bundle
|
18
22
|
#
|
19
23
|
|
20
24
|
#
|
21
|
-
#
|
25
|
+
# 3. Then you can use plugins with -b, --bundle BUNDLE_PATH command:
|
22
26
|
#
|
23
|
-
#
|
24
|
-
#
|
27
|
+
# $ embulk guess -b path/to/this/directory ...
|
28
|
+
# $ embulk run -b path/to/this/directory ...
|
29
|
+
# $ embulk preview -b path/to/this/directory ...
|
25
30
|
#
|
26
31
|
|
data/lib/embulk/version.rb
CHANGED
metadata
CHANGED
@@ -1,57 +1,57 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.8
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-11-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: bundler
|
15
15
|
version_requirements: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 1.10.6
|
20
20
|
requirement: !ruby/object:Gem::Requirement
|
21
21
|
requirements:
|
22
22
|
- - "~>"
|
23
23
|
- !ruby/object:Gem::Version
|
24
|
-
version:
|
24
|
+
version: 1.10.6
|
25
25
|
prerelease: false
|
26
26
|
type: :runtime
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: liquid
|
29
29
|
version_requirements: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: 3.0.6
|
34
34
|
requirement: !ruby/object:Gem::Requirement
|
35
35
|
requirements:
|
36
36
|
- - "~>"
|
37
37
|
- !ruby/object:Gem::Version
|
38
|
-
version:
|
38
|
+
version: 3.0.6
|
39
39
|
prerelease: false
|
40
40
|
type: :runtime
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: rjack-icu
|
43
43
|
version_requirements: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: 4.54.1.1
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
49
49
|
requirements:
|
50
|
-
- - "
|
50
|
+
- - "~>"
|
51
51
|
- !ruby/object:Gem::Version
|
52
|
-
version:
|
52
|
+
version: 4.54.1.1
|
53
53
|
prerelease: false
|
54
|
-
type: :
|
54
|
+
type: :runtime
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: rake
|
57
57
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -133,8 +133,8 @@ files:
|
|
133
133
|
- classpath/bval-jsr303-0.5.jar
|
134
134
|
- classpath/commons-beanutils-core-1.8.3.jar
|
135
135
|
- classpath/commons-lang3-3.1.jar
|
136
|
-
- classpath/embulk-core-0.7.
|
137
|
-
- classpath/embulk-standards-0.7.
|
136
|
+
- classpath/embulk-core-0.7.8.jar
|
137
|
+
- classpath/embulk-standards-0.7.8.jar
|
138
138
|
- classpath/guava-18.0.jar
|
139
139
|
- classpath/guice-4.0.jar
|
140
140
|
- classpath/guice-multibindings-4.0.jar
|
@@ -236,8 +236,10 @@ files:
|
|
236
236
|
- embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalFileInputWrapper.java
|
237
237
|
- embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalFileOutputWrapper.java
|
238
238
|
- embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalPageOutputWrapper.java
|
239
|
+
- embulk-core/src/main/java/org/embulk/spi/AbortTransactionResource.java
|
239
240
|
- embulk-core/src/main/java/org/embulk/spi/Buffer.java
|
240
241
|
- embulk-core/src/main/java/org/embulk/spi/BufferAllocator.java
|
242
|
+
- embulk-core/src/main/java/org/embulk/spi/CloseResource.java
|
241
243
|
- embulk-core/src/main/java/org/embulk/spi/Column.java
|
242
244
|
- embulk-core/src/main/java/org/embulk/spi/ColumnConfig.java
|
243
245
|
- embulk-core/src/main/java/org/embulk/spi/ColumnVisitor.java
|
@@ -439,6 +441,7 @@ files:
|
|
439
441
|
- embulk-docs/src/release/release-0.7.5.rst
|
440
442
|
- embulk-docs/src/release/release-0.7.6.rst
|
441
443
|
- embulk-docs/src/release/release-0.7.7.rst
|
444
|
+
- embulk-docs/src/release/release-0.7.8.rst
|
442
445
|
- embulk-standards/build.gradle
|
443
446
|
- embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
|
444
447
|
- embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java
|