embulk 0.8.18-java → 0.8.19-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +10 -0
- data/build.gradle +10 -3
- data/embulk-cli/build.gradle +2 -0
- data/embulk-cli/src/main/bat/selfrun.bat +98 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkExample.java +82 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkMigrate.java +458 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkNew.java +419 -0
- data/embulk-cli/src/main/java/org/embulk/cli/EmbulkSelfUpdate.java +248 -0
- data/embulk-cli/src/main/sh/selfrun.sh +0 -103
- data/embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java +158 -143
- data/embulk-core/build.gradle +2 -2
- data/embulk-core/src/main/java/org/embulk/EmbulkVersion.java +109 -0
- data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java +11 -0
- data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java +29 -3
- data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java +47 -13
- data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java +6 -3
- data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java +385 -64
- data/embulk-core/src/main/java/org/embulk/spi/TempFileSpace.java +2 -1
- data/embulk-core/src/test/java/org/embulk/spi/TestPageBuilderReader.java +62 -0
- data/embulk-docs/src/built-in.rst +59 -21
- data/embulk-docs/src/customization.rst +8 -8
- data/embulk-docs/src/developers/index.rst +45 -0
- data/embulk-docs/src/index.rst +11 -7
- data/embulk-docs/src/recipe.rst +1 -1
- data/embulk-docs/src/recipe/{scheduled-csv-load-to-elasticsearch-kibana4.rst → scheduled-csv-load-to-elasticsearch-kibana5.rst} +26 -24
- data/embulk-docs/src/release.rst +1 -0
- data/embulk-docs/src/release/release-0.4.0.rst +1 -1
- data/embulk-docs/src/release/release-0.5.0.rst +1 -1
- data/embulk-docs/src/release/release-0.6.0.rst +1 -1
- data/embulk-docs/src/release/release-0.6.20.rst +1 -1
- data/embulk-docs/src/release/release-0.8.19.rst +43 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +2 -2
- data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java +30 -1
- data/embulk-standards/src/test/java/org/embulk/standards/guess/TestCsvGuessPlugin.java +10 -0
- data/embulk-standards/src/test/java/org/embulk/standards/preview/TestFilePreview.java +73 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records.csv +5 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_guessed.yml +2 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_seed.yml +1 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes.csv +5 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_exec.yml +1 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_load.yml +19 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_previewed.csv +1 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple.csv +5 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_load.yml +19 -0
- data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_previewed.csv +4 -0
- data/embulk-test/src/main/java/org/embulk/test/PreviewResultInputPlugin.java +65 -0
- data/embulk-test/src/main/java/org/embulk/test/TestingBulkLoader.java +5 -0
- data/embulk-test/src/main/java/org/embulk/test/TestingEmbulk.java +59 -2
- data/embulk.gemspec +2 -1
- data/lib/embulk/command/embulk_run.rb +11 -49
- data/lib/embulk/data/new/README.md.vm +106 -0
- data/lib/embulk/data/new/{gitignore.erb → gitignore.vm} +3 -3
- data/lib/embulk/data/new/java/{build.gradle.erb → build.gradle.vm} +8 -8
- data/lib/embulk/data/new/java/{decoder.java.erb → decoder.java.vm} +6 -4
- data/lib/embulk/data/new/java/{encoder.java.erb → encoder.java.vm} +7 -5
- data/lib/embulk/data/new/java/{file_input.java.erb → file_input.java.vm} +9 -7
- data/lib/embulk/data/new/java/{file_output.java.erb → file_output.java.vm} +7 -5
- data/lib/embulk/data/new/java/{filter.java.erb → filter.java.vm} +4 -3
- data/lib/embulk/data/new/java/{formatter.java.erb → formatter.java.vm} +5 -4
- data/lib/embulk/data/new/java/{input.java.erb → input.java.vm} +6 -4
- data/lib/embulk/data/new/java/{output.java.erb → output.java.vm} +7 -5
- data/lib/embulk/data/new/java/{parser.java.erb → parser.java.vm} +5 -4
- data/lib/embulk/data/new/java/plugin_loader.rb.vm +3 -0
- data/lib/embulk/data/new/java/test.java.vm +5 -0
- data/lib/embulk/data/new/ruby/decoder_guess.rb.vm +25 -0
- data/lib/embulk/data/new/ruby/{filter.rb.erb → filter.rb.vm} +2 -2
- data/lib/embulk/data/new/ruby/{formatter.rb.erb → formatter.rb.vm} +2 -2
- data/lib/embulk/data/new/ruby/gemspec.vm +20 -0
- data/lib/embulk/data/new/ruby/{input.rb.erb → input.rb.vm} +10 -10
- data/lib/embulk/data/new/ruby/{output.rb.erb → output.rb.vm} +7 -7
- data/lib/embulk/data/new/ruby/{parser.rb.erb → parser.rb.vm} +2 -2
- data/lib/embulk/data/new/ruby/parser_guess.rb.vm +65 -0
- data/lib/embulk/guess/csv.rb +5 -0
- data/lib/embulk/version.rb +22 -1
- metadata +55 -35
- data/lib/embulk/command/embulk_example.rb +0 -33
- data/lib/embulk/command/embulk_generate_bin.rb +0 -62
- data/lib/embulk/command/embulk_migrate_plugin.rb +0 -244
- data/lib/embulk/command/embulk_new_plugin.rb +0 -126
- data/lib/embulk/command/embulk_selfupdate.rb +0 -121
- data/lib/embulk/data/new/README.md.erb +0 -111
- data/lib/embulk/data/new/java/plugin_loader.rb.erb +0 -3
- data/lib/embulk/data/new/java/test.java.erb +0 -5
- data/lib/embulk/data/new/ruby/decoder_guess.rb.erb +0 -25
- data/lib/embulk/data/new/ruby/gemspec.erb +0 -20
- data/lib/embulk/data/new/ruby/parser_guess.rb.erb +0 -65
@@ -1,121 +0,0 @@
|
|
1
|
-
module Embulk
|
2
|
-
def self.selfupdate(options={})
|
3
|
-
require 'uri'
|
4
|
-
|
5
|
-
jar, resource = __FILE__.split("!", 2)
|
6
|
-
jar_uri = URI.parse(jar).path rescue jar
|
7
|
-
unless resource && File.file?(jar_uri)
|
8
|
-
STDERR.puts ""
|
9
|
-
STDERR.puts "Embulk is not installed by a single jar. Selfupdate is not supported."
|
10
|
-
STDERR.puts "If you used gem to install embulk, please run: "
|
11
|
-
STDERR.puts ""
|
12
|
-
STDERR.puts " $ gem install embulk"
|
13
|
-
STDERR.puts ""
|
14
|
-
raise SystemExit.new(1)
|
15
|
-
end
|
16
|
-
jar_path = jar_uri
|
17
|
-
|
18
|
-
if version = options[:version]
|
19
|
-
puts "Checking version #{version}..."
|
20
|
-
target_version = check_target_version(version)
|
21
|
-
|
22
|
-
unless target_version
|
23
|
-
puts "Specified version does not exist: #{version}"
|
24
|
-
raise SystemExit.new(1)
|
25
|
-
end
|
26
|
-
puts "Found version #{target_version}."
|
27
|
-
|
28
|
-
else
|
29
|
-
puts "Checking the latest version..."
|
30
|
-
target_version = check_latest_version
|
31
|
-
|
32
|
-
current_version = Gem::Version.new(Embulk::VERSION)
|
33
|
-
if Gem::Version.new(target_version) <= current_version
|
34
|
-
puts "Already up-to-date. #{current_version} is the latest version."
|
35
|
-
return
|
36
|
-
end
|
37
|
-
|
38
|
-
puts "Found new version #{target_version}."
|
39
|
-
end
|
40
|
-
|
41
|
-
unless File.writable?(jar_path)
|
42
|
-
STDERR.puts ""
|
43
|
-
STDERR.puts "Installation path #{jar_path} is not writable."
|
44
|
-
STDERR.puts "Dou you need to run with sudo?"
|
45
|
-
STDERR.puts ""
|
46
|
-
raise SystemExit.new(1)
|
47
|
-
end
|
48
|
-
|
49
|
-
url = "https://dl.bintray.com/embulk/maven/embulk-#{target_version}.jar"
|
50
|
-
puts "Downloading #{url} ..."
|
51
|
-
|
52
|
-
require 'open-uri'
|
53
|
-
require 'tempfile'
|
54
|
-
Tempfile.open(["embulk-selfupdate", ".jar"]) do |tmp|
|
55
|
-
tmp.chmod(File.stat(jar_path).mode)
|
56
|
-
OpenURI.open_uri(url) do |f|
|
57
|
-
IO.copy_stream(f, tmp)
|
58
|
-
end
|
59
|
-
tmp.close(false)
|
60
|
-
|
61
|
-
# check corruption
|
62
|
-
unless options[:force]
|
63
|
-
begin
|
64
|
-
data = File.read("jar:#{java.io.File.new(tmp.path).toURI.toURL}!/embulk/version.rb")
|
65
|
-
m = Module.new
|
66
|
-
m.module_eval(data)
|
67
|
-
unless m::Embulk::VERSION == target_version
|
68
|
-
raise "Embulk::VERSION does not match with #{target_version}"
|
69
|
-
end
|
70
|
-
rescue => e
|
71
|
-
STDERR.puts "Corruption checking failed (#{e})."
|
72
|
-
STDERR.puts "This version might include incompatible changes."
|
73
|
-
STDERR.puts "Please add '-f' argument to selfupdate command to skip checking."
|
74
|
-
raise SystemExit.new(1, e.to_s)
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
File.rename(tmp.path, jar_path)
|
79
|
-
end
|
80
|
-
|
81
|
-
puts "Updated to #{target_version}."
|
82
|
-
end
|
83
|
-
|
84
|
-
private
|
85
|
-
|
86
|
-
def self.check_latest_version
|
87
|
-
start_bintray_http do |http|
|
88
|
-
response = http.get('/embulk/maven/embulk/_latestVersion')
|
89
|
-
if response.code != "302"
|
90
|
-
raise "Expected response code 302 Found but got #{response.code}"
|
91
|
-
end
|
92
|
-
location = response["Location"].to_s
|
93
|
-
m = /(\d+\.\d+[^\/]+)/.match(location)
|
94
|
-
unless m
|
95
|
-
raise "Cound not find version number in Location header '#{location}'"
|
96
|
-
end
|
97
|
-
return m[1]
|
98
|
-
end
|
99
|
-
end
|
100
|
-
|
101
|
-
def self.check_target_version(version=nil)
|
102
|
-
start_bintray_http do |http|
|
103
|
-
response = http.get("/embulk/maven/embulk/#{version}")
|
104
|
-
if response.code == "404"
|
105
|
-
return nil
|
106
|
-
elsif response.code != "200"
|
107
|
-
raise "Unexpected response code: #{response.code}"
|
108
|
-
else
|
109
|
-
return version
|
110
|
-
end
|
111
|
-
end
|
112
|
-
end
|
113
|
-
|
114
|
-
def self.start_bintray_http(&block)
|
115
|
-
require 'net/https'
|
116
|
-
bintray = Net::HTTP.new('bintray.com', 443)
|
117
|
-
bintray.use_ssl = true
|
118
|
-
bintray.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
119
|
-
bintray.start(&block)
|
120
|
-
end
|
121
|
-
end
|
@@ -1,111 +0,0 @@
|
|
1
|
-
# <%= display_name %> <%= display_category %> plugin for Embulk
|
2
|
-
|
3
|
-
%case language
|
4
|
-
%when :ruby
|
5
|
-
TODO: Write short description here and <%= full_project_name %>.gemspec file.
|
6
|
-
%when :java
|
7
|
-
TODO: Write short description here and build.gradle file.
|
8
|
-
%else
|
9
|
-
TODO: Write short description here.
|
10
|
-
%end
|
11
|
-
|
12
|
-
## Overview
|
13
|
-
|
14
|
-
%case category
|
15
|
-
%when :output, :file_output
|
16
|
-
* **Plugin type**: <%= display_category %>
|
17
|
-
* **Load all or nothing**: no
|
18
|
-
* **Resume supported**: no
|
19
|
-
* **Cleanup supported**: yes
|
20
|
-
%when :file_input
|
21
|
-
* **Plugin type**: <%= display_category %>
|
22
|
-
* **Resume supported**: yes
|
23
|
-
* **Cleanup supported**: yes
|
24
|
-
%when :input
|
25
|
-
* **Plugin type**: <%= display_category %>
|
26
|
-
* **Resume supported**: yes
|
27
|
-
* **Cleanup supported**: yes
|
28
|
-
* **Guess supported**: no
|
29
|
-
%when :parser, :decoder
|
30
|
-
* **Plugin type**: <%= display_category %>
|
31
|
-
* **Guess supported**: no
|
32
|
-
%else
|
33
|
-
* **Plugin type**: <%= display_category %>
|
34
|
-
%end
|
35
|
-
|
36
|
-
## Configuration
|
37
|
-
|
38
|
-
- **option1**: description (integer, required)
|
39
|
-
- **option2**: description (string, default: `"myvalue"`)
|
40
|
-
- **option3**: description (string, default: `null`)
|
41
|
-
|
42
|
-
## Example
|
43
|
-
|
44
|
-
```yaml
|
45
|
-
%case category
|
46
|
-
%when :input, :file_input
|
47
|
-
in:
|
48
|
-
type: <%= name %>
|
49
|
-
option1: example1
|
50
|
-
option2: example2
|
51
|
-
%when :output, :file_output
|
52
|
-
out:
|
53
|
-
type: <%= name %>
|
54
|
-
option1: example1
|
55
|
-
option2: example2
|
56
|
-
%when :filter
|
57
|
-
filters:
|
58
|
-
- type: <%= name %>
|
59
|
-
option1: example1
|
60
|
-
option2: example2
|
61
|
-
%when :parser
|
62
|
-
in:
|
63
|
-
type: any file input plugin type
|
64
|
-
parser:
|
65
|
-
type: <%= name %>
|
66
|
-
option1: example1
|
67
|
-
option2: example2
|
68
|
-
%when :formatter
|
69
|
-
out:
|
70
|
-
type: any output input plugin type
|
71
|
-
formatter:
|
72
|
-
type: <%= name %>
|
73
|
-
option1: example1
|
74
|
-
option2: example2
|
75
|
-
%when :decoder
|
76
|
-
in:
|
77
|
-
type: any output input plugin type
|
78
|
-
decoders:
|
79
|
-
- type: <%= name %>
|
80
|
-
option1: example1
|
81
|
-
option2: example2
|
82
|
-
%when :encoder
|
83
|
-
out:
|
84
|
-
type: any output input plugin type
|
85
|
-
encoders:
|
86
|
-
- type: <%= name %>
|
87
|
-
option1: example1
|
88
|
-
option2: example2
|
89
|
-
%end
|
90
|
-
```
|
91
|
-
|
92
|
-
%case category
|
93
|
-
%when :parser, :decoder
|
94
|
-
(If guess supported) you don't have to write `<%= category %>:` section in the configuration file. After writing `in:` section, you can let embulk guess `<%= category %>:` section using this command:
|
95
|
-
|
96
|
-
```
|
97
|
-
$ embulk gem install <%= full_project_name %>
|
98
|
-
$ embulk guess -g <%= name %> config.yml -o guessed.yml
|
99
|
-
```
|
100
|
-
%end
|
101
|
-
|
102
|
-
## Build
|
103
|
-
|
104
|
-
```
|
105
|
-
%case language
|
106
|
-
%when :ruby
|
107
|
-
$ rake
|
108
|
-
%when :java
|
109
|
-
$ ./gradlew gem # -t to watch change of files and rebuild continuously
|
110
|
-
% end
|
111
|
-
```
|
@@ -1,25 +0,0 @@
|
|
1
|
-
module Embulk
|
2
|
-
module Guess
|
3
|
-
|
4
|
-
# TODO implement guess plugin to make this command work:
|
5
|
-
# $ embulk guess -g <%= name.dump %> partial-config.yml
|
6
|
-
%if language == :ruby
|
7
|
-
|
8
|
-
#require <%= "embulk/#{embulk_category}/#{name}.rb".dump %>
|
9
|
-
%end
|
10
|
-
|
11
|
-
#class <%= ruby_class_name.sub('Plugin', 'GuessPlugin') %> < GuessPlugin
|
12
|
-
# Plugin.register_guess(<%= name.dump %>, self)
|
13
|
-
#
|
14
|
-
# FOO_BAR_HEADER = "\x1f\x8b".force_encoding('ASCII-8BIT').freeze
|
15
|
-
#
|
16
|
-
# def guess(config, sample_buffer)
|
17
|
-
# if sample_buffer[0,2] == FOO_BAR_HEADER
|
18
|
-
# return {"decoders" => [{"type" => <%= name.dump %>}]}
|
19
|
-
# end
|
20
|
-
# return {}
|
21
|
-
# end
|
22
|
-
#end
|
23
|
-
|
24
|
-
end
|
25
|
-
end
|
@@ -1,20 +0,0 @@
|
|
1
|
-
|
2
|
-
Gem::Specification.new do |spec|
|
3
|
-
spec.name = "<%= full_project_name %>"
|
4
|
-
spec.version = "0.1.0"
|
5
|
-
spec.authors = [<%= author.dump %>]
|
6
|
-
spec.summary = <%= "#{display_name} #{display_category} plugin for Embulk".dump %>
|
7
|
-
spec.description = <%= "#{description}".dump %>
|
8
|
-
spec.email = [<%= email.dump %>]
|
9
|
-
spec.licenses = ["MIT"]
|
10
|
-
# TODO set this: spec.homepage = "https://github.com/<%= email[/([^@]*)/] %>/<%= full_project_name %>"
|
11
|
-
|
12
|
-
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
13
|
-
spec.test_files = spec.files.grep(%r{^(test|spec)/})
|
14
|
-
spec.require_paths = ["lib"]
|
15
|
-
|
16
|
-
#spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
|
17
|
-
spec.add_development_dependency 'embulk', ['>= <%= Embulk::VERSION %>']
|
18
|
-
spec.add_development_dependency 'bundler', ['>= 1.10.6']
|
19
|
-
spec.add_development_dependency 'rake', ['>= 10.0']
|
20
|
-
end
|
@@ -1,65 +0,0 @@
|
|
1
|
-
module Embulk
|
2
|
-
module Guess
|
3
|
-
|
4
|
-
# TODO implement guess plugin to make this command work:
|
5
|
-
# $ embulk guess -g <%= name.dump %> partial-config.yml
|
6
|
-
#
|
7
|
-
# Depending on the file format the plugin uses, you can use choose
|
8
|
-
# one of binary guess (GuessPlugin), text guess (TextGuessPlugin),
|
9
|
-
# or line guess (LineGuessPlugin).
|
10
|
-
%if language == :ruby
|
11
|
-
|
12
|
-
#require <%= "embulk/#{embulk_category}/#{name}.rb".dump %>
|
13
|
-
%end
|
14
|
-
|
15
|
-
#class <%= ruby_class_name.sub('Plugin', 'GuessPlugin') %> < GuessPlugin
|
16
|
-
# Plugin.register_guess(<%= name.dump %>, self)
|
17
|
-
#
|
18
|
-
# def guess(config, sample_buffer)
|
19
|
-
# if sample_buffer[0,2] == GZIP_HEADER
|
20
|
-
# guessed = {}
|
21
|
-
# guessed["type"] = <%= name.dump %>
|
22
|
-
# guessed["property1"] = "guessed-value"
|
23
|
-
# return {"parser" => guessed}
|
24
|
-
# else
|
25
|
-
# return {}
|
26
|
-
# end
|
27
|
-
# end
|
28
|
-
#end
|
29
|
-
|
30
|
-
#class <%= ruby_class_name.sub('Plugin', 'GuessPlugin') %> < TextGuessPlugin
|
31
|
-
# Plugin.register_guess(<%= name.dump %>, self)
|
32
|
-
#
|
33
|
-
# def guess_text(config, sample_text)
|
34
|
-
# js = JSON.parse(sample_text) rescue nil
|
35
|
-
# if js && js["mykeyword"] == "keyword"
|
36
|
-
# guessed = {}
|
37
|
-
# guessed["type"] = <%= name.dump %>
|
38
|
-
# guessed["property1"] = "guessed-value"
|
39
|
-
# return {"parser" => guessed}
|
40
|
-
# else
|
41
|
-
# return {}
|
42
|
-
# end
|
43
|
-
# end
|
44
|
-
#end
|
45
|
-
|
46
|
-
#class <%= ruby_class_name.sub('Plugin', 'GuessPlugin') %> < LineGuessPlugin
|
47
|
-
# Plugin.register_guess(<%= name.dump %>, self)
|
48
|
-
#
|
49
|
-
# def guess_lines(config, sample_lines)
|
50
|
-
# all_line_matched = sample_lines.all? do |line|
|
51
|
-
# line =~ /mypattern/
|
52
|
-
# end
|
53
|
-
# if all_line_matched
|
54
|
-
# guessed = {}
|
55
|
-
# guessed["type"] = <%= name.dump %>
|
56
|
-
# guessed["property1"] = "guessed-value"
|
57
|
-
# return {"parser" => guessed}
|
58
|
-
# else
|
59
|
-
# return {}
|
60
|
-
# end
|
61
|
-
# end
|
62
|
-
#end
|
63
|
-
|
64
|
-
end
|
65
|
-
end
|