embulk 0.8.18-java → 0.8.19-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (87) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +10 -0
  3. data/build.gradle +10 -3
  4. data/embulk-cli/build.gradle +2 -0
  5. data/embulk-cli/src/main/bat/selfrun.bat +98 -0
  6. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkExample.java +82 -0
  7. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkMigrate.java +458 -0
  8. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkNew.java +419 -0
  9. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkSelfUpdate.java +248 -0
  10. data/embulk-cli/src/main/sh/selfrun.sh +0 -103
  11. data/embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java +158 -143
  12. data/embulk-core/build.gradle +2 -2
  13. data/embulk-core/src/main/java/org/embulk/EmbulkVersion.java +109 -0
  14. data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java +11 -0
  15. data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java +29 -3
  16. data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java +47 -13
  17. data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java +6 -3
  18. data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java +385 -64
  19. data/embulk-core/src/main/java/org/embulk/spi/TempFileSpace.java +2 -1
  20. data/embulk-core/src/test/java/org/embulk/spi/TestPageBuilderReader.java +62 -0
  21. data/embulk-docs/src/built-in.rst +59 -21
  22. data/embulk-docs/src/customization.rst +8 -8
  23. data/embulk-docs/src/developers/index.rst +45 -0
  24. data/embulk-docs/src/index.rst +11 -7
  25. data/embulk-docs/src/recipe.rst +1 -1
  26. data/embulk-docs/src/recipe/{scheduled-csv-load-to-elasticsearch-kibana4.rst → scheduled-csv-load-to-elasticsearch-kibana5.rst} +26 -24
  27. data/embulk-docs/src/release.rst +1 -0
  28. data/embulk-docs/src/release/release-0.4.0.rst +1 -1
  29. data/embulk-docs/src/release/release-0.5.0.rst +1 -1
  30. data/embulk-docs/src/release/release-0.6.0.rst +1 -1
  31. data/embulk-docs/src/release/release-0.6.20.rst +1 -1
  32. data/embulk-docs/src/release/release-0.8.19.rst +43 -0
  33. data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +2 -2
  34. data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java +30 -1
  35. data/embulk-standards/src/test/java/org/embulk/standards/guess/TestCsvGuessPlugin.java +10 -0
  36. data/embulk-standards/src/test/java/org/embulk/standards/preview/TestFilePreview.java +73 -0
  37. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records.csv +5 -0
  38. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_guessed.yml +2 -0
  39. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_seed.yml +1 -0
  40. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes.csv +5 -0
  41. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_exec.yml +1 -0
  42. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_load.yml +19 -0
  43. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_previewed.csv +1 -0
  44. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple.csv +5 -0
  45. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_load.yml +19 -0
  46. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_previewed.csv +4 -0
  47. data/embulk-test/src/main/java/org/embulk/test/PreviewResultInputPlugin.java +65 -0
  48. data/embulk-test/src/main/java/org/embulk/test/TestingBulkLoader.java +5 -0
  49. data/embulk-test/src/main/java/org/embulk/test/TestingEmbulk.java +59 -2
  50. data/embulk.gemspec +2 -1
  51. data/lib/embulk/command/embulk_run.rb +11 -49
  52. data/lib/embulk/data/new/README.md.vm +106 -0
  53. data/lib/embulk/data/new/{gitignore.erb → gitignore.vm} +3 -3
  54. data/lib/embulk/data/new/java/{build.gradle.erb → build.gradle.vm} +8 -8
  55. data/lib/embulk/data/new/java/{decoder.java.erb → decoder.java.vm} +6 -4
  56. data/lib/embulk/data/new/java/{encoder.java.erb → encoder.java.vm} +7 -5
  57. data/lib/embulk/data/new/java/{file_input.java.erb → file_input.java.vm} +9 -7
  58. data/lib/embulk/data/new/java/{file_output.java.erb → file_output.java.vm} +7 -5
  59. data/lib/embulk/data/new/java/{filter.java.erb → filter.java.vm} +4 -3
  60. data/lib/embulk/data/new/java/{formatter.java.erb → formatter.java.vm} +5 -4
  61. data/lib/embulk/data/new/java/{input.java.erb → input.java.vm} +6 -4
  62. data/lib/embulk/data/new/java/{output.java.erb → output.java.vm} +7 -5
  63. data/lib/embulk/data/new/java/{parser.java.erb → parser.java.vm} +5 -4
  64. data/lib/embulk/data/new/java/plugin_loader.rb.vm +3 -0
  65. data/lib/embulk/data/new/java/test.java.vm +5 -0
  66. data/lib/embulk/data/new/ruby/decoder_guess.rb.vm +25 -0
  67. data/lib/embulk/data/new/ruby/{filter.rb.erb → filter.rb.vm} +2 -2
  68. data/lib/embulk/data/new/ruby/{formatter.rb.erb → formatter.rb.vm} +2 -2
  69. data/lib/embulk/data/new/ruby/gemspec.vm +20 -0
  70. data/lib/embulk/data/new/ruby/{input.rb.erb → input.rb.vm} +10 -10
  71. data/lib/embulk/data/new/ruby/{output.rb.erb → output.rb.vm} +7 -7
  72. data/lib/embulk/data/new/ruby/{parser.rb.erb → parser.rb.vm} +2 -2
  73. data/lib/embulk/data/new/ruby/parser_guess.rb.vm +65 -0
  74. data/lib/embulk/guess/csv.rb +5 -0
  75. data/lib/embulk/version.rb +22 -1
  76. metadata +55 -35
  77. data/lib/embulk/command/embulk_example.rb +0 -33
  78. data/lib/embulk/command/embulk_generate_bin.rb +0 -62
  79. data/lib/embulk/command/embulk_migrate_plugin.rb +0 -244
  80. data/lib/embulk/command/embulk_new_plugin.rb +0 -126
  81. data/lib/embulk/command/embulk_selfupdate.rb +0 -121
  82. data/lib/embulk/data/new/README.md.erb +0 -111
  83. data/lib/embulk/data/new/java/plugin_loader.rb.erb +0 -3
  84. data/lib/embulk/data/new/java/test.java.erb +0 -5
  85. data/lib/embulk/data/new/ruby/decoder_guess.rb.erb +0 -25
  86. data/lib/embulk/data/new/ruby/gemspec.erb +0 -20
  87. data/lib/embulk/data/new/ruby/parser_guess.rb.erb +0 -65
@@ -1,121 +0,0 @@
1
- module Embulk
2
- def self.selfupdate(options={})
3
- require 'uri'
4
-
5
- jar, resource = __FILE__.split("!", 2)
6
- jar_uri = URI.parse(jar).path rescue jar
7
- unless resource && File.file?(jar_uri)
8
- STDERR.puts ""
9
- STDERR.puts "Embulk is not installed by a single jar. Selfupdate is not supported."
10
- STDERR.puts "If you used gem to install embulk, please run: "
11
- STDERR.puts ""
12
- STDERR.puts " $ gem install embulk"
13
- STDERR.puts ""
14
- raise SystemExit.new(1)
15
- end
16
- jar_path = jar_uri
17
-
18
- if version = options[:version]
19
- puts "Checking version #{version}..."
20
- target_version = check_target_version(version)
21
-
22
- unless target_version
23
- puts "Specified version does not exist: #{version}"
24
- raise SystemExit.new(1)
25
- end
26
- puts "Found version #{target_version}."
27
-
28
- else
29
- puts "Checking the latest version..."
30
- target_version = check_latest_version
31
-
32
- current_version = Gem::Version.new(Embulk::VERSION)
33
- if Gem::Version.new(target_version) <= current_version
34
- puts "Already up-to-date. #{current_version} is the latest version."
35
- return
36
- end
37
-
38
- puts "Found new version #{target_version}."
39
- end
40
-
41
- unless File.writable?(jar_path)
42
- STDERR.puts ""
43
- STDERR.puts "Installation path #{jar_path} is not writable."
44
- STDERR.puts "Dou you need to run with sudo?"
45
- STDERR.puts ""
46
- raise SystemExit.new(1)
47
- end
48
-
49
- url = "https://dl.bintray.com/embulk/maven/embulk-#{target_version}.jar"
50
- puts "Downloading #{url} ..."
51
-
52
- require 'open-uri'
53
- require 'tempfile'
54
- Tempfile.open(["embulk-selfupdate", ".jar"]) do |tmp|
55
- tmp.chmod(File.stat(jar_path).mode)
56
- OpenURI.open_uri(url) do |f|
57
- IO.copy_stream(f, tmp)
58
- end
59
- tmp.close(false)
60
-
61
- # check corruption
62
- unless options[:force]
63
- begin
64
- data = File.read("jar:#{java.io.File.new(tmp.path).toURI.toURL}!/embulk/version.rb")
65
- m = Module.new
66
- m.module_eval(data)
67
- unless m::Embulk::VERSION == target_version
68
- raise "Embulk::VERSION does not match with #{target_version}"
69
- end
70
- rescue => e
71
- STDERR.puts "Corruption checking failed (#{e})."
72
- STDERR.puts "This version might include incompatible changes."
73
- STDERR.puts "Please add '-f' argument to selfupdate command to skip checking."
74
- raise SystemExit.new(1, e.to_s)
75
- end
76
- end
77
-
78
- File.rename(tmp.path, jar_path)
79
- end
80
-
81
- puts "Updated to #{target_version}."
82
- end
83
-
84
- private
85
-
86
- def self.check_latest_version
87
- start_bintray_http do |http|
88
- response = http.get('/embulk/maven/embulk/_latestVersion')
89
- if response.code != "302"
90
- raise "Expected response code 302 Found but got #{response.code}"
91
- end
92
- location = response["Location"].to_s
93
- m = /(\d+\.\d+[^\/]+)/.match(location)
94
- unless m
95
- raise "Cound not find version number in Location header '#{location}'"
96
- end
97
- return m[1]
98
- end
99
- end
100
-
101
- def self.check_target_version(version=nil)
102
- start_bintray_http do |http|
103
- response = http.get("/embulk/maven/embulk/#{version}")
104
- if response.code == "404"
105
- return nil
106
- elsif response.code != "200"
107
- raise "Unexpected response code: #{response.code}"
108
- else
109
- return version
110
- end
111
- end
112
- end
113
-
114
- def self.start_bintray_http(&block)
115
- require 'net/https'
116
- bintray = Net::HTTP.new('bintray.com', 443)
117
- bintray.use_ssl = true
118
- bintray.verify_mode = OpenSSL::SSL::VERIFY_NONE
119
- bintray.start(&block)
120
- end
121
- end
@@ -1,111 +0,0 @@
1
- # <%= display_name %> <%= display_category %> plugin for Embulk
2
-
3
- %case language
4
- %when :ruby
5
- TODO: Write short description here and <%= full_project_name %>.gemspec file.
6
- %when :java
7
- TODO: Write short description here and build.gradle file.
8
- %else
9
- TODO: Write short description here.
10
- %end
11
-
12
- ## Overview
13
-
14
- %case category
15
- %when :output, :file_output
16
- * **Plugin type**: <%= display_category %>
17
- * **Load all or nothing**: no
18
- * **Resume supported**: no
19
- * **Cleanup supported**: yes
20
- %when :file_input
21
- * **Plugin type**: <%= display_category %>
22
- * **Resume supported**: yes
23
- * **Cleanup supported**: yes
24
- %when :input
25
- * **Plugin type**: <%= display_category %>
26
- * **Resume supported**: yes
27
- * **Cleanup supported**: yes
28
- * **Guess supported**: no
29
- %when :parser, :decoder
30
- * **Plugin type**: <%= display_category %>
31
- * **Guess supported**: no
32
- %else
33
- * **Plugin type**: <%= display_category %>
34
- %end
35
-
36
- ## Configuration
37
-
38
- - **option1**: description (integer, required)
39
- - **option2**: description (string, default: `"myvalue"`)
40
- - **option3**: description (string, default: `null`)
41
-
42
- ## Example
43
-
44
- ```yaml
45
- %case category
46
- %when :input, :file_input
47
- in:
48
- type: <%= name %>
49
- option1: example1
50
- option2: example2
51
- %when :output, :file_output
52
- out:
53
- type: <%= name %>
54
- option1: example1
55
- option2: example2
56
- %when :filter
57
- filters:
58
- - type: <%= name %>
59
- option1: example1
60
- option2: example2
61
- %when :parser
62
- in:
63
- type: any file input plugin type
64
- parser:
65
- type: <%= name %>
66
- option1: example1
67
- option2: example2
68
- %when :formatter
69
- out:
70
- type: any output input plugin type
71
- formatter:
72
- type: <%= name %>
73
- option1: example1
74
- option2: example2
75
- %when :decoder
76
- in:
77
- type: any output input plugin type
78
- decoders:
79
- - type: <%= name %>
80
- option1: example1
81
- option2: example2
82
- %when :encoder
83
- out:
84
- type: any output input plugin type
85
- encoders:
86
- - type: <%= name %>
87
- option1: example1
88
- option2: example2
89
- %end
90
- ```
91
-
92
- %case category
93
- %when :parser, :decoder
94
- (If guess supported) you don't have to write `<%= category %>:` section in the configuration file. After writing `in:` section, you can let embulk guess `<%= category %>:` section using this command:
95
-
96
- ```
97
- $ embulk gem install <%= full_project_name %>
98
- $ embulk guess -g <%= name %> config.yml -o guessed.yml
99
- ```
100
- %end
101
-
102
- ## Build
103
-
104
- ```
105
- %case language
106
- %when :ruby
107
- $ rake
108
- %when :java
109
- $ ./gradlew gem # -t to watch change of files and rebuild continuously
110
- % end
111
- ```
@@ -1,3 +0,0 @@
1
- Embulk::JavaPlugin.register_<%= embulk_category %>(
2
- <%= name.dump %>, <%= "#{java_package_name}.#{java_class_name}".dump %>,
3
- File.expand_path('../../../../classpath', __FILE__))
@@ -1,5 +0,0 @@
1
- package <%= java_package_name %>;
2
-
3
- public class Test<%= java_class_name %>
4
- {
5
- }
@@ -1,25 +0,0 @@
1
- module Embulk
2
- module Guess
3
-
4
- # TODO implement guess plugin to make this command work:
5
- # $ embulk guess -g <%= name.dump %> partial-config.yml
6
- %if language == :ruby
7
-
8
- #require <%= "embulk/#{embulk_category}/#{name}.rb".dump %>
9
- %end
10
-
11
- #class <%= ruby_class_name.sub('Plugin', 'GuessPlugin') %> < GuessPlugin
12
- # Plugin.register_guess(<%= name.dump %>, self)
13
- #
14
- # FOO_BAR_HEADER = "\x1f\x8b".force_encoding('ASCII-8BIT').freeze
15
- #
16
- # def guess(config, sample_buffer)
17
- # if sample_buffer[0,2] == FOO_BAR_HEADER
18
- # return {"decoders" => [{"type" => <%= name.dump %>}]}
19
- # end
20
- # return {}
21
- # end
22
- #end
23
-
24
- end
25
- end
@@ -1,20 +0,0 @@
1
-
2
- Gem::Specification.new do |spec|
3
- spec.name = "<%= full_project_name %>"
4
- spec.version = "0.1.0"
5
- spec.authors = [<%= author.dump %>]
6
- spec.summary = <%= "#{display_name} #{display_category} plugin for Embulk".dump %>
7
- spec.description = <%= "#{description}".dump %>
8
- spec.email = [<%= email.dump %>]
9
- spec.licenses = ["MIT"]
10
- # TODO set this: spec.homepage = "https://github.com/<%= email[/([^@]*)/] %>/<%= full_project_name %>"
11
-
12
- spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
13
- spec.test_files = spec.files.grep(%r{^(test|spec)/})
14
- spec.require_paths = ["lib"]
15
-
16
- #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
17
- spec.add_development_dependency 'embulk', ['>= <%= Embulk::VERSION %>']
18
- spec.add_development_dependency 'bundler', ['>= 1.10.6']
19
- spec.add_development_dependency 'rake', ['>= 10.0']
20
- end
@@ -1,65 +0,0 @@
1
- module Embulk
2
- module Guess
3
-
4
- # TODO implement guess plugin to make this command work:
5
- # $ embulk guess -g <%= name.dump %> partial-config.yml
6
- #
7
- # Depending on the file format the plugin uses, you can use choose
8
- # one of binary guess (GuessPlugin), text guess (TextGuessPlugin),
9
- # or line guess (LineGuessPlugin).
10
- %if language == :ruby
11
-
12
- #require <%= "embulk/#{embulk_category}/#{name}.rb".dump %>
13
- %end
14
-
15
- #class <%= ruby_class_name.sub('Plugin', 'GuessPlugin') %> < GuessPlugin
16
- # Plugin.register_guess(<%= name.dump %>, self)
17
- #
18
- # def guess(config, sample_buffer)
19
- # if sample_buffer[0,2] == GZIP_HEADER
20
- # guessed = {}
21
- # guessed["type"] = <%= name.dump %>
22
- # guessed["property1"] = "guessed-value"
23
- # return {"parser" => guessed}
24
- # else
25
- # return {}
26
- # end
27
- # end
28
- #end
29
-
30
- #class <%= ruby_class_name.sub('Plugin', 'GuessPlugin') %> < TextGuessPlugin
31
- # Plugin.register_guess(<%= name.dump %>, self)
32
- #
33
- # def guess_text(config, sample_text)
34
- # js = JSON.parse(sample_text) rescue nil
35
- # if js && js["mykeyword"] == "keyword"
36
- # guessed = {}
37
- # guessed["type"] = <%= name.dump %>
38
- # guessed["property1"] = "guessed-value"
39
- # return {"parser" => guessed}
40
- # else
41
- # return {}
42
- # end
43
- # end
44
- #end
45
-
46
- #class <%= ruby_class_name.sub('Plugin', 'GuessPlugin') %> < LineGuessPlugin
47
- # Plugin.register_guess(<%= name.dump %>, self)
48
- #
49
- # def guess_lines(config, sample_lines)
50
- # all_line_matched = sample_lines.all? do |line|
51
- # line =~ /mypattern/
52
- # end
53
- # if all_line_matched
54
- # guessed = {}
55
- # guessed["type"] = <%= name.dump %>
56
- # guessed["property1"] = "guessed-value"
57
- # return {"parser" => guessed}
58
- # else
59
- # return {}
60
- # end
61
- # end
62
- #end
63
-
64
- end
65
- end