embulk 0.7.2 → 0.7.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 69167ac2d89cb01cce7fe04d3380290604141720
4
- data.tar.gz: cfcc577be3131bd9792d420425fd10dedf2df974
3
+ metadata.gz: e54fd2e5f50348148e43d19d4bd832c1fc23b2bc
4
+ data.tar.gz: e9ee753c60761e9eb9304e2cc107808409e2e94a
5
5
  SHA512:
6
- metadata.gz: ebe4adc87fd92f411e19db78e3ea58569c7902bf8e7f9dee97b797d5d7b3f502c056804d8e4d8a9989b6e61dc8e36476588fbf279d8232d48a4ea7842fa3411b
7
- data.tar.gz: b2721835cd90a56cd9c97644a1af1429225542a8c0f1845014ce833e4f548dd48fed4511329f3f196da00d85783f8bb8bab73cf1e1f5effb6ff680fe5a13cacf
6
+ metadata.gz: 6ff353d37efad5f385422cc78af6fa6ea7530a0734ab33fbde2883cbde9363a9b45aa00c801087aa9631d805c3e62e5525363c7c8b094207377005fc00660aca
7
+ data.tar.gz: 43a9a324e3682d2f116843a7ace68c6b5fab138a9d06d9bbdf7e8c831f74c32069538de4f141826b58ed99a5cc8c26e5b09f818aac984b8c641ae27c1301ba6b
data/build.gradle CHANGED
@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
16
16
 
17
17
  allprojects {
18
18
  group = 'org.embulk'
19
- version = '0.7.2'
19
+ version = '0.7.3'
20
20
 
21
21
  ext {
22
22
  jrubyVersion = '9.0.0.0'
@@ -4,6 +4,7 @@ Release Notes
4
4
  .. toctree::
5
5
  :maxdepth: 1
6
6
 
7
+ release/release-0.7.3
7
8
  release/release-0.7.2
8
9
  release/release-0.7.1
9
10
  release/release-0.7.0
@@ -0,0 +1,21 @@
1
+ Release 0.7.3
2
+ ==================================
3
+
4
+ General Changes
5
+ ------------------
6
+
7
+ * ``embulk selfupdate`` command gets optoinal ``<version>`` argument to upgrade or downgrade to specific version (@cosmo0920++).
8
+ * ``embulk migrate`` command assumes that source code file encoding is UTF-8. This fixes encoding exception on Windows environment.
9
+
10
+
11
+ Java Plugin API
12
+ ------------------
13
+
14
+ * Fixed ``Embulk::Exec.preview?``.
15
+ * Fixed ``raise ConfigException``.
16
+ * added missing ``PluginLoadError`` class.
17
+
18
+
19
+ Release Date
20
+ ------------------
21
+ 2015-08-23
data/lib/embulk/buffer.rb CHANGED
@@ -1,22 +1,20 @@
1
1
 
2
2
  module Embulk
3
3
  class Buffer < String
4
- if Embulk.java?
5
- def self.from_java(java_buffer)
6
- byte_list = org.jruby.util.ByteList.new(java_buffer.array(), java_buffer.offset(), java_buffer.limit(), false)
7
- buffer = new
8
- buffer.replace(org.jruby.RubyString.new(JRuby.runtime, self, byte_list).dup) # TODO simplify
9
- buffer
10
- end
4
+ def self.from_java(java_buffer)
5
+ byte_list = org.jruby.util.ByteList.new(java_buffer.array(), java_buffer.offset(), java_buffer.limit(), false)
6
+ buffer = new
7
+ buffer.replace(org.jruby.RubyString.new(JRuby.runtime, self, byte_list).dup) # TODO simplify
8
+ buffer
9
+ end
11
10
 
12
- def self.from_ruby_string(string)
13
- b = Buffer.new(string)
14
- b.force_encoding('ASCII-8BIT')
15
- end
11
+ def self.from_ruby_string(string)
12
+ b = Buffer.new(string)
13
+ b.force_encoding('ASCII-8BIT')
14
+ end
16
15
 
17
- def to_java
18
- Java::Buffer.wrap(to_java_bytes)
19
- end
16
+ def to_java
17
+ Java::Buffer.wrap(to_java_bytes)
20
18
  end
21
19
  end
22
20
  end
data/lib/embulk/column.rb CHANGED
@@ -20,49 +20,45 @@ module Embulk
20
20
  end
21
21
  end
22
22
 
23
- if Embulk.java?
24
- def self.from_java(java_column)
25
- type = Type.from_java(java_column.getType)
26
- if type == :timestamp
27
- format = java_column.getType.getFormat
28
- else
29
- format = nil
30
- end
31
-
32
- Column.new(java_column.getIndex, java_column.getName, type, format)
23
+ def self.from_java(java_column)
24
+ type = Type.from_java(java_column.getType)
25
+ if type == :timestamp
26
+ format = java_column.getType.getFormat
27
+ else
28
+ format = nil
33
29
  end
34
30
 
35
- def to_java
36
- if type == :timestamp && format
37
- Java::Column.new(index, name, Type.new_java_type(type).withFormat(format))
38
- else
39
- Java::Column.new(index, name, Type.new_java_type(type))
40
- end
31
+ Column.new(java_column.getIndex, java_column.getName, type, format)
32
+ end
33
+
34
+ def to_java
35
+ if type == :timestamp && format
36
+ Java::Column.new(index, name, Type.new_java_type(type).withFormat(format))
37
+ else
38
+ Java::Column.new(index, name, Type.new_java_type(type))
41
39
  end
42
40
  end
43
41
  end
44
42
 
45
43
  module Type
46
- if Embulk.java?
47
- def self.from_java(java_type)
48
- java_type.getName.to_sym
49
- end
44
+ def self.from_java(java_type)
45
+ java_type.getName.to_sym
46
+ end
50
47
 
51
- def self.new_java_type(ruby_type)
52
- case ruby_type
53
- when :boolean
54
- Java::Types::BOOLEAN
55
- when :long
56
- Java::Types::LONG
57
- when :double
58
- Java::Types::DOUBLE
59
- when :string
60
- Java::Types::STRING
61
- when :timestamp
62
- Java::Types::TIMESTAMP
63
- else
64
- raise ArgumentError, "Unknown type #{ruby_type.inspect}: supported types are :boolean, :long, :double, :string and :timestamp"
65
- end
48
+ def self.new_java_type(ruby_type)
49
+ case ruby_type
50
+ when :boolean
51
+ Java::Types::BOOLEAN
52
+ when :long
53
+ Java::Types::LONG
54
+ when :double
55
+ Java::Types::DOUBLE
56
+ when :string
57
+ Java::Types::STRING
58
+ when :timestamp
59
+ Java::Types::TIMESTAMP
60
+ else
61
+ raise ArgumentError, "Unknown type #{ruby_type.inspect}: supported types are :boolean, :long, :double, :string and :timestamp"
66
62
  end
67
63
  end
68
64
  end
@@ -96,7 +96,7 @@ module Embulk
96
96
 
97
97
  def match(glob, pattern)
98
98
  ms = Dir[File.join(@path, glob)].map do |file|
99
- File.read(file).match(pattern)
99
+ read(file).match(pattern)
100
100
  end.compact
101
101
  return nil if ms.empty?
102
102
  ms
@@ -104,7 +104,7 @@ module Embulk
104
104
 
105
105
  def replace(glob, pattern, text=nil)
106
106
  ms = Dir[File.join(@path, glob)].map do |file|
107
- data = File.read(file)
107
+ data = read(file)
108
108
  first = nil
109
109
  pos = 0
110
110
  while pos < data.length
@@ -126,7 +126,7 @@ module Embulk
126
126
 
127
127
  def insert_line(glob, pattern, text=nil)
128
128
  ms = Dir[File.join(@path, glob)].map do |file|
129
- data = File.read(file)
129
+ data = read(file)
130
130
  if m = data.match(pattern)
131
131
  ln = m.pre_match.split("\n").count
132
132
  replace = text || yield(m)
@@ -142,7 +142,7 @@ module Embulk
142
142
  end
143
143
 
144
144
  def data(file)
145
- File.read(File.join(@path, file))
145
+ read(File.join(@path, file))
146
146
  end
147
147
 
148
148
  def write(file, data)
@@ -152,7 +152,7 @@ module Embulk
152
152
  private
153
153
 
154
154
  def modify(path, data)
155
- orig = File.read(path) rescue nil
155
+ orig = read(path) rescue nil
156
156
  if orig != data
157
157
  File.write(path, data)
158
158
  unless @modified_files.has_key?(path)
@@ -166,5 +166,11 @@ module Embulk
166
166
  end
167
167
  nil
168
168
  end
169
+
170
+ def read(path)
171
+ # external_encoding: assumes source code is written in UTF-8.
172
+ # internal_encoding: process files using UTF-8 so that modified data (data written to the file) becomes UTF-8.
173
+ File.read(path, external_encoding: 'UTF-8', internal_encoding: 'UTF-8')
174
+ end
169
175
  end
170
176
  end
@@ -175,7 +175,7 @@ examples:
175
175
  op.on('-f', "Skip corruption check", TrueClass) do |b|
176
176
  options[:force] = true
177
177
  end
178
- args = 0..0
178
+ args = 0..1
179
179
 
180
180
  when :example
181
181
  args = 0..1
@@ -253,6 +253,7 @@ examples:
253
253
 
254
254
  when :selfupdate
255
255
  require 'embulk/command/embulk_selfupdate'
256
+ options[:version] = ARGV[0]
256
257
  Embulk.selfupdate(options)
257
258
 
258
259
  else
@@ -424,16 +425,16 @@ examples:
424
425
  STDERR.puts "Embulk v#{Embulk::VERSION}"
425
426
  STDERR.puts "usage: <command> [--options]"
426
427
  STDERR.puts "commands:"
427
- STDERR.puts " bundle [directory] # create or update plugin environment."
428
- STDERR.puts " run <config.yml> # run a bulk load transaction."
429
- STDERR.puts " preview <config.yml> # dry-run the bulk load without output and show preview."
430
- STDERR.puts " guess <partial-config.yml> -o <output.yml> # guess missing parameters to create a complete configuration file."
431
- STDERR.puts " gem <install | list | help> # install a plugin or show installed plugins."
428
+ STDERR.puts " bundle [directory] # create or update plugin environment."
429
+ STDERR.puts " run <config.yml> # run a bulk load transaction."
430
+ STDERR.puts " preview <config.yml> # dry-run the bulk load without output and show preview."
431
+ STDERR.puts " guess <partial-config.yml> -o <output.yml> # guess missing parameters to create a complete configuration file."
432
+ STDERR.puts " gem <install | list | help> # install a plugin or show installed plugins."
432
433
  STDERR.puts " # plugin path is #{ENV['GEM_HOME']}"
433
- STDERR.puts " new <category> <name> # generates new plugin template"
434
- STDERR.puts " migrate <path> # modify plugin code to use the latest Embulk plugin API"
435
- STDERR.puts " example [path] # creates an example config file and csv file to try embulk."
436
- STDERR.puts " selfupdate # upgrades embulk to the latest released version."
434
+ STDERR.puts " new <category> <name> # generates new plugin template"
435
+ STDERR.puts " migrate <path> # modify plugin code to use the latest Embulk plugin API"
436
+ STDERR.puts " example [path] # creates an example config file and csv file to try embulk."
437
+ STDERR.puts " selfupdate [version] # upgrades embulk to the latest released version or to the specified version."
437
438
  STDERR.puts ""
438
439
  if message
439
440
  system_exit "error: #{message}"
@@ -14,15 +14,27 @@ module Embulk
14
14
  raise SystemExit.new(1)
15
15
  end
16
16
 
17
- puts "Checking the latest version..."
18
- latest_version = check_latest_version
17
+ if version = options[:version]
18
+ puts "Checking version #{version}..."
19
+ target_version = check_target_version(version)
19
20
 
20
- if Gem::Version.new(latest_version) <= Gem::Version.new(Embulk::VERSION)
21
- puts "Already up-to-date. #{latest_version} is the latest version."
22
- return
23
- end
21
+ unless target_version
22
+ puts "Specified version does not exist: #{version}"
23
+ raise SystemExit.new(1)
24
+ end
25
+ puts "Found version #{target_version}."
26
+
27
+ else
28
+ puts "Checking the latest version..."
29
+ target_version = check_latest_version
24
30
 
25
- puts "Found new version #{latest_version}."
31
+ if Gem::Version.new(target_version) <= Gem::Version.new(Embulk::VERSION)
32
+ puts "Already up-to-date. #{target_version} is the latest version."
33
+ return
34
+ end
35
+
36
+ puts "Found new version #{target_version}."
37
+ end
26
38
 
27
39
  unless File.writable?(jar_path)
28
40
  STDERR.puts ""
@@ -32,7 +44,7 @@ module Embulk
32
44
  raise SystemExit.new(1)
33
45
  end
34
46
 
35
- url = "https://dl.bintray.com/embulk/maven/embulk-#{latest_version}.jar"
47
+ url = "https://dl.bintray.com/embulk/maven/embulk-#{target_version}.jar"
36
48
  puts "Downloading #{url} ..."
37
49
 
38
50
  require 'open-uri'
@@ -50,8 +62,8 @@ module Embulk
50
62
  data = File.read("jar:#{java.io.File.new(tmp.path).toURI.toURL}!/embulk/version.rb")
51
63
  m = Module.new
52
64
  m.module_eval(data)
53
- unless m::Embulk::VERSION == latest_version
54
- raise "Embulk::VERSION does not match with #{latest_version}"
65
+ unless m::Embulk::VERSION == target_version
66
+ raise "Embulk::VERSION does not match with #{target_version}"
55
67
  end
56
68
  rescue => e
57
69
  STDERR.puts "Corruption checking failed (#{e})."
@@ -64,21 +76,44 @@ module Embulk
64
76
  File.rename(tmp.path, jar_path)
65
77
  end
66
78
 
67
- puts "Updated to #{latest_version}."
79
+ puts "Updated to #{target_version}."
68
80
  end
69
81
 
82
+ private
83
+
70
84
  def self.check_latest_version
71
- require 'net/https'
72
- bintray = Net::HTTP.new('bintray.com', 443)
73
- bintray.use_ssl = true
74
- bintray.verify_mode = OpenSSL::SSL::VERIFY_NONE
75
- bintray.start do
76
- response = bintray.get('/embulk/maven/embulk/_latestVersion')
77
- raise "Expected response code 302 Found but got #{response.code}" if response.code != "302"
85
+ start_bintray_http do |http|
86
+ response = http.get('/embulk/maven/embulk/_latestVersion')
87
+ if response.code != "302"
88
+ raise "Expected response code 302 Found but got #{response.code}"
89
+ end
78
90
  location = response["Location"].to_s
79
91
  m = /(\d+\.\d+[^\/]+)/.match(location)
80
- raise "Cound not find version number in Location header '#{location}'" unless m
92
+ unless m
93
+ raise "Cound not find version number in Location header '#{location}'"
94
+ end
81
95
  return m[1]
82
96
  end
83
97
  end
98
+
99
+ def self.check_target_version(version=nil)
100
+ start_bintray_http do |http|
101
+ response = http.get("/embulk/maven/embulk/#{version}")
102
+ if response.code == "404"
103
+ return nil
104
+ elsif response.code != "200"
105
+ raise "Unexpected response code: #{response.code}"
106
+ else
107
+ return version
108
+ end
109
+ end
110
+ end
111
+
112
+ def self.start_bintray_http(&block)
113
+ require 'net/https'
114
+ bintray = Net::HTTP.new('bintray.com', 443)
115
+ bintray.use_ssl = true
116
+ bintray.verify_mode = OpenSSL::SSL::VERIFY_NONE
117
+ bintray.start(&block)
118
+ end
84
119
  end
@@ -145,31 +145,31 @@ module Embulk
145
145
  begin
146
146
  Integer(v)
147
147
  rescue => e
148
- raise ConfigError, e
148
+ raise ConfigError.new e
149
149
  end
150
150
  when :float
151
151
  begin
152
152
  Float(v)
153
153
  rescue => e
154
- raise ConfigError, e
154
+ raise ConfigError.new e
155
155
  end
156
156
  when :string
157
157
  begin
158
158
  String(v).dup
159
159
  rescue => e
160
- raise ConfigError, e
160
+ raise ConfigError.new e
161
161
  end
162
162
  when :bool
163
163
  begin
164
164
  !!v # TODO validation
165
165
  rescue => e
166
- raise ConfigError, e
166
+ raise ConfigError.new e
167
167
  end
168
168
  when :hash
169
- raise ConfigError, "Invalid value for :hash" unless v.is_a?(Hash)
169
+ raise ConfigError.new "Invalid value for :hash" unless v.is_a?(Hash)
170
170
  DataSource.new.merge!(v)
171
171
  when :array
172
- raise ConfigError, "Invalid value for :array" unless v.is_a?(Array)
172
+ raise ConfigError.new "Invalid value for :array" unless v.is_a?(Array)
173
173
  v.dup
174
174
  else
175
175
  unless type.respond_to?(:load)
@@ -178,7 +178,7 @@ module Embulk
178
178
  begin
179
179
  type.load(v)
180
180
  rescue => e
181
- raise ConfigError, e
181
+ raise ConfigError.new e
182
182
  end
183
183
  end
184
184
 
@@ -186,34 +186,32 @@ module Embulk
186
186
  value = options[:default]
187
187
 
188
188
  else
189
- raise ConfigError, "Required field #{key.to_s.dump} is not set"
189
+ raise ConfigError.new "Required field #{key.to_s.dump} is not set"
190
190
  end
191
191
 
192
192
  return value
193
193
  end
194
194
 
195
- if Embulk.java?
196
- def self.from_java(java_data_source_impl)
197
- json = java_data_source_impl.toString
198
- new.merge!(JSON.parse(json))
199
- end
195
+ def self.from_java(java_data_source_impl)
196
+ json = java_data_source_impl.toString
197
+ new.merge!(JSON.parse(json))
198
+ end
200
199
 
201
- def self.from_ruby_hash(hash)
202
- new.merge!(hash)
203
- end
200
+ def self.from_ruby_hash(hash)
201
+ new.merge!(hash)
202
+ end
204
203
 
205
- def to_java
206
- json = to_json
207
- Java::Injected::ModelManager.readObject(Java::DataSourceImpl.java_class, json.to_java)
208
- end
204
+ def to_java
205
+ json = to_json
206
+ Java::Injected::ModelManager.readObject(Java::DataSourceImpl.java_class, json.to_java)
207
+ end
209
208
 
210
- def load_config(task_type)
211
- Java::Injected::ModelManager.readObjectWithConfigSerDe(task_type.java_class, to_json.to_java)
212
- end
209
+ def load_config(task_type)
210
+ Java::Injected::ModelManager.readObjectWithConfigSerDe(task_type.java_class, to_json.to_java)
211
+ end
213
212
 
214
- def load_task(task_type)
215
- Java::Injected::ModelManager.readObject(task_type.java_class, to_json.to_java)
216
- end
213
+ def load_task(task_type)
214
+ Java::Injected::ModelManager.readObject(task_type.java_class, to_json.to_java)
217
215
  end
218
216
  end
219
217