embulk 0.7.2-java → 0.7.3-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/build.gradle +1 -1
- data/embulk-docs/src/release.rst +1 -0
- data/embulk-docs/src/release/release-0.7.3.rst +21 -0
- data/lib/embulk/buffer.rb +12 -14
- data/lib/embulk/column.rb +31 -35
- data/lib/embulk/command/embulk_migrate_plugin.rb +11 -5
- data/lib/embulk/command/embulk_run.rb +11 -10
- data/lib/embulk/command/embulk_selfupdate.rb +54 -19
- data/lib/embulk/data_source.rb +24 -26
- data/lib/embulk/decoder_plugin.rb +7 -9
- data/lib/embulk/encoder_plugin.rb +7 -9
- data/lib/embulk/error.rb +3 -0
- data/lib/embulk/exec.rb +1 -1
- data/lib/embulk/executor_plugin.rb +7 -9
- data/lib/embulk/file_input_plugin.rb +11 -13
- data/lib/embulk/file_output_plugin.rb +11 -13
- data/lib/embulk/filter_plugin.rb +49 -51
- data/lib/embulk/formatter_plugin.rb +50 -52
- data/lib/embulk/guess_plugin.rb +25 -27
- data/lib/embulk/input_plugin.rb +61 -63
- data/lib/embulk/output_plugin.rb +75 -77
- data/lib/embulk/parser_plugin.rb +34 -36
- data/lib/embulk/plugin.rb +2 -2
- data/lib/embulk/schema.rb +6 -8
- data/lib/embulk/version.rb +1 -1
- metadata +5 -4
data/lib/embulk/input_plugin.rb
CHANGED
@@ -38,83 +38,81 @@ module Embulk
|
|
38
38
|
raise NotImplementedError, "InputPlugin#run must be implemented"
|
39
39
|
end
|
40
40
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
end
|
41
|
+
def self.new_java
|
42
|
+
JavaAdapter.new(self)
|
43
|
+
end
|
45
44
|
|
46
|
-
|
47
|
-
|
45
|
+
class JavaAdapter
|
46
|
+
include Java::InputPlugin
|
48
47
|
|
49
|
-
|
50
|
-
|
51
|
-
|
48
|
+
def initialize(ruby_class)
|
49
|
+
@ruby_class = ruby_class
|
50
|
+
end
|
52
51
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
end
|
63
|
-
# TODO check return type of #transaction
|
64
|
-
return DataSource.from_ruby_hash(config_diff_hash).to_java
|
52
|
+
def transaction(java_config, java_control)
|
53
|
+
config = DataSource.from_java(java_config)
|
54
|
+
config_diff_hash = @ruby_class.transaction(config) do |task_source_hash,columns,task_count|
|
55
|
+
java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
|
56
|
+
java_schema = Schema.new(columns).to_java
|
57
|
+
java_task_reports = java_control.run(java_task_source, java_schema, task_count)
|
58
|
+
java_task_reports.map {|java_task_report|
|
59
|
+
DataSource.from_java(java_task_report)
|
60
|
+
}
|
65
61
|
end
|
62
|
+
# TODO check return type of #transaction
|
63
|
+
return DataSource.from_ruby_hash(config_diff_hash).to_java
|
64
|
+
end
|
66
65
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
end
|
78
|
-
# TODO check return type of #resume
|
79
|
-
return DataSource.from_ruby_hash(config_diff_hash).to_java
|
66
|
+
def resume(java_task_source, java_schema, task_count, java_control)
|
67
|
+
task_source = DataSource.from_java(java_task_source)
|
68
|
+
schema = Schema.from_java(java_schema)
|
69
|
+
config_diff_hash = @ruby_class.resume(task_source, schema, task_count) do |task_source_hash,columns,task_count|
|
70
|
+
java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
|
71
|
+
java_schema = Schema.new(columns).to_java
|
72
|
+
java_task_reports = java_control.run(java_task_source, java_schema, task_count)
|
73
|
+
java_task_reports.map {|java_task_report|
|
74
|
+
DataSource.from_java(java_task_report)
|
75
|
+
}
|
80
76
|
end
|
77
|
+
# TODO check return type of #resume
|
78
|
+
return DataSource.from_ruby_hash(config_diff_hash).to_java
|
79
|
+
end
|
81
80
|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
81
|
+
def cleanup(java_task_source, java_schema, task_count, java_task_reports)
|
82
|
+
task_source = DataSource.from_java(java_task_source)
|
83
|
+
schema = Schema.from_java(java_schema)
|
84
|
+
task_reports = java_task_reports.map {|c| DataSource.from_java(c) }
|
85
|
+
@ruby_class.cleanup(task_source, schema, task_count, task_reports)
|
86
|
+
return nil
|
87
|
+
end
|
89
88
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
89
|
+
def guess(java_config)
|
90
|
+
config = DataSource.from_java(java_config)
|
91
|
+
config_diff_hash = @ruby_class.guess(config)
|
92
|
+
return DataSource.from_ruby_hash(config_diff_hash).to_java
|
93
|
+
end
|
95
94
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
end
|
95
|
+
def run(java_task_source, java_schema, processor_index, java_output)
|
96
|
+
task_source = DataSource.from_java(java_task_source)
|
97
|
+
schema = Schema.from_java(java_schema)
|
98
|
+
page_builder = PageBuilder.new(schema, java_output)
|
99
|
+
begin
|
100
|
+
task_report_hash = @ruby_class.new(task_source, schema, processor_index, page_builder).run
|
101
|
+
return DataSource.from_ruby_hash(task_report_hash).to_java
|
102
|
+
ensure
|
103
|
+
page_builder.close
|
106
104
|
end
|
107
105
|
end
|
106
|
+
end
|
108
107
|
|
109
|
-
|
110
|
-
|
111
|
-
|
108
|
+
def self.from_java(java_class)
|
109
|
+
JavaPlugin.ruby_adapter_class(java_class, InputPlugin, RubyAdapter)
|
110
|
+
end
|
112
111
|
|
113
|
-
|
114
|
-
|
115
|
-
end
|
116
|
-
# TODO
|
112
|
+
module RubyAdapter
|
113
|
+
module ClassMethods
|
117
114
|
end
|
115
|
+
# TODO
|
118
116
|
end
|
119
117
|
end
|
120
118
|
|
data/lib/embulk/output_plugin.rb
CHANGED
@@ -47,103 +47,101 @@ module Embulk
|
|
47
47
|
{}
|
48
48
|
end
|
49
49
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
end
|
50
|
+
def self.new_java
|
51
|
+
JavaAdapter.new(self)
|
52
|
+
end
|
54
53
|
|
55
|
-
|
56
|
-
|
54
|
+
class JavaAdapter
|
55
|
+
include Java::OutputPlugin
|
57
56
|
|
58
|
-
|
59
|
-
|
60
|
-
|
57
|
+
def initialize(ruby_class)
|
58
|
+
@ruby_class = ruby_class
|
59
|
+
end
|
61
60
|
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
end
|
72
|
-
# TODO check return type of #transaction
|
73
|
-
return DataSource.from_ruby_hash(config_diff_hash).to_java
|
61
|
+
def transaction(java_config, java_schema, task_count, java_control)
|
62
|
+
config = DataSource.from_java(java_config)
|
63
|
+
schema = Schema.from_java(java_schema)
|
64
|
+
config_diff_hash = @ruby_class.transaction(config, schema, task_count) do |task_source_hash|
|
65
|
+
java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
|
66
|
+
java_task_reports = java_control.run(java_task_source)
|
67
|
+
java_task_reports.map {|java_task_report|
|
68
|
+
DataSource.from_java(java_task_report)
|
69
|
+
}
|
74
70
|
end
|
71
|
+
# TODO check return type of #transaction
|
72
|
+
return DataSource.from_ruby_hash(config_diff_hash).to_java
|
73
|
+
end
|
75
74
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
end
|
86
|
-
# TODO check return type of #resume
|
87
|
-
return DataSource.from_ruby_hash(config_diff_hash).to_java
|
75
|
+
def resume(java_task_source, java_schema, task_count, java_control)
|
76
|
+
task_source = DataSource.from_java(java_task_source)
|
77
|
+
schema = Schema.from_java(java_schema)
|
78
|
+
config_diff_hash = @ruby_class.resume(task_source, schema, task_count) do |task_source_hash,columns,task_count|
|
79
|
+
java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
|
80
|
+
java_task_reports = java_control.run(java_task_source)
|
81
|
+
java_task_reports.map {|java_task_report|
|
82
|
+
DataSource.from_java(java_task_report)
|
83
|
+
}
|
88
84
|
end
|
85
|
+
# TODO check return type of #resume
|
86
|
+
return DataSource.from_ruby_hash(config_diff_hash).to_java
|
87
|
+
end
|
89
88
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
89
|
+
def cleanup(java_task_source, java_schema, task_count, java_task_reports)
|
90
|
+
task_source = DataSource.from_java(java_task_source)
|
91
|
+
schema = Schema.from_java(java_schema)
|
92
|
+
task_reports = java_task_reports.map {|c| DataSource.from_java(c) }
|
93
|
+
@ruby_class.cleanup(task_source, schema, task_count, task_reports)
|
94
|
+
return nil
|
95
|
+
end
|
97
96
|
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
97
|
+
def open(java_task_source, java_schema, processor_index)
|
98
|
+
task_source = DataSource.from_java(java_task_source)
|
99
|
+
schema = Schema.from_java(java_schema)
|
100
|
+
ruby_object = @ruby_class.new(task_source, schema, processor_index)
|
101
|
+
return OutputAdapter.new(ruby_object, schema)
|
102
|
+
end
|
104
103
|
|
105
|
-
|
106
|
-
|
104
|
+
class OutputAdapter
|
105
|
+
include Java::TransactionalPageOutput
|
107
106
|
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
107
|
+
def initialize(ruby_object, schema)
|
108
|
+
@ruby_object = ruby_object
|
109
|
+
@schema = schema
|
110
|
+
end
|
112
111
|
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
112
|
+
def add(java_page)
|
113
|
+
# TODO reuse page reader
|
114
|
+
@ruby_object.add Page.new(java_page, @schema)
|
115
|
+
end
|
117
116
|
|
118
|
-
|
119
|
-
|
120
|
-
|
117
|
+
def finish
|
118
|
+
@ruby_object.finish
|
119
|
+
end
|
121
120
|
|
122
|
-
|
123
|
-
|
124
|
-
|
121
|
+
def close
|
122
|
+
@ruby_object.close
|
123
|
+
end
|
125
124
|
|
126
|
-
|
127
|
-
|
128
|
-
|
125
|
+
def abort
|
126
|
+
@ruby_object.abort
|
127
|
+
end
|
129
128
|
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
end
|
129
|
+
def commit
|
130
|
+
task_report_hash = @ruby_object.commit
|
131
|
+
return DataSource.from_ruby_hash(task_report_hash).to_java
|
134
132
|
end
|
135
133
|
end
|
134
|
+
end
|
136
135
|
|
137
|
-
|
138
|
-
|
139
|
-
|
136
|
+
def self.from_java(java_class)
|
137
|
+
JavaPlugin.ruby_adapter_class(java_class, OutputPlugin, RubyAdapter)
|
138
|
+
end
|
140
139
|
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
end
|
145
|
-
# TODO add, finish, close, abort, commit
|
140
|
+
module RubyAdapter
|
141
|
+
module ClassMethods
|
142
|
+
# TODO transaction, resume, cleanup
|
146
143
|
end
|
144
|
+
# TODO add, finish, close, abort, commit
|
147
145
|
end
|
148
146
|
end
|
149
147
|
|
data/lib/embulk/parser_plugin.rb
CHANGED
@@ -26,52 +26,50 @@ module Embulk
|
|
26
26
|
raise NotImplementedError, "ParserPlugin#run(file_input) must be implemented"
|
27
27
|
end
|
28
28
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
end
|
29
|
+
def self.new_java
|
30
|
+
JavaAdapter.new(self)
|
31
|
+
end
|
33
32
|
|
34
|
-
|
35
|
-
|
33
|
+
class JavaAdapter
|
34
|
+
include Java::ParserPlugin
|
36
35
|
|
37
|
-
|
38
|
-
|
39
|
-
|
36
|
+
def initialize(ruby_class)
|
37
|
+
@ruby_class = ruby_class
|
38
|
+
end
|
40
39
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
end
|
48
|
-
nil
|
40
|
+
def transaction(java_config, java_control)
|
41
|
+
config = DataSource.from_java(java_config)
|
42
|
+
@ruby_class.transaction(config) do |task_source_hash,columns|
|
43
|
+
java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
|
44
|
+
java_schema = Schema.new(columns).to_java
|
45
|
+
java_control.run(java_task_source, java_schema)
|
49
46
|
end
|
47
|
+
nil
|
48
|
+
end
|
50
49
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
end
|
50
|
+
def run(java_task_source, java_schema, java_file_input, java_output)
|
51
|
+
task_source = DataSource.from_java(java_task_source)
|
52
|
+
schema = Schema.from_java(java_schema)
|
53
|
+
file_input = FileInput.new(java_file_input)
|
54
|
+
page_builder = PageBuilder.new(schema, java_output)
|
55
|
+
begin
|
56
|
+
@ruby_class.new(task_source, schema, page_builder).run(file_input)
|
57
|
+
nil
|
58
|
+
ensure
|
59
|
+
page_builder.close
|
60
|
+
# FileInput is closed by FileInputRunner
|
63
61
|
end
|
64
62
|
end
|
63
|
+
end
|
65
64
|
|
66
|
-
|
67
|
-
|
68
|
-
|
65
|
+
def self.from_java(java_class)
|
66
|
+
JavaPlugin.ruby_adapter_class(java_class, ParserPlugin, RubyAdapter)
|
67
|
+
end
|
69
68
|
|
70
|
-
|
71
|
-
|
72
|
-
end
|
73
|
-
# TODO
|
69
|
+
module RubyAdapter
|
70
|
+
module ClassMethods
|
74
71
|
end
|
72
|
+
# TODO
|
75
73
|
end
|
76
74
|
end
|
77
75
|
|
data/lib/embulk/plugin.rb
CHANGED
@@ -14,8 +14,8 @@ module Embulk
|
|
14
14
|
require 'embulk/encoder_plugin'
|
15
15
|
require 'embulk/guess_plugin'
|
16
16
|
require 'embulk/executor_plugin'
|
17
|
-
require 'embulk/java_plugin'
|
18
|
-
require 'embulk/exec'
|
17
|
+
require 'embulk/java_plugin'
|
18
|
+
require 'embulk/exec'
|
19
19
|
|
20
20
|
class PluginManager
|
21
21
|
def initialize
|
data/lib/embulk/schema.rb
CHANGED
@@ -88,15 +88,13 @@ module Embulk
|
|
88
88
|
@record_writer.call(page_builder, record)
|
89
89
|
end
|
90
90
|
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
end
|
91
|
+
def self.from_java(java_schema)
|
92
|
+
new java_schema.getColumns.map {|column| Column.from_java(column) }
|
93
|
+
end
|
95
94
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
end
|
95
|
+
def to_java
|
96
|
+
columns = self.map {|column| column.to_java }
|
97
|
+
Java::Schema.new(columns)
|
100
98
|
end
|
101
99
|
end
|
102
100
|
|
data/lib/embulk/version.rb
CHANGED