embulk 0.7.2 → 0.7.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/build.gradle +1 -1
- data/embulk-docs/src/release.rst +1 -0
- data/embulk-docs/src/release/release-0.7.3.rst +21 -0
- data/lib/embulk/buffer.rb +12 -14
- data/lib/embulk/column.rb +31 -35
- data/lib/embulk/command/embulk_migrate_plugin.rb +11 -5
- data/lib/embulk/command/embulk_run.rb +11 -10
- data/lib/embulk/command/embulk_selfupdate.rb +54 -19
- data/lib/embulk/data_source.rb +24 -26
- data/lib/embulk/decoder_plugin.rb +7 -9
- data/lib/embulk/encoder_plugin.rb +7 -9
- data/lib/embulk/error.rb +3 -0
- data/lib/embulk/exec.rb +1 -1
- data/lib/embulk/executor_plugin.rb +7 -9
- data/lib/embulk/file_input_plugin.rb +11 -13
- data/lib/embulk/file_output_plugin.rb +11 -13
- data/lib/embulk/filter_plugin.rb +49 -51
- data/lib/embulk/formatter_plugin.rb +50 -52
- data/lib/embulk/guess_plugin.rb +25 -27
- data/lib/embulk/input_plugin.rb +61 -63
- data/lib/embulk/output_plugin.rb +75 -77
- data/lib/embulk/parser_plugin.rb +34 -36
- data/lib/embulk/plugin.rb +2 -2
- data/lib/embulk/schema.rb +6 -8
- data/lib/embulk/version.rb +1 -1
- metadata +5 -4
@@ -9,18 +9,16 @@ module Embulk
|
|
9
9
|
|
10
10
|
# TODO
|
11
11
|
|
12
|
-
|
13
|
-
# TODO new_java
|
12
|
+
# TODO new_java
|
14
13
|
|
15
|
-
|
16
|
-
|
17
|
-
|
14
|
+
def self.from_java(java_class)
|
15
|
+
JavaPlugin.ruby_adapter_class(java_class, DecoderPlugin, RubyAdapter)
|
16
|
+
end
|
18
17
|
|
19
|
-
|
20
|
-
|
21
|
-
end
|
22
|
-
# TODO
|
18
|
+
module RubyAdapter
|
19
|
+
module ClassMethods
|
23
20
|
end
|
21
|
+
# TODO
|
24
22
|
end
|
25
23
|
end
|
26
24
|
|
@@ -9,18 +9,16 @@ module Embulk
|
|
9
9
|
|
10
10
|
# TODO
|
11
11
|
|
12
|
-
|
13
|
-
# TODO new_java
|
12
|
+
# TODO new_java
|
14
13
|
|
15
|
-
|
16
|
-
|
17
|
-
|
14
|
+
def self.from_java(java_class)
|
15
|
+
JavaPlugin.ruby_adapter_class(java_class, EncoderPlugin, RubyAdapter)
|
16
|
+
end
|
18
17
|
|
19
|
-
|
20
|
-
|
21
|
-
end
|
22
|
-
# TODO
|
18
|
+
module RubyAdapter
|
19
|
+
module ClassMethods
|
23
20
|
end
|
21
|
+
# TODO
|
24
22
|
end
|
25
23
|
end
|
26
24
|
|
data/lib/embulk/error.rb
CHANGED
data/lib/embulk/exec.rb
CHANGED
@@ -5,18 +5,16 @@ module Embulk
|
|
5
5
|
class ExecutorPlugin
|
6
6
|
# TODO
|
7
7
|
|
8
|
-
|
9
|
-
# TODO new_java
|
8
|
+
# TODO new_java
|
10
9
|
|
11
|
-
|
12
|
-
|
13
|
-
|
10
|
+
def self.from_java(java_class)
|
11
|
+
JavaPlugin.ruby_adapter_class(java_class, ExecutorPlugin, RubyAdapter)
|
12
|
+
end
|
14
13
|
|
15
|
-
|
16
|
-
|
17
|
-
end
|
18
|
-
# TODO
|
14
|
+
module RubyAdapter
|
15
|
+
module ClassMethods
|
19
16
|
end
|
17
|
+
# TODO
|
20
18
|
end
|
21
19
|
end
|
22
20
|
|
@@ -4,23 +4,21 @@ module Embulk
|
|
4
4
|
# TODO transaction, resume, cleanup
|
5
5
|
# TODO run
|
6
6
|
|
7
|
-
|
8
|
-
# TODO to_java
|
7
|
+
# TODO to_java
|
9
8
|
|
10
|
-
|
11
|
-
|
12
|
-
|
9
|
+
def self.from_java(java_class)
|
10
|
+
JavaPlugin.ruby_adapter_class(java_class, FileInputPlugin, RubyAdapter)
|
11
|
+
end
|
13
12
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
end
|
19
|
-
# TODO transaction, resume, cleanup
|
13
|
+
module RubyAdapter
|
14
|
+
module ClassMethods
|
15
|
+
def new_java
|
16
|
+
Java::FileInputRunner.new(Java.injector.getInstance(java_class))
|
20
17
|
end
|
21
|
-
|
22
|
-
# TODO run
|
18
|
+
# TODO transaction, resume, cleanup
|
23
19
|
end
|
20
|
+
|
21
|
+
# TODO run
|
24
22
|
end
|
25
23
|
end
|
26
24
|
|
@@ -4,23 +4,21 @@ module Embulk
|
|
4
4
|
# TODO transaction, resume, cleanup
|
5
5
|
# TODO add, finish, close, abort, commit
|
6
6
|
|
7
|
-
|
8
|
-
# TODO to_java
|
7
|
+
# TODO to_java
|
9
8
|
|
10
|
-
|
11
|
-
|
12
|
-
|
9
|
+
def self.from_java(java_class)
|
10
|
+
JavaPlugin.ruby_adapter_class(java_class, FileOutputPlugin, RubyAdapter)
|
11
|
+
end
|
13
12
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
end
|
19
|
-
# TODO transaction, resume, cleanup
|
13
|
+
module RubyAdapter
|
14
|
+
module ClassMethods
|
15
|
+
def new_java
|
16
|
+
Java::FileOutputRunner.new(Java.injector.getInstance(java_class))
|
20
17
|
end
|
21
|
-
|
22
|
-
# TODO add, finish, close, abort, commit
|
18
|
+
# TODO transaction, resume, cleanup
|
23
19
|
end
|
20
|
+
|
21
|
+
# TODO add, finish, close, abort, commit
|
24
22
|
end
|
25
23
|
end
|
26
24
|
|
data/lib/embulk/filter_plugin.rb
CHANGED
@@ -34,72 +34,70 @@ module Embulk
|
|
34
34
|
def close
|
35
35
|
end
|
36
36
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
end
|
37
|
+
def self.new_java
|
38
|
+
JavaAdapter.new(self)
|
39
|
+
end
|
41
40
|
|
42
|
-
|
43
|
-
|
41
|
+
class JavaAdapter
|
42
|
+
include Java::FilterPlugin
|
44
43
|
|
45
|
-
|
46
|
-
|
47
|
-
|
44
|
+
def initialize(ruby_class)
|
45
|
+
@ruby_class = ruby_class
|
46
|
+
end
|
48
47
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
end
|
57
|
-
nil
|
48
|
+
def transaction(java_config, java_in_schema, java_control)
|
49
|
+
config = DataSource.from_java(java_config)
|
50
|
+
in_schema = Schema.from_java(java_in_schema)
|
51
|
+
@ruby_class.transaction(config, in_schema) do |task_source_hash, out_columns|
|
52
|
+
java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
|
53
|
+
java_out_schemas = Schema.new(out_columns).to_java
|
54
|
+
java_control.run(java_task_source, java_out_schemas)
|
58
55
|
end
|
56
|
+
nil
|
57
|
+
end
|
59
58
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
59
|
+
def open(java_task_source, java_in_schema, java_out_schema, java_output)
|
60
|
+
task_source = DataSource.from_java(java_task_source)
|
61
|
+
in_schema = Schema.from_java(java_in_schema)
|
62
|
+
out_schema = Schema.from_java(java_out_schema)
|
63
|
+
page_builder = PageBuilder.new(out_schema, java_output)
|
64
|
+
ruby_object = @ruby_class.new(task_source, in_schema, out_schema, page_builder)
|
65
|
+
return OutputAdapter.new(ruby_object, in_schema, page_builder)
|
66
|
+
end
|
68
67
|
|
69
|
-
|
70
|
-
|
68
|
+
class OutputAdapter
|
69
|
+
include Java::TransactionalPageOutput
|
71
70
|
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
71
|
+
def initialize(ruby_object, in_schema, page_builder)
|
72
|
+
@ruby_object = ruby_object
|
73
|
+
@in_schema = in_schema
|
74
|
+
@page_builder = page_builder
|
75
|
+
end
|
77
76
|
|
78
|
-
|
79
|
-
|
80
|
-
|
77
|
+
def add(java_page)
|
78
|
+
@ruby_object.add Page.new(java_page, @in_schema)
|
79
|
+
end
|
81
80
|
|
82
|
-
|
83
|
-
|
84
|
-
|
81
|
+
def finish
|
82
|
+
@ruby_object.finish
|
83
|
+
end
|
85
84
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
end
|
85
|
+
def close
|
86
|
+
@ruby_object.close
|
87
|
+
ensure
|
88
|
+
@page_builder.close
|
91
89
|
end
|
92
90
|
end
|
91
|
+
end
|
93
92
|
|
94
|
-
|
95
|
-
|
96
|
-
|
93
|
+
def self.from_java(java_class)
|
94
|
+
JavaPlugin.ruby_adapter_class(java_class, FilterPlugin, RubyAdapter)
|
95
|
+
end
|
97
96
|
|
98
|
-
|
99
|
-
|
100
|
-
end
|
101
|
-
# TODO
|
97
|
+
module RubyAdapter
|
98
|
+
module ClassMethods
|
102
99
|
end
|
100
|
+
# TODO
|
103
101
|
end
|
104
102
|
end
|
105
103
|
end
|
@@ -33,72 +33,70 @@ module Embulk
|
|
33
33
|
def close
|
34
34
|
end
|
35
35
|
|
36
|
-
|
37
|
-
|
38
|
-
|
36
|
+
def self.new_java
|
37
|
+
JavaAdapter.new(self)
|
38
|
+
end
|
39
|
+
|
40
|
+
class JavaAdapter
|
41
|
+
include Java::FormatterPlugin
|
42
|
+
|
43
|
+
def initialize(ruby_class)
|
44
|
+
@ruby_class = ruby_class
|
45
|
+
end
|
46
|
+
|
47
|
+
def transaction(java_config, java_schema, java_control)
|
48
|
+
config = DataSource.from_java(java_config)
|
49
|
+
schema = Schema.from_java(java_schema)
|
50
|
+
@ruby_class.transaction(config, schema) do |task_source_hash|
|
51
|
+
java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
|
52
|
+
java_control.run(java_task_source)
|
53
|
+
end
|
54
|
+
nil
|
39
55
|
end
|
40
56
|
|
41
|
-
|
42
|
-
|
57
|
+
def open(java_task_source, java_schema, java_file_output)
|
58
|
+
task_source = DataSource.from_java(java_task_source)
|
59
|
+
schema = Schema.from_java(java_schema)
|
60
|
+
file_output = FileOutput.new(java_file_output)
|
61
|
+
ruby_object = @ruby_class.new(task_source, schema, file_output)
|
62
|
+
return OutputAdapter.new(ruby_object, schema, file_output)
|
63
|
+
end
|
64
|
+
|
65
|
+
class OutputAdapter
|
66
|
+
include Java::TransactionalPageOutput
|
43
67
|
|
44
|
-
def initialize(
|
45
|
-
@
|
68
|
+
def initialize(ruby_object, schema, file_output)
|
69
|
+
@ruby_object = ruby_object
|
70
|
+
@schema = schema
|
71
|
+
@file_output = file_output
|
46
72
|
end
|
47
73
|
|
48
|
-
def
|
49
|
-
|
50
|
-
|
51
|
-
@ruby_class.transaction(config, schema) do |task_source_hash|
|
52
|
-
java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
|
53
|
-
java_control.run(java_task_source)
|
54
|
-
end
|
55
|
-
nil
|
74
|
+
def add(java_page)
|
75
|
+
# TODO reuse page reader
|
76
|
+
@ruby_object.add Page.new(java_page, @schema)
|
56
77
|
end
|
57
78
|
|
58
|
-
def
|
59
|
-
|
60
|
-
schema = Schema.from_java(java_schema)
|
61
|
-
file_output = FileOutput.new(java_file_output)
|
62
|
-
ruby_object = @ruby_class.new(task_source, schema, file_output)
|
63
|
-
return OutputAdapter.new(ruby_object, schema, file_output)
|
79
|
+
def finish
|
80
|
+
@ruby_object.finish
|
64
81
|
end
|
65
82
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
@ruby_object = ruby_object
|
71
|
-
@schema = schema
|
72
|
-
@file_output = file_output
|
73
|
-
end
|
74
|
-
|
75
|
-
def add(java_page)
|
76
|
-
# TODO reuse page reader
|
77
|
-
@ruby_object.add Page.new(java_page, @schema)
|
78
|
-
end
|
79
|
-
|
80
|
-
def finish
|
81
|
-
@ruby_object.finish
|
82
|
-
end
|
83
|
-
|
84
|
-
def close
|
85
|
-
@ruby_object.close
|
86
|
-
ensure
|
87
|
-
@file_output.close
|
88
|
-
end
|
83
|
+
def close
|
84
|
+
@ruby_object.close
|
85
|
+
ensure
|
86
|
+
@file_output.close
|
89
87
|
end
|
90
88
|
end
|
89
|
+
end
|
91
90
|
|
92
|
-
|
93
|
-
|
94
|
-
|
91
|
+
def self.from_java(java_class)
|
92
|
+
JavaPlugin.ruby_adapter_class(java_class, FormatterPlugin, RubyAdapter)
|
93
|
+
end
|
95
94
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
end
|
100
|
-
# TODO add, finish, close
|
95
|
+
module RubyAdapter
|
96
|
+
module ClassMethods
|
97
|
+
# TODO transaction, resume, cleanup
|
101
98
|
end
|
99
|
+
# TODO add, finish, close
|
102
100
|
end
|
103
101
|
end
|
104
102
|
|
data/lib/embulk/guess_plugin.rb
CHANGED
@@ -7,40 +7,38 @@ module Embulk
|
|
7
7
|
raise NotImplementedError, "GuessPlugin#guess(config, sample_buffer) must be implemented"
|
8
8
|
end
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
end
|
14
|
-
|
15
|
-
class JavaAdapter
|
16
|
-
include Java::GuessPlugin
|
10
|
+
def self.new_java
|
11
|
+
JavaAdapter.new(new)
|
12
|
+
end
|
17
13
|
|
18
|
-
|
19
|
-
|
20
|
-
end
|
14
|
+
class JavaAdapter
|
15
|
+
include Java::GuessPlugin
|
21
16
|
|
22
|
-
|
23
|
-
|
24
|
-
sample = Buffer.from_java(java_sample)
|
25
|
-
config_diff_hash = @ruby_guess.guess(config, sample)
|
26
|
-
return DataSource.from_ruby_hash(config_diff_hash).to_java
|
27
|
-
end
|
17
|
+
def initialize(ruby_guess)
|
18
|
+
@ruby_guess = ruby_guess
|
28
19
|
end
|
29
20
|
|
30
|
-
def
|
31
|
-
|
21
|
+
def guess(java_config, java_sample)
|
22
|
+
config = DataSource.from_java(java_config)
|
23
|
+
sample = Buffer.from_java(java_sample)
|
24
|
+
config_diff_hash = @ruby_guess.guess(config, sample)
|
25
|
+
return DataSource.from_ruby_hash(config_diff_hash).to_java
|
32
26
|
end
|
27
|
+
end
|
33
28
|
|
34
|
-
|
35
|
-
|
36
|
-
|
29
|
+
def self.from_java(java_class)
|
30
|
+
JavaPlugin.ruby_adapter(java_class, GuessPlugin, RubyAdapter)
|
31
|
+
end
|
37
32
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
33
|
+
module RubyAdapter
|
34
|
+
module ClassMethods
|
35
|
+
end
|
36
|
+
|
37
|
+
def guess(config, sample)
|
38
|
+
java_config = config.to_java
|
39
|
+
java_sample = sample.to_java
|
40
|
+
java_config_diff = java_object.guess(java_config, java_sample)
|
41
|
+
return DataSource.from_java(java_config_diff)
|
44
42
|
end
|
45
43
|
end
|
46
44
|
end
|