embulk 0.7.2 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/build.gradle +1 -1
- data/embulk-docs/src/release.rst +1 -0
- data/embulk-docs/src/release/release-0.7.3.rst +21 -0
- data/lib/embulk/buffer.rb +12 -14
- data/lib/embulk/column.rb +31 -35
- data/lib/embulk/command/embulk_migrate_plugin.rb +11 -5
- data/lib/embulk/command/embulk_run.rb +11 -10
- data/lib/embulk/command/embulk_selfupdate.rb +54 -19
- data/lib/embulk/data_source.rb +24 -26
- data/lib/embulk/decoder_plugin.rb +7 -9
- data/lib/embulk/encoder_plugin.rb +7 -9
- data/lib/embulk/error.rb +3 -0
- data/lib/embulk/exec.rb +1 -1
- data/lib/embulk/executor_plugin.rb +7 -9
- data/lib/embulk/file_input_plugin.rb +11 -13
- data/lib/embulk/file_output_plugin.rb +11 -13
- data/lib/embulk/filter_plugin.rb +49 -51
- data/lib/embulk/formatter_plugin.rb +50 -52
- data/lib/embulk/guess_plugin.rb +25 -27
- data/lib/embulk/input_plugin.rb +61 -63
- data/lib/embulk/output_plugin.rb +75 -77
- data/lib/embulk/parser_plugin.rb +34 -36
- data/lib/embulk/plugin.rb +2 -2
- data/lib/embulk/schema.rb +6 -8
- data/lib/embulk/version.rb +1 -1
- metadata +5 -4
@@ -9,18 +9,16 @@ module Embulk
|
|
9
9
|
|
10
10
|
# TODO
|
11
11
|
|
12
|
-
|
13
|
-
# TODO new_java
|
12
|
+
# TODO new_java
|
14
13
|
|
15
|
-
|
16
|
-
|
17
|
-
|
14
|
+
def self.from_java(java_class)
|
15
|
+
JavaPlugin.ruby_adapter_class(java_class, DecoderPlugin, RubyAdapter)
|
16
|
+
end
|
18
17
|
|
19
|
-
|
20
|
-
|
21
|
-
end
|
22
|
-
# TODO
|
18
|
+
module RubyAdapter
|
19
|
+
module ClassMethods
|
23
20
|
end
|
21
|
+
# TODO
|
24
22
|
end
|
25
23
|
end
|
26
24
|
|
@@ -9,18 +9,16 @@ module Embulk
|
|
9
9
|
|
10
10
|
# TODO
|
11
11
|
|
12
|
-
|
13
|
-
# TODO new_java
|
12
|
+
# TODO new_java
|
14
13
|
|
15
|
-
|
16
|
-
|
17
|
-
|
14
|
+
def self.from_java(java_class)
|
15
|
+
JavaPlugin.ruby_adapter_class(java_class, EncoderPlugin, RubyAdapter)
|
16
|
+
end
|
18
17
|
|
19
|
-
|
20
|
-
|
21
|
-
end
|
22
|
-
# TODO
|
18
|
+
module RubyAdapter
|
19
|
+
module ClassMethods
|
23
20
|
end
|
21
|
+
# TODO
|
24
22
|
end
|
25
23
|
end
|
26
24
|
|
data/lib/embulk/error.rb
CHANGED
data/lib/embulk/exec.rb
CHANGED
@@ -5,18 +5,16 @@ module Embulk
|
|
5
5
|
class ExecutorPlugin
|
6
6
|
# TODO
|
7
7
|
|
8
|
-
|
9
|
-
# TODO new_java
|
8
|
+
# TODO new_java
|
10
9
|
|
11
|
-
|
12
|
-
|
13
|
-
|
10
|
+
def self.from_java(java_class)
|
11
|
+
JavaPlugin.ruby_adapter_class(java_class, ExecutorPlugin, RubyAdapter)
|
12
|
+
end
|
14
13
|
|
15
|
-
|
16
|
-
|
17
|
-
end
|
18
|
-
# TODO
|
14
|
+
module RubyAdapter
|
15
|
+
module ClassMethods
|
19
16
|
end
|
17
|
+
# TODO
|
20
18
|
end
|
21
19
|
end
|
22
20
|
|
@@ -4,23 +4,21 @@ module Embulk
|
|
4
4
|
# TODO transaction, resume, cleanup
|
5
5
|
# TODO run
|
6
6
|
|
7
|
-
|
8
|
-
# TODO to_java
|
7
|
+
# TODO to_java
|
9
8
|
|
10
|
-
|
11
|
-
|
12
|
-
|
9
|
+
def self.from_java(java_class)
|
10
|
+
JavaPlugin.ruby_adapter_class(java_class, FileInputPlugin, RubyAdapter)
|
11
|
+
end
|
13
12
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
end
|
19
|
-
# TODO transaction, resume, cleanup
|
13
|
+
module RubyAdapter
|
14
|
+
module ClassMethods
|
15
|
+
def new_java
|
16
|
+
Java::FileInputRunner.new(Java.injector.getInstance(java_class))
|
20
17
|
end
|
21
|
-
|
22
|
-
# TODO run
|
18
|
+
# TODO transaction, resume, cleanup
|
23
19
|
end
|
20
|
+
|
21
|
+
# TODO run
|
24
22
|
end
|
25
23
|
end
|
26
24
|
|
@@ -4,23 +4,21 @@ module Embulk
|
|
4
4
|
# TODO transaction, resume, cleanup
|
5
5
|
# TODO add, finish, close, abort, commit
|
6
6
|
|
7
|
-
|
8
|
-
# TODO to_java
|
7
|
+
# TODO to_java
|
9
8
|
|
10
|
-
|
11
|
-
|
12
|
-
|
9
|
+
def self.from_java(java_class)
|
10
|
+
JavaPlugin.ruby_adapter_class(java_class, FileOutputPlugin, RubyAdapter)
|
11
|
+
end
|
13
12
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
end
|
19
|
-
# TODO transaction, resume, cleanup
|
13
|
+
module RubyAdapter
|
14
|
+
module ClassMethods
|
15
|
+
def new_java
|
16
|
+
Java::FileOutputRunner.new(Java.injector.getInstance(java_class))
|
20
17
|
end
|
21
|
-
|
22
|
-
# TODO add, finish, close, abort, commit
|
18
|
+
# TODO transaction, resume, cleanup
|
23
19
|
end
|
20
|
+
|
21
|
+
# TODO add, finish, close, abort, commit
|
24
22
|
end
|
25
23
|
end
|
26
24
|
|
data/lib/embulk/filter_plugin.rb
CHANGED
@@ -34,72 +34,70 @@ module Embulk
|
|
34
34
|
def close
|
35
35
|
end
|
36
36
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
end
|
37
|
+
def self.new_java
|
38
|
+
JavaAdapter.new(self)
|
39
|
+
end
|
41
40
|
|
42
|
-
|
43
|
-
|
41
|
+
class JavaAdapter
|
42
|
+
include Java::FilterPlugin
|
44
43
|
|
45
|
-
|
46
|
-
|
47
|
-
|
44
|
+
def initialize(ruby_class)
|
45
|
+
@ruby_class = ruby_class
|
46
|
+
end
|
48
47
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
end
|
57
|
-
nil
|
48
|
+
def transaction(java_config, java_in_schema, java_control)
|
49
|
+
config = DataSource.from_java(java_config)
|
50
|
+
in_schema = Schema.from_java(java_in_schema)
|
51
|
+
@ruby_class.transaction(config, in_schema) do |task_source_hash, out_columns|
|
52
|
+
java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
|
53
|
+
java_out_schemas = Schema.new(out_columns).to_java
|
54
|
+
java_control.run(java_task_source, java_out_schemas)
|
58
55
|
end
|
56
|
+
nil
|
57
|
+
end
|
59
58
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
59
|
+
def open(java_task_source, java_in_schema, java_out_schema, java_output)
|
60
|
+
task_source = DataSource.from_java(java_task_source)
|
61
|
+
in_schema = Schema.from_java(java_in_schema)
|
62
|
+
out_schema = Schema.from_java(java_out_schema)
|
63
|
+
page_builder = PageBuilder.new(out_schema, java_output)
|
64
|
+
ruby_object = @ruby_class.new(task_source, in_schema, out_schema, page_builder)
|
65
|
+
return OutputAdapter.new(ruby_object, in_schema, page_builder)
|
66
|
+
end
|
68
67
|
|
69
|
-
|
70
|
-
|
68
|
+
class OutputAdapter
|
69
|
+
include Java::TransactionalPageOutput
|
71
70
|
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
71
|
+
def initialize(ruby_object, in_schema, page_builder)
|
72
|
+
@ruby_object = ruby_object
|
73
|
+
@in_schema = in_schema
|
74
|
+
@page_builder = page_builder
|
75
|
+
end
|
77
76
|
|
78
|
-
|
79
|
-
|
80
|
-
|
77
|
+
def add(java_page)
|
78
|
+
@ruby_object.add Page.new(java_page, @in_schema)
|
79
|
+
end
|
81
80
|
|
82
|
-
|
83
|
-
|
84
|
-
|
81
|
+
def finish
|
82
|
+
@ruby_object.finish
|
83
|
+
end
|
85
84
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
end
|
85
|
+
def close
|
86
|
+
@ruby_object.close
|
87
|
+
ensure
|
88
|
+
@page_builder.close
|
91
89
|
end
|
92
90
|
end
|
91
|
+
end
|
93
92
|
|
94
|
-
|
95
|
-
|
96
|
-
|
93
|
+
def self.from_java(java_class)
|
94
|
+
JavaPlugin.ruby_adapter_class(java_class, FilterPlugin, RubyAdapter)
|
95
|
+
end
|
97
96
|
|
98
|
-
|
99
|
-
|
100
|
-
end
|
101
|
-
# TODO
|
97
|
+
module RubyAdapter
|
98
|
+
module ClassMethods
|
102
99
|
end
|
100
|
+
# TODO
|
103
101
|
end
|
104
102
|
end
|
105
103
|
end
|
@@ -33,72 +33,70 @@ module Embulk
|
|
33
33
|
def close
|
34
34
|
end
|
35
35
|
|
36
|
-
|
37
|
-
|
38
|
-
|
36
|
+
def self.new_java
|
37
|
+
JavaAdapter.new(self)
|
38
|
+
end
|
39
|
+
|
40
|
+
class JavaAdapter
|
41
|
+
include Java::FormatterPlugin
|
42
|
+
|
43
|
+
def initialize(ruby_class)
|
44
|
+
@ruby_class = ruby_class
|
45
|
+
end
|
46
|
+
|
47
|
+
def transaction(java_config, java_schema, java_control)
|
48
|
+
config = DataSource.from_java(java_config)
|
49
|
+
schema = Schema.from_java(java_schema)
|
50
|
+
@ruby_class.transaction(config, schema) do |task_source_hash|
|
51
|
+
java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
|
52
|
+
java_control.run(java_task_source)
|
53
|
+
end
|
54
|
+
nil
|
39
55
|
end
|
40
56
|
|
41
|
-
|
42
|
-
|
57
|
+
def open(java_task_source, java_schema, java_file_output)
|
58
|
+
task_source = DataSource.from_java(java_task_source)
|
59
|
+
schema = Schema.from_java(java_schema)
|
60
|
+
file_output = FileOutput.new(java_file_output)
|
61
|
+
ruby_object = @ruby_class.new(task_source, schema, file_output)
|
62
|
+
return OutputAdapter.new(ruby_object, schema, file_output)
|
63
|
+
end
|
64
|
+
|
65
|
+
class OutputAdapter
|
66
|
+
include Java::TransactionalPageOutput
|
43
67
|
|
44
|
-
def initialize(
|
45
|
-
@
|
68
|
+
def initialize(ruby_object, schema, file_output)
|
69
|
+
@ruby_object = ruby_object
|
70
|
+
@schema = schema
|
71
|
+
@file_output = file_output
|
46
72
|
end
|
47
73
|
|
48
|
-
def
|
49
|
-
|
50
|
-
|
51
|
-
@ruby_class.transaction(config, schema) do |task_source_hash|
|
52
|
-
java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
|
53
|
-
java_control.run(java_task_source)
|
54
|
-
end
|
55
|
-
nil
|
74
|
+
def add(java_page)
|
75
|
+
# TODO reuse page reader
|
76
|
+
@ruby_object.add Page.new(java_page, @schema)
|
56
77
|
end
|
57
78
|
|
58
|
-
def
|
59
|
-
|
60
|
-
schema = Schema.from_java(java_schema)
|
61
|
-
file_output = FileOutput.new(java_file_output)
|
62
|
-
ruby_object = @ruby_class.new(task_source, schema, file_output)
|
63
|
-
return OutputAdapter.new(ruby_object, schema, file_output)
|
79
|
+
def finish
|
80
|
+
@ruby_object.finish
|
64
81
|
end
|
65
82
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
@ruby_object = ruby_object
|
71
|
-
@schema = schema
|
72
|
-
@file_output = file_output
|
73
|
-
end
|
74
|
-
|
75
|
-
def add(java_page)
|
76
|
-
# TODO reuse page reader
|
77
|
-
@ruby_object.add Page.new(java_page, @schema)
|
78
|
-
end
|
79
|
-
|
80
|
-
def finish
|
81
|
-
@ruby_object.finish
|
82
|
-
end
|
83
|
-
|
84
|
-
def close
|
85
|
-
@ruby_object.close
|
86
|
-
ensure
|
87
|
-
@file_output.close
|
88
|
-
end
|
83
|
+
def close
|
84
|
+
@ruby_object.close
|
85
|
+
ensure
|
86
|
+
@file_output.close
|
89
87
|
end
|
90
88
|
end
|
89
|
+
end
|
91
90
|
|
92
|
-
|
93
|
-
|
94
|
-
|
91
|
+
def self.from_java(java_class)
|
92
|
+
JavaPlugin.ruby_adapter_class(java_class, FormatterPlugin, RubyAdapter)
|
93
|
+
end
|
95
94
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
end
|
100
|
-
# TODO add, finish, close
|
95
|
+
module RubyAdapter
|
96
|
+
module ClassMethods
|
97
|
+
# TODO transaction, resume, cleanup
|
101
98
|
end
|
99
|
+
# TODO add, finish, close
|
102
100
|
end
|
103
101
|
end
|
104
102
|
|
data/lib/embulk/guess_plugin.rb
CHANGED
@@ -7,40 +7,38 @@ module Embulk
|
|
7
7
|
raise NotImplementedError, "GuessPlugin#guess(config, sample_buffer) must be implemented"
|
8
8
|
end
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
end
|
14
|
-
|
15
|
-
class JavaAdapter
|
16
|
-
include Java::GuessPlugin
|
10
|
+
def self.new_java
|
11
|
+
JavaAdapter.new(new)
|
12
|
+
end
|
17
13
|
|
18
|
-
|
19
|
-
|
20
|
-
end
|
14
|
+
class JavaAdapter
|
15
|
+
include Java::GuessPlugin
|
21
16
|
|
22
|
-
|
23
|
-
|
24
|
-
sample = Buffer.from_java(java_sample)
|
25
|
-
config_diff_hash = @ruby_guess.guess(config, sample)
|
26
|
-
return DataSource.from_ruby_hash(config_diff_hash).to_java
|
27
|
-
end
|
17
|
+
def initialize(ruby_guess)
|
18
|
+
@ruby_guess = ruby_guess
|
28
19
|
end
|
29
20
|
|
30
|
-
def
|
31
|
-
|
21
|
+
def guess(java_config, java_sample)
|
22
|
+
config = DataSource.from_java(java_config)
|
23
|
+
sample = Buffer.from_java(java_sample)
|
24
|
+
config_diff_hash = @ruby_guess.guess(config, sample)
|
25
|
+
return DataSource.from_ruby_hash(config_diff_hash).to_java
|
32
26
|
end
|
27
|
+
end
|
33
28
|
|
34
|
-
|
35
|
-
|
36
|
-
|
29
|
+
def self.from_java(java_class)
|
30
|
+
JavaPlugin.ruby_adapter(java_class, GuessPlugin, RubyAdapter)
|
31
|
+
end
|
37
32
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
33
|
+
module RubyAdapter
|
34
|
+
module ClassMethods
|
35
|
+
end
|
36
|
+
|
37
|
+
def guess(config, sample)
|
38
|
+
java_config = config.to_java
|
39
|
+
java_sample = sample.to_java
|
40
|
+
java_config_diff = java_object.guess(java_config, java_sample)
|
41
|
+
return DataSource.from_java(java_config_diff)
|
44
42
|
end
|
45
43
|
end
|
46
44
|
end
|