flydata 0.5.17 → 0.5.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata-core/lib/flydata-core/event/event_handler_base.rb +56 -0
- data/flydata-core/lib/flydata-core/event/flydata_event.rb +29 -0
- data/flydata-core/lib/flydata-core/event/flydata_event_handler_registry.rb +11 -0
- data/flydata-core/lib/flydata-core/event/flydata_event_processor.rb +19 -0
- data/flydata-core/lib/flydata-core/event/flydata_event_sender.rb +27 -0
- data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb +4 -5
- data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +12 -2
- data/flydata-core/spec/event/flydata_event_handler_registry_spec.rb +33 -0
- data/flydata-core/spec/event/flydata_event_handler_spec.rb +39 -0
- data/flydata-core/spec/event/flydata_event_processor_spec.rb +42 -0
- data/flydata-core/spec/event/flydata_event_sender_spec.rb +33 -0
- data/flydata-core/spec/event/flydata_event_spec.rb +39 -0
- data/flydata-core/spec/event/shared_event.rb +33 -0
- data/flydata-core/spec/mysql/compatibility_checker_spec.rb +4 -5
- data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb +25 -25
- data/flydata-core/spec/table_def/redshift_table_def_spec.rb +4 -4
- data/flydata.gemspec +0 -0
- data/lib/flydata/command/start.rb +15 -8
- data/lib/flydata/command/sync.rb +79 -61
- data/lib/flydata/errors.rb +1 -1
- data/lib/flydata/event/api_event_sender.rb +16 -0
- data/lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb +8 -0
- data/lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb +1 -1
- data/lib/flydata/output/forwarder.rb +7 -5
- data/lib/flydata/parser/mysql/dump_parser.rb +76 -12
- data/lib/flydata/queueable_thread.rb +32 -0
- data/lib/flydata/sync_file_manager.rb +45 -0
- data/spec/flydata/command/start_spec.rb +1 -1
- data/spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb +10 -3
- data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb +10 -10
- data/spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb +11 -2
- data/spec/flydata/output/forwarder_spec.rb +4 -4
- metadata +15 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2731322ee5aa072665e488692665d0ba9732f720
|
4
|
+
data.tar.gz: 681f290e03fdaeccf47ecb80d17d1a039d528458
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c443a473619d395c61bb6e08b6ce5de4b0c874efecafe0a24bff411c28287aa4a6e0b5d33537a316be7428a7f9700f9f519d781cfcf16d2c808fc93e6832fa96
|
7
|
+
data.tar.gz: 860cd66add7614fdbc5f68bba8be6a41adcfbaf704a02caa4bb22c3aca85f6b82261147ab52eaa3e2edea379ae02b7c04b13df812ffcbcab716bf720f5478ceb
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.5.
|
1
|
+
0.5.20
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module FlydataCore
|
2
|
+
module Event
|
3
|
+
class EventHandlerBase
|
4
|
+
EVENT_VERB_ANY = "__ANY__"
|
5
|
+
DEFAULT_ORDER = 500
|
6
|
+
|
7
|
+
@@handlers_hash = {}
|
8
|
+
|
9
|
+
def self.handle(event)
|
10
|
+
raise "handle method not implemented"
|
11
|
+
end
|
12
|
+
|
13
|
+
protected
|
14
|
+
def self.register_handler(opts)
|
15
|
+
@noun = opts[:noun]
|
16
|
+
@verb = opts[:verb] || EVENT_VERB_ANY
|
17
|
+
@execution_order = opts[:execution_order] || DEFAULT_ORDER
|
18
|
+
|
19
|
+
handlers = @@handlers_hash[self.key] ||= []
|
20
|
+
insert_handler(handlers, self)
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
def self.handlers(noun, verb)
|
25
|
+
handlers = (@@handlers_hash[key(noun, verb)] || []).clone
|
26
|
+
|
27
|
+
any_handlers = @@handlers_hash[key(noun,EVENT_VERB_ANY)]
|
28
|
+
if any_handlers
|
29
|
+
any_handlers.each { |any_handler| self.insert_handler(handlers,any_handler) }
|
30
|
+
end
|
31
|
+
handlers
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.key(noun = nil, verb = nil)
|
35
|
+
noun ||= @noun
|
36
|
+
verb ||= @verb
|
37
|
+
"#{noun}_#{verb}"
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.insert_handler(handlers, handler)
|
41
|
+
handlers.each_with_index do |h, i|
|
42
|
+
if handler.execution_order < h.execution_order
|
43
|
+
handlers.insert(i, handler)
|
44
|
+
handler = nil
|
45
|
+
break
|
46
|
+
end
|
47
|
+
end
|
48
|
+
handlers << handler if handler
|
49
|
+
end
|
50
|
+
|
51
|
+
def self.execution_order
|
52
|
+
@execution_order
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'securerandom'
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
module FlydataCore
|
5
|
+
module Event
|
6
|
+
class FlydataEvent
|
7
|
+
attr_accessor :noun
|
8
|
+
attr_accessor :verb
|
9
|
+
attr_accessor :data
|
10
|
+
attr_accessor :id
|
11
|
+
attr_accessor :created
|
12
|
+
attr_accessor :data_entry_id
|
13
|
+
attr_accessor :data_port_id
|
14
|
+
|
15
|
+
def initialize (noun, verb, data=nil, de_id=nil,dp_id=nil, id=nil, timestamp=nil)
|
16
|
+
@noun = noun
|
17
|
+
@verb = verb
|
18
|
+
@data_entry_id = de_id
|
19
|
+
@data_port_id = dp_id
|
20
|
+
@data = data
|
21
|
+
@id = id.nil? ? SecureRandom.uuid: id
|
22
|
+
@created = timestamp.nil? ? Time.now : timestamp
|
23
|
+
end
|
24
|
+
def to_json
|
25
|
+
{:noun => @noun, :verb => @verb, :data_entry_id=>@data_entry_id, :data_port_id=> @data_port_id, id => @id, :created => @created, :data => @data}.to_json
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'flydata-core/event/flydata_event_handler_registry'
|
2
|
+
|
3
|
+
module FlydataCore
|
4
|
+
module Event
|
5
|
+
class FlydataEventProcessor
|
6
|
+
|
7
|
+
def self.process(event)
|
8
|
+
handlers = FlydataEventHandlerRegistry.handlers(event.noun, event.verb)
|
9
|
+
if handlers.nil? || handlers.empty?
|
10
|
+
return
|
11
|
+
end
|
12
|
+
handlers.each do |handler|
|
13
|
+
handler.handle(event)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'flydata-core/event/flydata_event'
|
2
|
+
|
3
|
+
module FlydataCore
|
4
|
+
module Event
|
5
|
+
|
6
|
+
class FlydataEventSender
|
7
|
+
|
8
|
+
def send_event(noun, verb, data=nil, data_entry_id=nil, data_port_id=nil, routing_key="default")
|
9
|
+
event = FlydataEvent.new(noun, verb, data, data_entry_id, data_port_id)
|
10
|
+
send(event, routing_key)
|
11
|
+
end
|
12
|
+
|
13
|
+
protected
|
14
|
+
def send(event, routing_key)
|
15
|
+
raise "send method not implemented"
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
class SimpleEventSender <FlydataEventSender
|
20
|
+
protected
|
21
|
+
def send(event, routing_key)
|
22
|
+
FlydataEventProcessor.process(event)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
end
|
@@ -263,11 +263,10 @@ SELECT
|
|
263
263
|
t.table_name,
|
264
264
|
SUM(IF(tc.constraint_type='PRIMARY KEY', 1, 0)) as num_pk
|
265
265
|
FROM
|
266
|
-
information_schema.tables t
|
267
|
-
LEFT OUTER JOIN
|
268
|
-
|
269
|
-
|
270
|
-
AND t.table_name in (%{table_names})
|
266
|
+
(select * from information_schema.tables where table_schema = '%{database}' AND table_name in (%{table_names})) t
|
267
|
+
LEFT OUTER JOIN
|
268
|
+
(select * from information_schema.table_constraints where table_schema = '%{database}' AND table_name in (%{table_names})) tc
|
269
|
+
USING (table_schema, table_name)
|
271
270
|
GROUP BY
|
272
271
|
t.table_schema, t.table_name
|
273
272
|
HAVING
|
@@ -162,11 +162,21 @@ EOS
|
|
162
162
|
type_info[:type]
|
163
163
|
end
|
164
164
|
line = %Q| "#{convert_to_valid_column_name(column[:column])}" #{rs_type}|
|
165
|
-
|
165
|
+
|
166
|
+
not_null = nil
|
167
|
+
# not_null = column[:not_null]
|
168
|
+
# `not_null` is always `nil`.
|
169
|
+
# ***`column[:not_null]` is ignored for Redshift***
|
170
|
+
# Avoid explicitly creating NOT NULL column in Redshift.
|
171
|
+
# Otherwise, removing NOT NULL attribute from the column later would cause an error
|
172
|
+
# because Redshift doesn't support 'ALTER TABLE tbl MODIFY column column_attributes'.
|
173
|
+
|
174
|
+
line += " NOT NULL" if not_null
|
175
|
+
|
166
176
|
if (column.has_key?(:default))
|
167
177
|
val = replace_default_value(type, type_info[:type], column[:default])
|
168
178
|
line += " DEFAULT #{val}"
|
169
|
-
elsif
|
179
|
+
elsif not_null && opt[:for] == :alter_table
|
170
180
|
# Redshift doesn't allow adding a not null column without default value
|
171
181
|
# Add a defalt value
|
172
182
|
line += " DEFAULT '#{type_info[:default_value]}'"
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'flydata-core/event/flydata_event_handler_registry'
|
3
|
+
require 'event/shared_event'
|
4
|
+
|
5
|
+
|
6
|
+
module FlydataCore
|
7
|
+
module Event
|
8
|
+
describe FlydataEventHandlerRegistry do
|
9
|
+
|
10
|
+
describe "#handlers" do
|
11
|
+
context "with event that has specific handlers" do
|
12
|
+
it "it returns correct hanlders" do
|
13
|
+
handlers = FlydataEventHandlerRegistry.handlers("test", "testing")
|
14
|
+
expect(handlers.size).to eql(2)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
context "with event that has specific handlers" do
|
18
|
+
it "it returns correct hanlders case-1" do
|
19
|
+
handlers = FlydataEventHandlerRegistry.handlers("test2", "testing")
|
20
|
+
expect(handlers.size).to eql(2)
|
21
|
+
end
|
22
|
+
it "it returns any hanlders for unknown verb" do
|
23
|
+
handlers = FlydataEventHandlerRegistry.handlers("test2", "abcd")
|
24
|
+
expect(handlers.size).to eql(1)
|
25
|
+
expect(handlers.to_a()[0]).to be(AnyHandler)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'event/shared_event'
|
3
|
+
|
4
|
+
module FlydataCore
|
5
|
+
module Event
|
6
|
+
describe EventHandlerBase do
|
7
|
+
describe "initializers" do
|
8
|
+
context "when a handler gives explicit execution order" do
|
9
|
+
it {
|
10
|
+
expect(Handler1.instance_variable_get(:@noun)).to eql("test")
|
11
|
+
expect(Handler1.instance_variable_get(:@verb)).to eql("testing")
|
12
|
+
expect(Handler1.instance_variable_get(:@execution_order)).to eql(1)
|
13
|
+
}
|
14
|
+
end
|
15
|
+
|
16
|
+
context "when a handler gives no explicit execution order" do
|
17
|
+
it {
|
18
|
+
expect(Handler3.instance_variable_get(:@noun)).to eql("test2")
|
19
|
+
expect(Handler3.instance_variable_get(:@verb)).to eql("testing")
|
20
|
+
expect(Handler3.instance_variable_get(:@execution_order)).to eql(500)
|
21
|
+
}
|
22
|
+
end
|
23
|
+
context "when a verb is nil" do
|
24
|
+
it {
|
25
|
+
expect(AnyHandler.instance_variable_get(:@noun)).to eql("test2")
|
26
|
+
expect(AnyHandler.instance_variable_get(:@verb)).to eql(EventHandlerBase::EVENT_VERB_ANY)
|
27
|
+
}
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
describe "#key" do
|
32
|
+
it "calcluates key correctly" do
|
33
|
+
expect(EventHandlerBase.key("test", "testing")).to eql("test_testing")
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'flydata-core/event/flydata_event_processor'
|
3
|
+
require 'flydata-core/event/flydata_event'
|
4
|
+
require 'event/shared_event'
|
5
|
+
|
6
|
+
module FlydataCore
|
7
|
+
module Event
|
8
|
+
describe FlydataEventProcessor do
|
9
|
+
describe "#process" do
|
10
|
+
|
11
|
+
context "when event has one verb handler and one any verb handler," do
|
12
|
+
let(:event) {FlydataEvent.new("test2","testing",nil, 1)}
|
13
|
+
it "should call both handlers" do
|
14
|
+
expect(AnyHandler).to receive(:handle).with(event)
|
15
|
+
expect(Handler3).to receive(:handle).with(event)
|
16
|
+
FlydataEventProcessor.process(event)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
context "when event has two handlers," do
|
21
|
+
let(:event) {FlydataEvent.new("test","testing",nil, 1)}
|
22
|
+
it "should call both handlers" do
|
23
|
+
expect(Handler1).to receive(:handle).with(event)
|
24
|
+
expect(Handler2).to receive(:handle).with(event)
|
25
|
+
FlydataEventProcessor.process(event)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
context "when a event has no handlers," do
|
30
|
+
let(:event) {FlydataEvent.new("test5","testing")}
|
31
|
+
it "no handler should be called" do
|
32
|
+
expect(Handler1).not_to receive(:handle).with(event)
|
33
|
+
expect(Handler2).not_to receive(:handle).with(event)
|
34
|
+
expect(Handler3).not_to receive(:handle).with(event)
|
35
|
+
FlydataEventProcessor.process(event)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'event/shared_event'
|
3
|
+
require 'flydata-core/event/flydata_event_sender'
|
4
|
+
require 'flydata-core/event/flydata_event_processor'
|
5
|
+
|
6
|
+
|
7
|
+
module FlydataCore
|
8
|
+
module Event
|
9
|
+
describe "FlydataEventSender" do
|
10
|
+
subject {FlydataEventSender.new}
|
11
|
+
|
12
|
+
context "#send_event" do
|
13
|
+
subject{
|
14
|
+
send_event("test","testing",nil, 1)
|
15
|
+
}
|
16
|
+
it "should raise exception" do
|
17
|
+
expect { raise Exception}
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe "SimpleEventSender" do
|
23
|
+
sender = SimpleEventSender.new
|
24
|
+
context "#send_event" do
|
25
|
+
it do
|
26
|
+
expect(Handler1).to receive(:handle)
|
27
|
+
expect(Handler2).to receive(:handle)
|
28
|
+
sender.send_event("test","testing")
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'flydata-core/event/flydata_event'
|
3
|
+
|
4
|
+
module FlydataCore
|
5
|
+
module Event
|
6
|
+
describe FlydataEvent do
|
7
|
+
describe "#initialize" do
|
8
|
+
let(:noun){"test"}
|
9
|
+
let(:verb){"testing"}
|
10
|
+
let(:data){{test:123}}
|
11
|
+
|
12
|
+
context "when noun and verb are passed" do
|
13
|
+
it {
|
14
|
+
event = FlydataEvent.new(noun,verb, nil,1)
|
15
|
+
expect(event.noun).to eql(noun)
|
16
|
+
expect(event.verb).to eql(verb)
|
17
|
+
expect(event.data).to be_nil
|
18
|
+
expect(event.id).not_to be_nil
|
19
|
+
expect(event.data_entry_id).to eql(1)
|
20
|
+
expect(event.created).not_to be_nil
|
21
|
+
}
|
22
|
+
end
|
23
|
+
|
24
|
+
context "when noun, verb and data are passed" do
|
25
|
+
it {
|
26
|
+
event = FlydataEvent.new(noun,verb,data,1)
|
27
|
+
expect(event.noun).to eql(noun)
|
28
|
+
expect(event.verb).to eql(verb)
|
29
|
+
expect(event.data).to eql(data)
|
30
|
+
expect(event.data_entry_id).to eql(1)
|
31
|
+
expect(event.id).not_to be_nil
|
32
|
+
expect(event.created).not_to be_nil
|
33
|
+
}
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'flydata-core/event/event_handler_base'
|
2
|
+
|
3
|
+
module FlydataCore
|
4
|
+
module Event
|
5
|
+
class Handler1 < EventHandlerBase
|
6
|
+
register_handler noun: "test", verb: "testing", execution_order: 1
|
7
|
+
def self.handle(event)
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
class Handler2 < EventHandlerBase
|
12
|
+
register_handler noun: "test", verb: "testing", execution_order: 2
|
13
|
+
|
14
|
+
def self.handle(event)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
class Handler3 < EventHandlerBase
|
19
|
+
register_handler noun: "test2", verb: "testing"
|
20
|
+
|
21
|
+
def handle(event)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
class AnyHandler < EventHandlerBase
|
26
|
+
register_handler noun: "test2", execution_order: 1
|
27
|
+
|
28
|
+
def self.handle(event)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
@@ -66,11 +66,10 @@ SELECT
|
|
66
66
|
t.table_name,
|
67
67
|
SUM(IF(tc.constraint_type='PRIMARY KEY', 1, 0)) as num_pk
|
68
68
|
FROM
|
69
|
-
information_schema.tables t
|
70
|
-
LEFT OUTER JOIN
|
71
|
-
|
72
|
-
|
73
|
-
AND t.table_name in ('table_1','table_2','table_3')
|
69
|
+
(select * from information_schema.tables where table_schema = 'test_db' AND table_name in ('table_1','table_2','table_3')) t
|
70
|
+
LEFT OUTER JOIN
|
71
|
+
(select * from information_schema.table_constraints where table_schema = 'test_db' AND table_name in ('table_1','table_2','table_3')) tc
|
72
|
+
USING (table_schema, table_name)
|
74
73
|
GROUP BY
|
75
74
|
t.table_schema, t.table_name
|
76
75
|
HAVING
|