flydata 0.5.17 → 0.5.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. checksums.yaml +4 -4
  2. data/VERSION +1 -1
  3. data/flydata-core/lib/flydata-core/event/event_handler_base.rb +56 -0
  4. data/flydata-core/lib/flydata-core/event/flydata_event.rb +29 -0
  5. data/flydata-core/lib/flydata-core/event/flydata_event_handler_registry.rb +11 -0
  6. data/flydata-core/lib/flydata-core/event/flydata_event_processor.rb +19 -0
  7. data/flydata-core/lib/flydata-core/event/flydata_event_sender.rb +27 -0
  8. data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb +4 -5
  9. data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +12 -2
  10. data/flydata-core/spec/event/flydata_event_handler_registry_spec.rb +33 -0
  11. data/flydata-core/spec/event/flydata_event_handler_spec.rb +39 -0
  12. data/flydata-core/spec/event/flydata_event_processor_spec.rb +42 -0
  13. data/flydata-core/spec/event/flydata_event_sender_spec.rb +33 -0
  14. data/flydata-core/spec/event/flydata_event_spec.rb +39 -0
  15. data/flydata-core/spec/event/shared_event.rb +33 -0
  16. data/flydata-core/spec/mysql/compatibility_checker_spec.rb +4 -5
  17. data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb +25 -25
  18. data/flydata-core/spec/table_def/redshift_table_def_spec.rb +4 -4
  19. data/flydata.gemspec +0 -0
  20. data/lib/flydata/command/start.rb +15 -8
  21. data/lib/flydata/command/sync.rb +79 -61
  22. data/lib/flydata/errors.rb +1 -1
  23. data/lib/flydata/event/api_event_sender.rb +16 -0
  24. data/lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb +8 -0
  25. data/lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb +1 -1
  26. data/lib/flydata/output/forwarder.rb +7 -5
  27. data/lib/flydata/parser/mysql/dump_parser.rb +76 -12
  28. data/lib/flydata/queueable_thread.rb +32 -0
  29. data/lib/flydata/sync_file_manager.rb +45 -0
  30. data/spec/flydata/command/start_spec.rb +1 -1
  31. data/spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb +10 -3
  32. data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb +10 -10
  33. data/spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb +11 -2
  34. data/spec/flydata/output/forwarder_spec.rb +4 -4
  35. metadata +15 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: db5ab3ad859c85fb9fcdb2171ad04b0f2e468eb9
4
- data.tar.gz: 1225c311e10046702a2cf89b5b2d3f8114022ee9
3
+ metadata.gz: 2731322ee5aa072665e488692665d0ba9732f720
4
+ data.tar.gz: 681f290e03fdaeccf47ecb80d17d1a039d528458
5
5
  SHA512:
6
- metadata.gz: c4c2d116d3279e3794b510898bf385aa7aaa22703afdca15bfcae4d7574764ccaa84731b159a98cac7b745de1f46b6fc4e1cf547e06183ee8e46a2c1abc95b60
7
- data.tar.gz: 746ccd2cc7cac24b1745e9c08a3184acaa9b7f37a0ed25f2015dd077acb997a3cf25caa812d17c0f78a5591d95b7dd618a38efb61e3f9722e7da13183df34825
6
+ metadata.gz: c443a473619d395c61bb6e08b6ce5de4b0c874efecafe0a24bff411c28287aa4a6e0b5d33537a316be7428a7f9700f9f519d781cfcf16d2c808fc93e6832fa96
7
+ data.tar.gz: 860cd66add7614fdbc5f68bba8be6a41adcfbaf704a02caa4bb22c3aca85f6b82261147ab52eaa3e2edea379ae02b7c04b13df812ffcbcab716bf720f5478ceb
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.5.17
1
+ 0.5.20
@@ -0,0 +1,56 @@
1
+ module FlydataCore
2
+ module Event
3
+ class EventHandlerBase
4
+ EVENT_VERB_ANY = "__ANY__"
5
+ DEFAULT_ORDER = 500
6
+
7
+ @@handlers_hash = {}
8
+
9
+ def self.handle(event)
10
+ raise "handle method not implemented"
11
+ end
12
+
13
+ protected
14
+ def self.register_handler(opts)
15
+ @noun = opts[:noun]
16
+ @verb = opts[:verb] || EVENT_VERB_ANY
17
+ @execution_order = opts[:execution_order] || DEFAULT_ORDER
18
+
19
+ handlers = @@handlers_hash[self.key] ||= []
20
+ insert_handler(handlers, self)
21
+ end
22
+
23
+ private
24
+ def self.handlers(noun, verb)
25
+ handlers = (@@handlers_hash[key(noun, verb)] || []).clone
26
+
27
+ any_handlers = @@handlers_hash[key(noun,EVENT_VERB_ANY)]
28
+ if any_handlers
29
+ any_handlers.each { |any_handler| self.insert_handler(handlers,any_handler) }
30
+ end
31
+ handlers
32
+ end
33
+
34
+ def self.key(noun = nil, verb = nil)
35
+ noun ||= @noun
36
+ verb ||= @verb
37
+ "#{noun}_#{verb}"
38
+ end
39
+
40
+ def self.insert_handler(handlers, handler)
41
+ handlers.each_with_index do |h, i|
42
+ if handler.execution_order < h.execution_order
43
+ handlers.insert(i, handler)
44
+ handler = nil
45
+ break
46
+ end
47
+ end
48
+ handlers << handler if handler
49
+ end
50
+
51
+ def self.execution_order
52
+ @execution_order
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,29 @@
1
+ require 'securerandom'
2
+ require 'json'
3
+
4
+ module FlydataCore
5
+ module Event
6
+ class FlydataEvent
7
+ attr_accessor :noun
8
+ attr_accessor :verb
9
+ attr_accessor :data
10
+ attr_accessor :id
11
+ attr_accessor :created
12
+ attr_accessor :data_entry_id
13
+ attr_accessor :data_port_id
14
+
15
+ def initialize (noun, verb, data=nil, de_id=nil,dp_id=nil, id=nil, timestamp=nil)
16
+ @noun = noun
17
+ @verb = verb
18
+ @data_entry_id = de_id
19
+ @data_port_id = dp_id
20
+ @data = data
21
+ @id = id.nil? ? SecureRandom.uuid: id
22
+ @created = timestamp.nil? ? Time.now : timestamp
23
+ end
24
+ def to_json
25
+ {:noun => @noun, :verb => @verb, :data_entry_id=>@data_entry_id, :data_port_id=> @data_port_id, id => @id, :created => @created, :data => @data}.to_json
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,11 @@
1
+ require 'flydata-core/event/event_handler_base'
2
+
3
+ module FlydataCore
4
+ module Event
5
+ module FlydataEventHandlerRegistry
6
+ def self.handlers(noun, verb)
7
+ EventHandlerBase.send(:handlers ,noun, verb)
8
+ end
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,19 @@
1
+ require 'flydata-core/event/flydata_event_handler_registry'
2
+
3
+ module FlydataCore
4
+ module Event
5
+ class FlydataEventProcessor
6
+
7
+ def self.process(event)
8
+ handlers = FlydataEventHandlerRegistry.handlers(event.noun, event.verb)
9
+ if handlers.nil? || handlers.empty?
10
+ return
11
+ end
12
+ handlers.each do |handler|
13
+ handler.handle(event)
14
+ end
15
+ end
16
+ end
17
+
18
+ end
19
+ end
@@ -0,0 +1,27 @@
1
+ require 'flydata-core/event/flydata_event'
2
+
3
+ module FlydataCore
4
+ module Event
5
+
6
+ class FlydataEventSender
7
+
8
+ def send_event(noun, verb, data=nil, data_entry_id=nil, data_port_id=nil, routing_key="default")
9
+ event = FlydataEvent.new(noun, verb, data, data_entry_id, data_port_id)
10
+ send(event, routing_key)
11
+ end
12
+
13
+ protected
14
+ def send(event, routing_key)
15
+ raise "send method not implemented"
16
+ end
17
+ end
18
+
19
+ class SimpleEventSender <FlydataEventSender
20
+ protected
21
+ def send(event, routing_key)
22
+ FlydataEventProcessor.process(event)
23
+ end
24
+ end
25
+
26
+ end
27
+ end
@@ -263,11 +263,10 @@ SELECT
263
263
  t.table_name,
264
264
  SUM(IF(tc.constraint_type='PRIMARY KEY', 1, 0)) as num_pk
265
265
  FROM
266
- information_schema.tables t
267
- LEFT OUTER JOIN information_schema.table_constraints tc USING (table_schema, table_name)
268
- WHERE
269
- t.table_schema = '%{database}'
270
- AND t.table_name in (%{table_names})
266
+ (select * from information_schema.tables where table_schema = '%{database}' AND table_name in (%{table_names})) t
267
+ LEFT OUTER JOIN
268
+ (select * from information_schema.table_constraints where table_schema = '%{database}' AND table_name in (%{table_names})) tc
269
+ USING (table_schema, table_name)
271
270
  GROUP BY
272
271
  t.table_schema, t.table_name
273
272
  HAVING
@@ -162,11 +162,21 @@ EOS
162
162
  type_info[:type]
163
163
  end
164
164
  line = %Q| "#{convert_to_valid_column_name(column[:column])}" #{rs_type}|
165
- line += " NOT NULL" if column[:not_null]
165
+
166
+ not_null = nil
167
+ # not_null = column[:not_null]
168
+ # `not_null` is always `nil`.
169
+ # ***`column[:not_null]` is ignored for Redshift***
170
+ # Avoid explicitly creating NOT NULL column in Redshift.
171
+ # Otherwise, removing NOT NULL attribute from the column later would cause an error
172
+ # because Redshift doesn't support 'ALTER TABLE tbl MODIFY column column_attributes'.
173
+
174
+ line += " NOT NULL" if not_null
175
+
166
176
  if (column.has_key?(:default))
167
177
  val = replace_default_value(type, type_info[:type], column[:default])
168
178
  line += " DEFAULT #{val}"
169
- elsif column[:not_null] && opt[:for] == :alter_table
179
+ elsif not_null && opt[:for] == :alter_table
170
180
  # Redshift doesn't allow adding a not null column without default value
171
181
  # Add a defalt value
172
182
  line += " DEFAULT '#{type_info[:default_value]}'"
@@ -0,0 +1,33 @@
1
+ require 'spec_helper'
2
+ require 'flydata-core/event/flydata_event_handler_registry'
3
+ require 'event/shared_event'
4
+
5
+
6
+ module FlydataCore
7
+ module Event
8
+ describe FlydataEventHandlerRegistry do
9
+
10
+ describe "#handlers" do
11
+ context "with event that has specific handlers" do
12
+ it "it returns correct hanlders" do
13
+ handlers = FlydataEventHandlerRegistry.handlers("test", "testing")
14
+ expect(handlers.size).to eql(2)
15
+ end
16
+ end
17
+ context "with event that has specific handlers" do
18
+ it "it returns correct hanlders case-1" do
19
+ handlers = FlydataEventHandlerRegistry.handlers("test2", "testing")
20
+ expect(handlers.size).to eql(2)
21
+ end
22
+ it "it returns any hanlders for unknown verb" do
23
+ handlers = FlydataEventHandlerRegistry.handlers("test2", "abcd")
24
+ expect(handlers.size).to eql(1)
25
+ expect(handlers.to_a()[0]).to be(AnyHandler)
26
+ end
27
+ end
28
+
29
+ end
30
+
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,39 @@
1
+ require 'spec_helper'
2
+ require 'event/shared_event'
3
+
4
+ module FlydataCore
5
+ module Event
6
+ describe EventHandlerBase do
7
+ describe "initializers" do
8
+ context "when a handler gives explicit execution order" do
9
+ it {
10
+ expect(Handler1.instance_variable_get(:@noun)).to eql("test")
11
+ expect(Handler1.instance_variable_get(:@verb)).to eql("testing")
12
+ expect(Handler1.instance_variable_get(:@execution_order)).to eql(1)
13
+ }
14
+ end
15
+
16
+ context "when a handler gives no explicit execution order" do
17
+ it {
18
+ expect(Handler3.instance_variable_get(:@noun)).to eql("test2")
19
+ expect(Handler3.instance_variable_get(:@verb)).to eql("testing")
20
+ expect(Handler3.instance_variable_get(:@execution_order)).to eql(500)
21
+ }
22
+ end
23
+ context "when a verb is nil" do
24
+ it {
25
+ expect(AnyHandler.instance_variable_get(:@noun)).to eql("test2")
26
+ expect(AnyHandler.instance_variable_get(:@verb)).to eql(EventHandlerBase::EVENT_VERB_ANY)
27
+ }
28
+ end
29
+ end
30
+
31
+ describe "#key" do
32
+ it "calcluates key correctly" do
33
+ expect(EventHandlerBase.key("test", "testing")).to eql("test_testing")
34
+ end
35
+ end
36
+
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,42 @@
1
+ require 'spec_helper'
2
+ require 'flydata-core/event/flydata_event_processor'
3
+ require 'flydata-core/event/flydata_event'
4
+ require 'event/shared_event'
5
+
6
+ module FlydataCore
7
+ module Event
8
+ describe FlydataEventProcessor do
9
+ describe "#process" do
10
+
11
+ context "when event has one verb handler and one any verb handler," do
12
+ let(:event) {FlydataEvent.new("test2","testing",nil, 1)}
13
+ it "should call both handlers" do
14
+ expect(AnyHandler).to receive(:handle).with(event)
15
+ expect(Handler3).to receive(:handle).with(event)
16
+ FlydataEventProcessor.process(event)
17
+ end
18
+ end
19
+
20
+ context "when event has two handlers," do
21
+ let(:event) {FlydataEvent.new("test","testing",nil, 1)}
22
+ it "should call both handlers" do
23
+ expect(Handler1).to receive(:handle).with(event)
24
+ expect(Handler2).to receive(:handle).with(event)
25
+ FlydataEventProcessor.process(event)
26
+ end
27
+ end
28
+
29
+ context "when a event has no handlers," do
30
+ let(:event) {FlydataEvent.new("test5","testing")}
31
+ it "no handler should be called" do
32
+ expect(Handler1).not_to receive(:handle).with(event)
33
+ expect(Handler2).not_to receive(:handle).with(event)
34
+ expect(Handler3).not_to receive(:handle).with(event)
35
+ FlydataEventProcessor.process(event)
36
+ end
37
+ end
38
+
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,33 @@
1
+ require 'spec_helper'
2
+ require 'event/shared_event'
3
+ require 'flydata-core/event/flydata_event_sender'
4
+ require 'flydata-core/event/flydata_event_processor'
5
+
6
+
7
+ module FlydataCore
8
+ module Event
9
+ describe "FlydataEventSender" do
10
+ subject {FlydataEventSender.new}
11
+
12
+ context "#send_event" do
13
+ subject{
14
+ send_event("test","testing",nil, 1)
15
+ }
16
+ it "should raise exception" do
17
+ expect { raise Exception}
18
+ end
19
+ end
20
+ end
21
+
22
+ describe "SimpleEventSender" do
23
+ sender = SimpleEventSender.new
24
+ context "#send_event" do
25
+ it do
26
+ expect(Handler1).to receive(:handle)
27
+ expect(Handler2).to receive(:handle)
28
+ sender.send_event("test","testing")
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,39 @@
1
+ require 'spec_helper'
2
+ require 'flydata-core/event/flydata_event'
3
+
4
+ module FlydataCore
5
+ module Event
6
+ describe FlydataEvent do
7
+ describe "#initialize" do
8
+ let(:noun){"test"}
9
+ let(:verb){"testing"}
10
+ let(:data){{test:123}}
11
+
12
+ context "when noun and verb are passed" do
13
+ it {
14
+ event = FlydataEvent.new(noun,verb, nil,1)
15
+ expect(event.noun).to eql(noun)
16
+ expect(event.verb).to eql(verb)
17
+ expect(event.data).to be_nil
18
+ expect(event.id).not_to be_nil
19
+ expect(event.data_entry_id).to eql(1)
20
+ expect(event.created).not_to be_nil
21
+ }
22
+ end
23
+
24
+ context "when noun, verb and data are passed" do
25
+ it {
26
+ event = FlydataEvent.new(noun,verb,data,1)
27
+ expect(event.noun).to eql(noun)
28
+ expect(event.verb).to eql(verb)
29
+ expect(event.data).to eql(data)
30
+ expect(event.data_entry_id).to eql(1)
31
+ expect(event.id).not_to be_nil
32
+ expect(event.created).not_to be_nil
33
+ }
34
+ end
35
+
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,33 @@
1
+ require 'flydata-core/event/event_handler_base'
2
+
3
+ module FlydataCore
4
+ module Event
5
+ class Handler1 < EventHandlerBase
6
+ register_handler noun: "test", verb: "testing", execution_order: 1
7
+ def self.handle(event)
8
+ end
9
+ end
10
+
11
+ class Handler2 < EventHandlerBase
12
+ register_handler noun: "test", verb: "testing", execution_order: 2
13
+
14
+ def self.handle(event)
15
+ end
16
+ end
17
+
18
+ class Handler3 < EventHandlerBase
19
+ register_handler noun: "test2", verb: "testing"
20
+
21
+ def handle(event)
22
+ end
23
+ end
24
+
25
+ class AnyHandler < EventHandlerBase
26
+ register_handler noun: "test2", execution_order: 1
27
+
28
+ def self.handle(event)
29
+ end
30
+ end
31
+
32
+ end
33
+ end
@@ -66,11 +66,10 @@ SELECT
66
66
  t.table_name,
67
67
  SUM(IF(tc.constraint_type='PRIMARY KEY', 1, 0)) as num_pk
68
68
  FROM
69
- information_schema.tables t
70
- LEFT OUTER JOIN information_schema.table_constraints tc USING (table_schema, table_name)
71
- WHERE
72
- t.table_schema = 'test_db'
73
- AND t.table_name in ('table_1','table_2','table_3')
69
+ (select * from information_schema.tables where table_schema = 'test_db' AND table_name in ('table_1','table_2','table_3')) t
70
+ LEFT OUTER JOIN
71
+ (select * from information_schema.table_constraints where table_schema = 'test_db' AND table_name in ('table_1','table_2','table_3')) tc
72
+ USING (table_schema, table_name)
74
73
  GROUP BY
75
74
  t.table_schema, t.table_name
76
75
  HAVING