flydata 0.3.19 → 0.3.20
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb +1 -0
- data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +1 -0
- data/flydata-core/spec/table_def/mysql_table_def_spec.rb +37 -0
- data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb +128 -10
- data/flydata-core/spec/table_def/redshift_table_def_spec.rb +4 -2
- data/flydata.gemspec +3 -3
- data/lib/flydata/fluent-plugins/idle_event_detector.rb +6 -2
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +8 -4
- data/spec/flydata/fluent-plugins/idle_event_detector_spec.rb +54 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e06e855feb4eedb44d34602d0e6d447f72eddc2f
|
4
|
+
data.tar.gz: 960b559b4006bfcea9fecb6b96ef0457f2eed949
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0cf8f7f398d49b1081be49e96d0111a5e9bb88102bdea6fb148a015629cd62d0140e11f22a0ad0e6bd490a3993d362a0d5c7af29fa58829434649487d0dcf7b2
|
7
|
+
data.tar.gz: cf814b03327a2230bb90058c82c72b3c6e1b06225a2048dff20dd6744656614418cf609bd7c4a452570d229ff34c1ba4eca54e33b99f202f6a6c54850fab183c
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.3.
|
1
|
+
0.3.20
|
@@ -266,6 +266,7 @@ EOS
|
|
266
266
|
values = []
|
267
267
|
values << "('#{flydata_tabledef[:table_name]}', 'cs', '#{escape(flydata_tabledef[:default_charset])}')"
|
268
268
|
values << "('#{flydata_tabledef[:table_name]}', 'revision', 1)"
|
269
|
+
values << "('#{flydata_tabledef[:table_name]}', 'src_ddl', '#{escape(flydata_tabledef[:src_ddl])}')"
|
269
270
|
sql += values.join(",\n") + ';'
|
270
271
|
sql
|
271
272
|
end
|
@@ -71,6 +71,43 @@ describe MysqlTableDef do
|
|
71
71
|
it 'comment should be set' do
|
72
72
|
expect(subject[:comment]).to eq('test table includes all kind of format type')
|
73
73
|
end
|
74
|
+
|
75
|
+
it 'return value with src_ddl' do
|
76
|
+
src_ddl = "CREATE TABLE `test_table_all` (
|
77
|
+
`id` bigint(20) NOT NULL AUTO_INCREMENT,
|
78
|
+
`col_binary` binary(100) DEFAULT NULL,
|
79
|
+
`col_blob` blob,
|
80
|
+
`col_bool` tinyint(1) DEFAULT '0',
|
81
|
+
`col_char` char(6) DEFAULT NULL,
|
82
|
+
`col_date` date DEFAULT NULL,
|
83
|
+
`col_datetime` datetime DEFAULT NULL,
|
84
|
+
`col_decimal` decimal(5,2) DEFAULT NULL,
|
85
|
+
`col_double` double DEFAULT NULL,
|
86
|
+
`col_float` float DEFAULT NULL,
|
87
|
+
`col_float_4_2` float(4,2) DEFAULT NULL,
|
88
|
+
`col_int` int(11) DEFAULT NULL,
|
89
|
+
`col_int_6` int(6) DEFAULT NULL,
|
90
|
+
`col_longblob` longblob,
|
91
|
+
`col_longtext` longtext,
|
92
|
+
`col_mediumblob` mediumblob,
|
93
|
+
`col_mediumint` mediumint(9) DEFAULT NULL,
|
94
|
+
`col_mediumtext` mediumtext,
|
95
|
+
`col_smallint` smallint(6) DEFAULT NULL,
|
96
|
+
`col_text` text,
|
97
|
+
`col_time` time DEFAULT NULL,
|
98
|
+
`col_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
99
|
+
`col_tinyblob` tinyblob,
|
100
|
+
`col_tinyint` tinyint(4) DEFAULT NULL,
|
101
|
+
`col_tinytext` tinytext,
|
102
|
+
`col_varbinary` varbinary(255) DEFAULT NULL,
|
103
|
+
`col_varchar` varchar(124) DEFAULT NULL,
|
104
|
+
`col_year` year DEFAULT NULL,
|
105
|
+
`col_year_4` year(4) DEFAULT NULL,
|
106
|
+
`col_year_2` year(2) DEFAULT NULL,
|
107
|
+
PRIMARY KEY (`id`)
|
108
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='test table includes all kind of format type';".split("\n").join
|
109
|
+
expect(subject[:src_ddl]).to eq(src_ddl)
|
110
|
+
end
|
74
111
|
end
|
75
112
|
|
76
113
|
context 'when table does not have primary key' do
|
@@ -113,7 +113,40 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
113
113
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_all';
|
114
114
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
115
115
|
('test_table_all', 'cs', 'UTF_8'),
|
116
|
-
('test_table_all', 'revision', 1)
|
116
|
+
('test_table_all', 'revision', 1),
|
117
|
+
('test_table_all', 'src_ddl', '#{"CREATE TABLE `test_table_all` (
|
118
|
+
`id` bigint(20) NOT NULL AUTO_INCREMENT,
|
119
|
+
`col_binary` binary(100) DEFAULT NULL,
|
120
|
+
`col_blob` blob,
|
121
|
+
`col_bool` tinyint(1) DEFAULT \\'0\\',
|
122
|
+
`col_char` char(6) DEFAULT NULL,
|
123
|
+
`col_date` date DEFAULT NULL,
|
124
|
+
`col_datetime` datetime DEFAULT NULL,
|
125
|
+
`col_decimal` decimal(5,2) DEFAULT NULL,
|
126
|
+
`col_double` double DEFAULT NULL,
|
127
|
+
`col_float` float DEFAULT NULL,
|
128
|
+
`col_float_4_2` float(4,2) DEFAULT NULL,
|
129
|
+
`col_int` int(11) DEFAULT NULL,
|
130
|
+
`col_int_6` int(6) DEFAULT NULL,
|
131
|
+
`col_longblob` longblob,
|
132
|
+
`col_longtext` longtext,
|
133
|
+
`col_mediumblob` mediumblob,
|
134
|
+
`col_mediumint` mediumint(9) DEFAULT NULL,
|
135
|
+
`col_mediumtext` mediumtext,
|
136
|
+
`col_smallint` smallint(6) DEFAULT NULL,
|
137
|
+
`col_text` text,
|
138
|
+
`col_time` time DEFAULT NULL,
|
139
|
+
`col_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
140
|
+
`col_tinyblob` tinyblob,
|
141
|
+
`col_tinyint` tinyint(4) DEFAULT NULL,
|
142
|
+
`col_tinytext` tinytext,
|
143
|
+
`col_varbinary` varbinary(255) DEFAULT NULL,
|
144
|
+
`col_varchar` varchar(124) DEFAULT NULL,
|
145
|
+
`col_year` year DEFAULT NULL,
|
146
|
+
`col_year_4` year(4) DEFAULT NULL,
|
147
|
+
`col_year_2` year(2) DEFAULT NULL,
|
148
|
+
PRIMARY KEY (`id`)
|
149
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT=\\'test table includes all kind of format type\\';".split("\n").join}');
|
117
150
|
EOT
|
118
151
|
end
|
119
152
|
end
|
@@ -155,7 +188,13 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
155
188
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'bit_test_def_1';
|
156
189
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
157
190
|
('bit_test_def_1', 'cs', 'UTF_8'),
|
158
|
-
('bit_test_def_1', 'revision', 1)
|
191
|
+
('bit_test_def_1', 'revision', 1),
|
192
|
+
('bit_test_def_1', 'src_ddl', '#{"CREATE TABLE `bit_test_def_1` (
|
193
|
+
`id` int(11) NOT NULL,
|
194
|
+
`bit_value` bit(1) DEFAULT b\\'1\\',
|
195
|
+
`int_value` int(11) DEFAULT x\\'10\\',
|
196
|
+
PRIMARY KEY (`id`)
|
197
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8;".split("\n").join}');
|
159
198
|
EOT
|
160
199
|
end
|
161
200
|
end
|
@@ -199,7 +238,18 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
199
238
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'product_order';
|
200
239
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
201
240
|
('product_order', 'cs', 'UTF_8'),
|
202
|
-
('product_order', 'revision', 1)
|
241
|
+
('product_order', 'revision', 1),
|
242
|
+
('product_order', 'src_ddl', '#{"CREATE TABLE `product_order` (
|
243
|
+
`no` int(11) NOT NULL AUTO_INCREMENT,
|
244
|
+
`product_category` int(11) NOT NULL,
|
245
|
+
`product_id` int(11) NOT NULL,
|
246
|
+
`customer_id` int(11) NOT NULL,
|
247
|
+
PRIMARY KEY (`no`),
|
248
|
+
KEY `product_category` (`product_category`,`product_id`),
|
249
|
+
KEY `customer_id` (`customer_id`),
|
250
|
+
CONSTRAINT `product_order_ibfk_1` FOREIGN KEY (`product_category`, `product_id`) REFERENCES `product` (`category`, `id`) ON UPDATE CASCADE,
|
251
|
+
CONSTRAINT `product_order_ibfk_2` FOREIGN KEY (`customer_id`) REFERENCES `customer` (`id`)
|
252
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8;".split("\n").join}');
|
203
253
|
EOT
|
204
254
|
end
|
205
255
|
end
|
@@ -241,7 +291,12 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
241
291
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_column_comment';
|
242
292
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
243
293
|
('test_table_column_comment', 'cs', 'UTF_8'),
|
244
|
-
('test_table_column_comment', 'revision', 1)
|
294
|
+
('test_table_column_comment', 'revision', 1),
|
295
|
+
('test_table_column_comment', 'src_ddl', '#{"CREATE TABLE `test_table_column_comment` (
|
296
|
+
`id` int(11) NOT NULL DEFAULT \\'0\\' COMMENT \\'this is primary key\\',
|
297
|
+
`value` text,
|
298
|
+
PRIMARY KEY (`id`)
|
299
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT=\\'column comment test\\';".split("\n").join}');
|
245
300
|
EOT
|
246
301
|
end
|
247
302
|
end
|
@@ -285,7 +340,14 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
285
340
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_enum';
|
286
341
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
287
342
|
('test_table_enum', 'cs', 'UTF_8'),
|
288
|
-
('test_table_enum', 'revision', 1)
|
343
|
+
('test_table_enum', 'revision', 1),
|
344
|
+
('test_table_enum', 'src_ddl', '#{"CREATE TABLE `test_table_enum` (
|
345
|
+
`id` int(11) NOT NULL,
|
346
|
+
`enum_1` enum(\\'apple\\',\\'orange\\',\\'banana\\') DEFAULT NULL,
|
347
|
+
`enum_2` enum(\\'a\\',\\'b\\',\\'c\\') DEFAULT \\'a\\',
|
348
|
+
`enum_3` enum(\\'e\\',\\'f\\',\\'g\\') NOT NULL,
|
349
|
+
PRIMARY KEY (`id`)
|
350
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT=\\'with enum column\\';".split("\n").join}');
|
289
351
|
EOT
|
290
352
|
end
|
291
353
|
end
|
@@ -327,7 +389,13 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
327
389
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_multi_pk';
|
328
390
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
329
391
|
('test_table_multi_pk', 'cs', 'UTF_8'),
|
330
|
-
('test_table_multi_pk', 'revision', 1)
|
392
|
+
('test_table_multi_pk', 'revision', 1),
|
393
|
+
('test_table_multi_pk', 'src_ddl', '#{"CREATE TABLE `test_table_multi_pk` (
|
394
|
+
`id1` int(11) NOT NULL DEFAULT \\'0\\',
|
395
|
+
`id2` int(11) NOT NULL DEFAULT \\'0\\',
|
396
|
+
`value` text,
|
397
|
+
PRIMARY KEY (`id1`,`id2`)
|
398
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT=\\'multiple primary key\\';".split("\n").join}');
|
331
399
|
EOT
|
332
400
|
end
|
333
401
|
end
|
@@ -379,7 +447,16 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
379
447
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'sample1';
|
380
448
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
381
449
|
('sample1', 'cs', 'ISO_8859_1'),
|
382
|
-
('sample1', 'revision', 1)
|
450
|
+
('sample1', 'revision', 1),
|
451
|
+
('sample1', 'src_ddl', '#{"CREATE TABLE `sample1` (
|
452
|
+
`id` int(11) NOT NULL AUTO_INCREMENT,
|
453
|
+
`title` varchar(256) DEFAULT NULL,
|
454
|
+
`name` text,
|
455
|
+
`num` int(11) DEFAULT NULL,
|
456
|
+
PRIMARY KEY (`id`),
|
457
|
+
UNIQUE KEY `title` (`title`),
|
458
|
+
UNIQUE KEY `num` (`num`)
|
459
|
+
) ENGINE=InnoDB DEFAULT CHARSET=latin1;".split("\n").join}');
|
383
460
|
EOT
|
384
461
|
end
|
385
462
|
end
|
@@ -423,7 +500,16 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
423
500
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'sample1';
|
424
501
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
425
502
|
('sample1', 'cs', 'ISO_8859_1'),
|
426
|
-
('sample1', 'revision', 1)
|
503
|
+
('sample1', 'revision', 1),
|
504
|
+
('sample1', 'src_ddl', '#{"CREATE TABLE `sample1` (
|
505
|
+
`id` int(11) NOT NULL AUTO_INCREMENT,
|
506
|
+
`title` varchar(256) DEFAULT NULL,
|
507
|
+
`name` text,
|
508
|
+
`num` int(11) DEFAULT NULL,
|
509
|
+
PRIMARY KEY (`id`),
|
510
|
+
UNIQUE KEY `title` (`title`),
|
511
|
+
UNIQUE KEY `index_num` (`num`)
|
512
|
+
) ENGINE=InnoDB DEFAULT CHARSET=latin1;".split("\n").join}');
|
427
513
|
EOT
|
428
514
|
end
|
429
515
|
end
|
@@ -495,7 +581,29 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
495
581
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'invoice_items';
|
496
582
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
497
583
|
('invoice_items', 'cs', 'UTF_8'),
|
498
|
-
('invoice_items', 'revision', 1)
|
584
|
+
('invoice_items', 'revision', 1),
|
585
|
+
('invoice_items', 'src_ddl', '#{"CREATE TABLE `invoice_items` (
|
586
|
+
`id` int(11) NOT NULL AUTO_INCREMENT,
|
587
|
+
`app_id` int(11) NOT NULL,
|
588
|
+
`subscription_id` int(11) NOT NULL,
|
589
|
+
`overage_id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
|
590
|
+
`stripe_invoice_item_id` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
|
591
|
+
`stripe_error` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
|
592
|
+
`synced_to_stripe` tinyint(1) NOT NULL DEFAULT \\'0\\',
|
593
|
+
`description` text COLLATE utf8mb4_unicode_ci,
|
594
|
+
`item_name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
|
595
|
+
`item_cost` int(11) NOT NULL,
|
596
|
+
`item_count` int(11) NOT NULL,
|
597
|
+
`total_cost` int(11) NOT NULL,
|
598
|
+
`created_at` datetime DEFAULT NULL,
|
599
|
+
`updated_at` datetime DEFAULT NULL,
|
600
|
+
`bill_ahead` tinyint(1) DEFAULT \\'0\\',
|
601
|
+
`bill_ahead_resolved_at` datetime DEFAULT NULL,
|
602
|
+
`stripe_invoice_id` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
|
603
|
+
`is_vat` tinyint(1) NOT NULL DEFAULT \\'0\\',
|
604
|
+
PRIMARY KEY (`id`),
|
605
|
+
UNIQUE KEY `index_invoice_items_on_app_id_and_overage_id_and_item_name` (`app_id`,`overage_id`(32),`item_name`(32)) USING BTREE
|
606
|
+
) ENGINE=InnoDB AUTO_INCREMENT=16654 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;".split("\n").join}');
|
499
607
|
EOT
|
500
608
|
end
|
501
609
|
end
|
@@ -545,7 +653,17 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
545
653
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'zerofill_table';
|
546
654
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
547
655
|
('zerofill_table', 'cs', 'ISO_8859_1'),
|
548
|
-
('zerofill_table', 'revision', 1)
|
656
|
+
('zerofill_table', 'revision', 1),
|
657
|
+
('zerofill_table', 'src_ddl', '#{"CREATE TABLE `zerofill_table` (
|
658
|
+
`id` int(11) NOT NULL AUTO_INCREMENT,
|
659
|
+
`value_int` int(10) unsigned zerofill DEFAULT NULL,
|
660
|
+
`value_float` float unsigned zerofill DEFAULT NULL,
|
661
|
+
`value_dec` decimal(10,2) unsigned zerofill DEFAULT NULL,
|
662
|
+
`value_double` double unsigned zerofill DEFAULT NULL,
|
663
|
+
`name` varchar(256) DEFAULT NULL,
|
664
|
+
`value_small_int` smallint(5) unsigned DEFAULT NULL,
|
665
|
+
PRIMARY KEY (`id`)
|
666
|
+
) ENGINE=InnoDB AUTO_INCREMENT=7 DEFAULT CHARSET=latin1;".split("\n").join}');
|
549
667
|
EOT
|
550
668
|
end
|
551
669
|
end
|
@@ -20,7 +20,8 @@ describe RedshiftTableDef do
|
|
20
20
|
{ column: "age", type: "int4(11) unsigned" },
|
21
21
|
value_column,
|
22
22
|
],
|
23
|
-
default_charset: "UTF_8"
|
23
|
+
default_charset: "UTF_8",
|
24
|
+
src_ddl: "dummy_src_ddl"} }
|
24
25
|
let(:option) { { flydata_ctl_table: false } }
|
25
26
|
|
26
27
|
let(:schema_prefix) { "" }
|
@@ -43,7 +44,8 @@ INSERT INTO #{schema_prefix}"flydata_ctl_columns" (table_name, column_name, src_
|
|
43
44
|
DELETE FROM #{schema_prefix}"flydata_ctl_tables" WHERE table_name = 'test_table';
|
44
45
|
INSERT INTO #{schema_prefix}"flydata_ctl_tables" (table_name, attribute, value) VALUES
|
45
46
|
('test_table', 'cs', 'UTF_8'),
|
46
|
-
('test_table', 'revision', 1)
|
47
|
+
('test_table', 'revision', 1),
|
48
|
+
('test_table', 'src_ddl', 'dummy_src_ddl');
|
47
49
|
EOT
|
48
50
|
|
49
51
|
let(:flydata_ctl_create) { <<EOT.strip }
|
data/flydata.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: flydata 0.3.
|
5
|
+
# stub: flydata 0.3.20 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "flydata"
|
9
|
-
s.version = "0.3.
|
9
|
+
s.version = "0.3.20"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib"]
|
13
13
|
s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
|
14
|
-
s.date = "2015-05-
|
14
|
+
s.date = "2015-05-13"
|
15
15
|
s.description = "FlyData Agent"
|
16
16
|
s.email = "sysadmin@flydata.com"
|
17
17
|
s.executables = ["fdmysqldump", "flydata", "serverinfo"]
|
@@ -1,11 +1,12 @@
|
|
1
1
|
module Fluent
|
2
2
|
|
3
3
|
class IdleEventDetector
|
4
|
-
def initialize(initial_interval, continuous_interval, check_interval = 5)
|
4
|
+
def initialize(initial_interval, continuous_interval, check_interval = 5, idle_timeout = 3600)
|
5
5
|
@mutex = Mutex.new
|
6
6
|
@initial_interval = initial_interval
|
7
7
|
@continuous_interval = continuous_interval
|
8
8
|
@check_interval = check_interval
|
9
|
+
@idle_timeout = idle_timeout
|
9
10
|
end
|
10
11
|
|
11
12
|
def start(&block)
|
@@ -50,7 +51,10 @@ class IdleEventDetector
|
|
50
51
|
def check_state
|
51
52
|
@mutex.synchronize do
|
52
53
|
if @is_absent
|
53
|
-
if Time.current - @
|
54
|
+
if Time.current - @last_event_at >= @idle_timeout
|
55
|
+
@callback.call(:event_idle_timeout, @last_event_at) if @callback
|
56
|
+
@timestamp = Time.current
|
57
|
+
elsif Time.current - @timestamp >= @continuous_interval
|
54
58
|
@callback.call(:event_still_not_coming, @last_event_at) if @callback
|
55
59
|
@timestamp = Time.current
|
56
60
|
end
|
@@ -52,9 +52,10 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
|
|
52
52
|
config_param :database, :string
|
53
53
|
config_param :tables, :string
|
54
54
|
config_param :tables_append_only, :string
|
55
|
-
config_param :initial_idle_interval, :integer, :default => 30
|
56
|
-
config_param :continuous_idle_interval, :integer, :default => 600
|
57
|
-
config_param :
|
55
|
+
config_param :initial_idle_interval, :integer, :default => 30 # 30 sec
|
56
|
+
config_param :continuous_idle_interval, :integer, :default => 600 # 600 sec
|
57
|
+
config_param :idle_timeout, :time, :default => 3600 # 3600 sec
|
58
|
+
config_param :check_interval, :integer, :default => 5 # 5 sec
|
58
59
|
config_param :ssl_ca_content, :string, :default => ''
|
59
60
|
|
60
61
|
def configure(conf)
|
@@ -118,7 +119,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
|
|
118
119
|
table_meta: table_meta, table_revs: table_revs,
|
119
120
|
)
|
120
121
|
@record_dispatcher = Mysql::FlydataBinlogRecordDispatcher.new(@context)
|
121
|
-
@idle_event_detector = IdleEventDetector.new(@initial_idle_interval, @continuous_idle_interval, @check_interval)
|
122
|
+
@idle_event_detector = IdleEventDetector.new(@initial_idle_interval, @continuous_idle_interval, @check_interval, @idle_timeout)
|
122
123
|
end
|
123
124
|
|
124
125
|
def start
|
@@ -131,6 +132,9 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
|
|
131
132
|
$log.warn "No binary log event since #{timestamp}"
|
132
133
|
when :event_arrived_finally
|
133
134
|
$log.info "Binary log event has come at #{timestamp}"
|
135
|
+
when :event_idle_timeout
|
136
|
+
$log.error "No binary log event since #{timestamp}. Restart the process."
|
137
|
+
Process.kill(:HUP, Process.ppid)
|
134
138
|
end
|
135
139
|
end
|
136
140
|
|
@@ -11,8 +11,9 @@ describe IdleEventDetector do
|
|
11
11
|
let(:initial_interval) { 0.1 }
|
12
12
|
let(:continuous_interval) { 0.18 }
|
13
13
|
let(:check_interval) { 0.1 }
|
14
|
+
let(:idle_timeout) { 1.0 }
|
14
15
|
|
15
|
-
subject { described_class.new(initial_interval, continuous_interval, check_interval) }
|
16
|
+
subject { described_class.new(initial_interval, continuous_interval, check_interval, idle_timeout) }
|
16
17
|
|
17
18
|
describe '#start' do
|
18
19
|
context 'without block' do
|
@@ -380,6 +381,58 @@ describe IdleEventDetector do
|
|
380
381
|
@arrived_callback_called = false
|
381
382
|
end
|
382
383
|
end
|
384
|
+
|
385
|
+
context 'no event during test which is long enough for idle timeout callbacks' do
|
386
|
+
let(:callback) {
|
387
|
+
Proc.new do |reason|
|
388
|
+
case reason
|
389
|
+
when :event_not_coming
|
390
|
+
@initial_callback_called = true
|
391
|
+
when :event_still_not_coming
|
392
|
+
@continuous_callback_called = true
|
393
|
+
when :event_arrived_finally
|
394
|
+
@arrived_callback_called = true
|
395
|
+
when
|
396
|
+
@timeout_callback_called = true
|
397
|
+
end
|
398
|
+
end
|
399
|
+
}
|
400
|
+
before do
|
401
|
+
Timecop.freeze(time)
|
402
|
+
@initial_callback_called = false
|
403
|
+
@continuous_callback_called = false
|
404
|
+
@arrived_callback_called = false
|
405
|
+
@timeout_callback_called = false
|
406
|
+
end
|
407
|
+
it 'fires idle_timeout callbacks after an event_not_coming callback' do
|
408
|
+
subject.start(&callback)
|
409
|
+
|
410
|
+
Timecop.freeze(time + initial_interval * 1.1)
|
411
|
+
|
412
|
+
sleep check_interval * 1.2
|
413
|
+
|
414
|
+
Timecop.freeze(time + initial_interval + continuous_interval * 1.1)
|
415
|
+
|
416
|
+
sleep check_interval * 1.2
|
417
|
+
|
418
|
+
Timecop.freeze(time + initial_interval + idle_timeout * 1.1)
|
419
|
+
|
420
|
+
sleep check_interval * 1.2
|
421
|
+
|
422
|
+
expect(@initial_callback_called).to eq true
|
423
|
+
expect(@continuous_callback_called).to eq true
|
424
|
+
expect(@arrived_callback_called).to eq false
|
425
|
+
expect(@timeout_callback_called).to eq true
|
426
|
+
|
427
|
+
subject.stop
|
428
|
+
end
|
429
|
+
after do
|
430
|
+
Timecop.return
|
431
|
+
@initial_callback_called = false
|
432
|
+
@continuous_callback_called = false
|
433
|
+
@arrived_callback_called = false
|
434
|
+
end
|
435
|
+
end
|
383
436
|
end
|
384
437
|
describe '#stop' do
|
385
438
|
context 'stop without corresponding start' do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.20
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2015-05-
|
15
|
+
date: 2015-05-13 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|