flydata 0.3.19 → 0.3.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb +1 -0
- data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +1 -0
- data/flydata-core/spec/table_def/mysql_table_def_spec.rb +37 -0
- data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb +128 -10
- data/flydata-core/spec/table_def/redshift_table_def_spec.rb +4 -2
- data/flydata.gemspec +3 -3
- data/lib/flydata/fluent-plugins/idle_event_detector.rb +6 -2
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +8 -4
- data/spec/flydata/fluent-plugins/idle_event_detector_spec.rb +54 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e06e855feb4eedb44d34602d0e6d447f72eddc2f
|
|
4
|
+
data.tar.gz: 960b559b4006bfcea9fecb6b96ef0457f2eed949
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0cf8f7f398d49b1081be49e96d0111a5e9bb88102bdea6fb148a015629cd62d0140e11f22a0ad0e6bd490a3993d362a0d5c7af29fa58829434649487d0dcf7b2
|
|
7
|
+
data.tar.gz: cf814b03327a2230bb90058c82c72b3c6e1b06225a2048dff20dd6744656614418cf609bd7c4a452570d229ff34c1ba4eca54e33b99f202f6a6c54850fab183c
|
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
0.3.
|
|
1
|
+
0.3.20
|
|
@@ -266,6 +266,7 @@ EOS
|
|
|
266
266
|
values = []
|
|
267
267
|
values << "('#{flydata_tabledef[:table_name]}', 'cs', '#{escape(flydata_tabledef[:default_charset])}')"
|
|
268
268
|
values << "('#{flydata_tabledef[:table_name]}', 'revision', 1)"
|
|
269
|
+
values << "('#{flydata_tabledef[:table_name]}', 'src_ddl', '#{escape(flydata_tabledef[:src_ddl])}')"
|
|
269
270
|
sql += values.join(",\n") + ';'
|
|
270
271
|
sql
|
|
271
272
|
end
|
|
@@ -71,6 +71,43 @@ describe MysqlTableDef do
|
|
|
71
71
|
it 'comment should be set' do
|
|
72
72
|
expect(subject[:comment]).to eq('test table includes all kind of format type')
|
|
73
73
|
end
|
|
74
|
+
|
|
75
|
+
it 'return value with src_ddl' do
|
|
76
|
+
src_ddl = "CREATE TABLE `test_table_all` (
|
|
77
|
+
`id` bigint(20) NOT NULL AUTO_INCREMENT,
|
|
78
|
+
`col_binary` binary(100) DEFAULT NULL,
|
|
79
|
+
`col_blob` blob,
|
|
80
|
+
`col_bool` tinyint(1) DEFAULT '0',
|
|
81
|
+
`col_char` char(6) DEFAULT NULL,
|
|
82
|
+
`col_date` date DEFAULT NULL,
|
|
83
|
+
`col_datetime` datetime DEFAULT NULL,
|
|
84
|
+
`col_decimal` decimal(5,2) DEFAULT NULL,
|
|
85
|
+
`col_double` double DEFAULT NULL,
|
|
86
|
+
`col_float` float DEFAULT NULL,
|
|
87
|
+
`col_float_4_2` float(4,2) DEFAULT NULL,
|
|
88
|
+
`col_int` int(11) DEFAULT NULL,
|
|
89
|
+
`col_int_6` int(6) DEFAULT NULL,
|
|
90
|
+
`col_longblob` longblob,
|
|
91
|
+
`col_longtext` longtext,
|
|
92
|
+
`col_mediumblob` mediumblob,
|
|
93
|
+
`col_mediumint` mediumint(9) DEFAULT NULL,
|
|
94
|
+
`col_mediumtext` mediumtext,
|
|
95
|
+
`col_smallint` smallint(6) DEFAULT NULL,
|
|
96
|
+
`col_text` text,
|
|
97
|
+
`col_time` time DEFAULT NULL,
|
|
98
|
+
`col_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
99
|
+
`col_tinyblob` tinyblob,
|
|
100
|
+
`col_tinyint` tinyint(4) DEFAULT NULL,
|
|
101
|
+
`col_tinytext` tinytext,
|
|
102
|
+
`col_varbinary` varbinary(255) DEFAULT NULL,
|
|
103
|
+
`col_varchar` varchar(124) DEFAULT NULL,
|
|
104
|
+
`col_year` year DEFAULT NULL,
|
|
105
|
+
`col_year_4` year(4) DEFAULT NULL,
|
|
106
|
+
`col_year_2` year(2) DEFAULT NULL,
|
|
107
|
+
PRIMARY KEY (`id`)
|
|
108
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='test table includes all kind of format type';".split("\n").join
|
|
109
|
+
expect(subject[:src_ddl]).to eq(src_ddl)
|
|
110
|
+
end
|
|
74
111
|
end
|
|
75
112
|
|
|
76
113
|
context 'when table does not have primary key' do
|
|
@@ -113,7 +113,40 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
|
113
113
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_all';
|
|
114
114
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
|
115
115
|
('test_table_all', 'cs', 'UTF_8'),
|
|
116
|
-
('test_table_all', 'revision', 1)
|
|
116
|
+
('test_table_all', 'revision', 1),
|
|
117
|
+
('test_table_all', 'src_ddl', '#{"CREATE TABLE `test_table_all` (
|
|
118
|
+
`id` bigint(20) NOT NULL AUTO_INCREMENT,
|
|
119
|
+
`col_binary` binary(100) DEFAULT NULL,
|
|
120
|
+
`col_blob` blob,
|
|
121
|
+
`col_bool` tinyint(1) DEFAULT \\'0\\',
|
|
122
|
+
`col_char` char(6) DEFAULT NULL,
|
|
123
|
+
`col_date` date DEFAULT NULL,
|
|
124
|
+
`col_datetime` datetime DEFAULT NULL,
|
|
125
|
+
`col_decimal` decimal(5,2) DEFAULT NULL,
|
|
126
|
+
`col_double` double DEFAULT NULL,
|
|
127
|
+
`col_float` float DEFAULT NULL,
|
|
128
|
+
`col_float_4_2` float(4,2) DEFAULT NULL,
|
|
129
|
+
`col_int` int(11) DEFAULT NULL,
|
|
130
|
+
`col_int_6` int(6) DEFAULT NULL,
|
|
131
|
+
`col_longblob` longblob,
|
|
132
|
+
`col_longtext` longtext,
|
|
133
|
+
`col_mediumblob` mediumblob,
|
|
134
|
+
`col_mediumint` mediumint(9) DEFAULT NULL,
|
|
135
|
+
`col_mediumtext` mediumtext,
|
|
136
|
+
`col_smallint` smallint(6) DEFAULT NULL,
|
|
137
|
+
`col_text` text,
|
|
138
|
+
`col_time` time DEFAULT NULL,
|
|
139
|
+
`col_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
140
|
+
`col_tinyblob` tinyblob,
|
|
141
|
+
`col_tinyint` tinyint(4) DEFAULT NULL,
|
|
142
|
+
`col_tinytext` tinytext,
|
|
143
|
+
`col_varbinary` varbinary(255) DEFAULT NULL,
|
|
144
|
+
`col_varchar` varchar(124) DEFAULT NULL,
|
|
145
|
+
`col_year` year DEFAULT NULL,
|
|
146
|
+
`col_year_4` year(4) DEFAULT NULL,
|
|
147
|
+
`col_year_2` year(2) DEFAULT NULL,
|
|
148
|
+
PRIMARY KEY (`id`)
|
|
149
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT=\\'test table includes all kind of format type\\';".split("\n").join}');
|
|
117
150
|
EOT
|
|
118
151
|
end
|
|
119
152
|
end
|
|
@@ -155,7 +188,13 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
|
155
188
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'bit_test_def_1';
|
|
156
189
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
|
157
190
|
('bit_test_def_1', 'cs', 'UTF_8'),
|
|
158
|
-
('bit_test_def_1', 'revision', 1)
|
|
191
|
+
('bit_test_def_1', 'revision', 1),
|
|
192
|
+
('bit_test_def_1', 'src_ddl', '#{"CREATE TABLE `bit_test_def_1` (
|
|
193
|
+
`id` int(11) NOT NULL,
|
|
194
|
+
`bit_value` bit(1) DEFAULT b\\'1\\',
|
|
195
|
+
`int_value` int(11) DEFAULT x\\'10\\',
|
|
196
|
+
PRIMARY KEY (`id`)
|
|
197
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8;".split("\n").join}');
|
|
159
198
|
EOT
|
|
160
199
|
end
|
|
161
200
|
end
|
|
@@ -199,7 +238,18 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
|
199
238
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'product_order';
|
|
200
239
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
|
201
240
|
('product_order', 'cs', 'UTF_8'),
|
|
202
|
-
('product_order', 'revision', 1)
|
|
241
|
+
('product_order', 'revision', 1),
|
|
242
|
+
('product_order', 'src_ddl', '#{"CREATE TABLE `product_order` (
|
|
243
|
+
`no` int(11) NOT NULL AUTO_INCREMENT,
|
|
244
|
+
`product_category` int(11) NOT NULL,
|
|
245
|
+
`product_id` int(11) NOT NULL,
|
|
246
|
+
`customer_id` int(11) NOT NULL,
|
|
247
|
+
PRIMARY KEY (`no`),
|
|
248
|
+
KEY `product_category` (`product_category`,`product_id`),
|
|
249
|
+
KEY `customer_id` (`customer_id`),
|
|
250
|
+
CONSTRAINT `product_order_ibfk_1` FOREIGN KEY (`product_category`, `product_id`) REFERENCES `product` (`category`, `id`) ON UPDATE CASCADE,
|
|
251
|
+
CONSTRAINT `product_order_ibfk_2` FOREIGN KEY (`customer_id`) REFERENCES `customer` (`id`)
|
|
252
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8;".split("\n").join}');
|
|
203
253
|
EOT
|
|
204
254
|
end
|
|
205
255
|
end
|
|
@@ -241,7 +291,12 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
|
241
291
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_column_comment';
|
|
242
292
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
|
243
293
|
('test_table_column_comment', 'cs', 'UTF_8'),
|
|
244
|
-
('test_table_column_comment', 'revision', 1)
|
|
294
|
+
('test_table_column_comment', 'revision', 1),
|
|
295
|
+
('test_table_column_comment', 'src_ddl', '#{"CREATE TABLE `test_table_column_comment` (
|
|
296
|
+
`id` int(11) NOT NULL DEFAULT \\'0\\' COMMENT \\'this is primary key\\',
|
|
297
|
+
`value` text,
|
|
298
|
+
PRIMARY KEY (`id`)
|
|
299
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT=\\'column comment test\\';".split("\n").join}');
|
|
245
300
|
EOT
|
|
246
301
|
end
|
|
247
302
|
end
|
|
@@ -285,7 +340,14 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
|
285
340
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_enum';
|
|
286
341
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
|
287
342
|
('test_table_enum', 'cs', 'UTF_8'),
|
|
288
|
-
('test_table_enum', 'revision', 1)
|
|
343
|
+
('test_table_enum', 'revision', 1),
|
|
344
|
+
('test_table_enum', 'src_ddl', '#{"CREATE TABLE `test_table_enum` (
|
|
345
|
+
`id` int(11) NOT NULL,
|
|
346
|
+
`enum_1` enum(\\'apple\\',\\'orange\\',\\'banana\\') DEFAULT NULL,
|
|
347
|
+
`enum_2` enum(\\'a\\',\\'b\\',\\'c\\') DEFAULT \\'a\\',
|
|
348
|
+
`enum_3` enum(\\'e\\',\\'f\\',\\'g\\') NOT NULL,
|
|
349
|
+
PRIMARY KEY (`id`)
|
|
350
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT=\\'with enum column\\';".split("\n").join}');
|
|
289
351
|
EOT
|
|
290
352
|
end
|
|
291
353
|
end
|
|
@@ -327,7 +389,13 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
|
327
389
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_multi_pk';
|
|
328
390
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
|
329
391
|
('test_table_multi_pk', 'cs', 'UTF_8'),
|
|
330
|
-
('test_table_multi_pk', 'revision', 1)
|
|
392
|
+
('test_table_multi_pk', 'revision', 1),
|
|
393
|
+
('test_table_multi_pk', 'src_ddl', '#{"CREATE TABLE `test_table_multi_pk` (
|
|
394
|
+
`id1` int(11) NOT NULL DEFAULT \\'0\\',
|
|
395
|
+
`id2` int(11) NOT NULL DEFAULT \\'0\\',
|
|
396
|
+
`value` text,
|
|
397
|
+
PRIMARY KEY (`id1`,`id2`)
|
|
398
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT=\\'multiple primary key\\';".split("\n").join}');
|
|
331
399
|
EOT
|
|
332
400
|
end
|
|
333
401
|
end
|
|
@@ -379,7 +447,16 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
|
379
447
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'sample1';
|
|
380
448
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
|
381
449
|
('sample1', 'cs', 'ISO_8859_1'),
|
|
382
|
-
('sample1', 'revision', 1)
|
|
450
|
+
('sample1', 'revision', 1),
|
|
451
|
+
('sample1', 'src_ddl', '#{"CREATE TABLE `sample1` (
|
|
452
|
+
`id` int(11) NOT NULL AUTO_INCREMENT,
|
|
453
|
+
`title` varchar(256) DEFAULT NULL,
|
|
454
|
+
`name` text,
|
|
455
|
+
`num` int(11) DEFAULT NULL,
|
|
456
|
+
PRIMARY KEY (`id`),
|
|
457
|
+
UNIQUE KEY `title` (`title`),
|
|
458
|
+
UNIQUE KEY `num` (`num`)
|
|
459
|
+
) ENGINE=InnoDB DEFAULT CHARSET=latin1;".split("\n").join}');
|
|
383
460
|
EOT
|
|
384
461
|
end
|
|
385
462
|
end
|
|
@@ -423,7 +500,16 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
|
423
500
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'sample1';
|
|
424
501
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
|
425
502
|
('sample1', 'cs', 'ISO_8859_1'),
|
|
426
|
-
('sample1', 'revision', 1)
|
|
503
|
+
('sample1', 'revision', 1),
|
|
504
|
+
('sample1', 'src_ddl', '#{"CREATE TABLE `sample1` (
|
|
505
|
+
`id` int(11) NOT NULL AUTO_INCREMENT,
|
|
506
|
+
`title` varchar(256) DEFAULT NULL,
|
|
507
|
+
`name` text,
|
|
508
|
+
`num` int(11) DEFAULT NULL,
|
|
509
|
+
PRIMARY KEY (`id`),
|
|
510
|
+
UNIQUE KEY `title` (`title`),
|
|
511
|
+
UNIQUE KEY `index_num` (`num`)
|
|
512
|
+
) ENGINE=InnoDB DEFAULT CHARSET=latin1;".split("\n").join}');
|
|
427
513
|
EOT
|
|
428
514
|
end
|
|
429
515
|
end
|
|
@@ -495,7 +581,29 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
|
495
581
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'invoice_items';
|
|
496
582
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
|
497
583
|
('invoice_items', 'cs', 'UTF_8'),
|
|
498
|
-
('invoice_items', 'revision', 1)
|
|
584
|
+
('invoice_items', 'revision', 1),
|
|
585
|
+
('invoice_items', 'src_ddl', '#{"CREATE TABLE `invoice_items` (
|
|
586
|
+
`id` int(11) NOT NULL AUTO_INCREMENT,
|
|
587
|
+
`app_id` int(11) NOT NULL,
|
|
588
|
+
`subscription_id` int(11) NOT NULL,
|
|
589
|
+
`overage_id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
|
|
590
|
+
`stripe_invoice_item_id` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
|
|
591
|
+
`stripe_error` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
|
|
592
|
+
`synced_to_stripe` tinyint(1) NOT NULL DEFAULT \\'0\\',
|
|
593
|
+
`description` text COLLATE utf8mb4_unicode_ci,
|
|
594
|
+
`item_name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
|
|
595
|
+
`item_cost` int(11) NOT NULL,
|
|
596
|
+
`item_count` int(11) NOT NULL,
|
|
597
|
+
`total_cost` int(11) NOT NULL,
|
|
598
|
+
`created_at` datetime DEFAULT NULL,
|
|
599
|
+
`updated_at` datetime DEFAULT NULL,
|
|
600
|
+
`bill_ahead` tinyint(1) DEFAULT \\'0\\',
|
|
601
|
+
`bill_ahead_resolved_at` datetime DEFAULT NULL,
|
|
602
|
+
`stripe_invoice_id` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
|
|
603
|
+
`is_vat` tinyint(1) NOT NULL DEFAULT \\'0\\',
|
|
604
|
+
PRIMARY KEY (`id`),
|
|
605
|
+
UNIQUE KEY `index_invoice_items_on_app_id_and_overage_id_and_item_name` (`app_id`,`overage_id`(32),`item_name`(32)) USING BTREE
|
|
606
|
+
) ENGINE=InnoDB AUTO_INCREMENT=16654 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;".split("\n").join}');
|
|
499
607
|
EOT
|
|
500
608
|
end
|
|
501
609
|
end
|
|
@@ -545,7 +653,17 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
|
545
653
|
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'zerofill_table';
|
|
546
654
|
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
|
547
655
|
('zerofill_table', 'cs', 'ISO_8859_1'),
|
|
548
|
-
('zerofill_table', 'revision', 1)
|
|
656
|
+
('zerofill_table', 'revision', 1),
|
|
657
|
+
('zerofill_table', 'src_ddl', '#{"CREATE TABLE `zerofill_table` (
|
|
658
|
+
`id` int(11) NOT NULL AUTO_INCREMENT,
|
|
659
|
+
`value_int` int(10) unsigned zerofill DEFAULT NULL,
|
|
660
|
+
`value_float` float unsigned zerofill DEFAULT NULL,
|
|
661
|
+
`value_dec` decimal(10,2) unsigned zerofill DEFAULT NULL,
|
|
662
|
+
`value_double` double unsigned zerofill DEFAULT NULL,
|
|
663
|
+
`name` varchar(256) DEFAULT NULL,
|
|
664
|
+
`value_small_int` smallint(5) unsigned DEFAULT NULL,
|
|
665
|
+
PRIMARY KEY (`id`)
|
|
666
|
+
) ENGINE=InnoDB AUTO_INCREMENT=7 DEFAULT CHARSET=latin1;".split("\n").join}');
|
|
549
667
|
EOT
|
|
550
668
|
end
|
|
551
669
|
end
|
|
@@ -20,7 +20,8 @@ describe RedshiftTableDef do
|
|
|
20
20
|
{ column: "age", type: "int4(11) unsigned" },
|
|
21
21
|
value_column,
|
|
22
22
|
],
|
|
23
|
-
default_charset: "UTF_8"
|
|
23
|
+
default_charset: "UTF_8",
|
|
24
|
+
src_ddl: "dummy_src_ddl"} }
|
|
24
25
|
let(:option) { { flydata_ctl_table: false } }
|
|
25
26
|
|
|
26
27
|
let(:schema_prefix) { "" }
|
|
@@ -43,7 +44,8 @@ INSERT INTO #{schema_prefix}"flydata_ctl_columns" (table_name, column_name, src_
|
|
|
43
44
|
DELETE FROM #{schema_prefix}"flydata_ctl_tables" WHERE table_name = 'test_table';
|
|
44
45
|
INSERT INTO #{schema_prefix}"flydata_ctl_tables" (table_name, attribute, value) VALUES
|
|
45
46
|
('test_table', 'cs', 'UTF_8'),
|
|
46
|
-
('test_table', 'revision', 1)
|
|
47
|
+
('test_table', 'revision', 1),
|
|
48
|
+
('test_table', 'src_ddl', 'dummy_src_ddl');
|
|
47
49
|
EOT
|
|
48
50
|
|
|
49
51
|
let(:flydata_ctl_create) { <<EOT.strip }
|
data/flydata.gemspec
CHANGED
|
@@ -2,16 +2,16 @@
|
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
|
4
4
|
# -*- encoding: utf-8 -*-
|
|
5
|
-
# stub: flydata 0.3.
|
|
5
|
+
# stub: flydata 0.3.20 ruby lib
|
|
6
6
|
|
|
7
7
|
Gem::Specification.new do |s|
|
|
8
8
|
s.name = "flydata"
|
|
9
|
-
s.version = "0.3.
|
|
9
|
+
s.version = "0.3.20"
|
|
10
10
|
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
|
12
12
|
s.require_paths = ["lib"]
|
|
13
13
|
s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
|
|
14
|
-
s.date = "2015-05-
|
|
14
|
+
s.date = "2015-05-13"
|
|
15
15
|
s.description = "FlyData Agent"
|
|
16
16
|
s.email = "sysadmin@flydata.com"
|
|
17
17
|
s.executables = ["fdmysqldump", "flydata", "serverinfo"]
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
module Fluent
|
|
2
2
|
|
|
3
3
|
class IdleEventDetector
|
|
4
|
-
def initialize(initial_interval, continuous_interval, check_interval = 5)
|
|
4
|
+
def initialize(initial_interval, continuous_interval, check_interval = 5, idle_timeout = 3600)
|
|
5
5
|
@mutex = Mutex.new
|
|
6
6
|
@initial_interval = initial_interval
|
|
7
7
|
@continuous_interval = continuous_interval
|
|
8
8
|
@check_interval = check_interval
|
|
9
|
+
@idle_timeout = idle_timeout
|
|
9
10
|
end
|
|
10
11
|
|
|
11
12
|
def start(&block)
|
|
@@ -50,7 +51,10 @@ class IdleEventDetector
|
|
|
50
51
|
def check_state
|
|
51
52
|
@mutex.synchronize do
|
|
52
53
|
if @is_absent
|
|
53
|
-
if Time.current - @
|
|
54
|
+
if Time.current - @last_event_at >= @idle_timeout
|
|
55
|
+
@callback.call(:event_idle_timeout, @last_event_at) if @callback
|
|
56
|
+
@timestamp = Time.current
|
|
57
|
+
elsif Time.current - @timestamp >= @continuous_interval
|
|
54
58
|
@callback.call(:event_still_not_coming, @last_event_at) if @callback
|
|
55
59
|
@timestamp = Time.current
|
|
56
60
|
end
|
|
@@ -52,9 +52,10 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
|
|
|
52
52
|
config_param :database, :string
|
|
53
53
|
config_param :tables, :string
|
|
54
54
|
config_param :tables_append_only, :string
|
|
55
|
-
config_param :initial_idle_interval, :integer, :default => 30
|
|
56
|
-
config_param :continuous_idle_interval, :integer, :default => 600
|
|
57
|
-
config_param :
|
|
55
|
+
config_param :initial_idle_interval, :integer, :default => 30 # 30 sec
|
|
56
|
+
config_param :continuous_idle_interval, :integer, :default => 600 # 600 sec
|
|
57
|
+
config_param :idle_timeout, :time, :default => 3600 # 3600 sec
|
|
58
|
+
config_param :check_interval, :integer, :default => 5 # 5 sec
|
|
58
59
|
config_param :ssl_ca_content, :string, :default => ''
|
|
59
60
|
|
|
60
61
|
def configure(conf)
|
|
@@ -118,7 +119,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
|
|
|
118
119
|
table_meta: table_meta, table_revs: table_revs,
|
|
119
120
|
)
|
|
120
121
|
@record_dispatcher = Mysql::FlydataBinlogRecordDispatcher.new(@context)
|
|
121
|
-
@idle_event_detector = IdleEventDetector.new(@initial_idle_interval, @continuous_idle_interval, @check_interval)
|
|
122
|
+
@idle_event_detector = IdleEventDetector.new(@initial_idle_interval, @continuous_idle_interval, @check_interval, @idle_timeout)
|
|
122
123
|
end
|
|
123
124
|
|
|
124
125
|
def start
|
|
@@ -131,6 +132,9 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
|
|
|
131
132
|
$log.warn "No binary log event since #{timestamp}"
|
|
132
133
|
when :event_arrived_finally
|
|
133
134
|
$log.info "Binary log event has come at #{timestamp}"
|
|
135
|
+
when :event_idle_timeout
|
|
136
|
+
$log.error "No binary log event since #{timestamp}. Restart the process."
|
|
137
|
+
Process.kill(:HUP, Process.ppid)
|
|
134
138
|
end
|
|
135
139
|
end
|
|
136
140
|
|
|
@@ -11,8 +11,9 @@ describe IdleEventDetector do
|
|
|
11
11
|
let(:initial_interval) { 0.1 }
|
|
12
12
|
let(:continuous_interval) { 0.18 }
|
|
13
13
|
let(:check_interval) { 0.1 }
|
|
14
|
+
let(:idle_timeout) { 1.0 }
|
|
14
15
|
|
|
15
|
-
subject { described_class.new(initial_interval, continuous_interval, check_interval) }
|
|
16
|
+
subject { described_class.new(initial_interval, continuous_interval, check_interval, idle_timeout) }
|
|
16
17
|
|
|
17
18
|
describe '#start' do
|
|
18
19
|
context 'without block' do
|
|
@@ -380,6 +381,58 @@ describe IdleEventDetector do
|
|
|
380
381
|
@arrived_callback_called = false
|
|
381
382
|
end
|
|
382
383
|
end
|
|
384
|
+
|
|
385
|
+
context 'no event during test which is long enough for idle timeout callbacks' do
|
|
386
|
+
let(:callback) {
|
|
387
|
+
Proc.new do |reason|
|
|
388
|
+
case reason
|
|
389
|
+
when :event_not_coming
|
|
390
|
+
@initial_callback_called = true
|
|
391
|
+
when :event_still_not_coming
|
|
392
|
+
@continuous_callback_called = true
|
|
393
|
+
when :event_arrived_finally
|
|
394
|
+
@arrived_callback_called = true
|
|
395
|
+
when
|
|
396
|
+
@timeout_callback_called = true
|
|
397
|
+
end
|
|
398
|
+
end
|
|
399
|
+
}
|
|
400
|
+
before do
|
|
401
|
+
Timecop.freeze(time)
|
|
402
|
+
@initial_callback_called = false
|
|
403
|
+
@continuous_callback_called = false
|
|
404
|
+
@arrived_callback_called = false
|
|
405
|
+
@timeout_callback_called = false
|
|
406
|
+
end
|
|
407
|
+
it 'fires idle_timeout callbacks after an event_not_coming callback' do
|
|
408
|
+
subject.start(&callback)
|
|
409
|
+
|
|
410
|
+
Timecop.freeze(time + initial_interval * 1.1)
|
|
411
|
+
|
|
412
|
+
sleep check_interval * 1.2
|
|
413
|
+
|
|
414
|
+
Timecop.freeze(time + initial_interval + continuous_interval * 1.1)
|
|
415
|
+
|
|
416
|
+
sleep check_interval * 1.2
|
|
417
|
+
|
|
418
|
+
Timecop.freeze(time + initial_interval + idle_timeout * 1.1)
|
|
419
|
+
|
|
420
|
+
sleep check_interval * 1.2
|
|
421
|
+
|
|
422
|
+
expect(@initial_callback_called).to eq true
|
|
423
|
+
expect(@continuous_callback_called).to eq true
|
|
424
|
+
expect(@arrived_callback_called).to eq false
|
|
425
|
+
expect(@timeout_callback_called).to eq true
|
|
426
|
+
|
|
427
|
+
subject.stop
|
|
428
|
+
end
|
|
429
|
+
after do
|
|
430
|
+
Timecop.return
|
|
431
|
+
@initial_callback_called = false
|
|
432
|
+
@continuous_callback_called = false
|
|
433
|
+
@arrived_callback_called = false
|
|
434
|
+
end
|
|
435
|
+
end
|
|
383
436
|
end
|
|
384
437
|
describe '#stop' do
|
|
385
438
|
context 'stop without corresponding start' do
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: flydata
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.20
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Koichi Fujikawa
|
|
@@ -12,7 +12,7 @@ authors:
|
|
|
12
12
|
autorequire:
|
|
13
13
|
bindir: bin
|
|
14
14
|
cert_chain: []
|
|
15
|
-
date: 2015-05-
|
|
15
|
+
date: 2015-05-13 00:00:00.000000000 Z
|
|
16
16
|
dependencies:
|
|
17
17
|
- !ruby/object:Gem::Dependency
|
|
18
18
|
name: rest-client
|