fluent-plugin-mysql-appender 0.3.4 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/example/mysql_multi_table_to_tresure_data.md +0 -2
- data/example/mysql_single_table_to_treasure_data.md +0 -1
- data/fluent-plugin-mysql-appender.gemspec +1 -1
- data/lib/fluent/plugin/in_mysql_appender.rb +15 -21
- data/lib/fluent/plugin/in_mysql_appender_multi.rb +17 -23
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b9f6408d1a7b224768ddef9852ce5f68344b15c7
|
|
4
|
+
data.tar.gz: 628c15bf44d0d25a1495652f4bc5aa291dad8749
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0414ae7c3bb515fc587e1e950da5e543787c98721f33a950eb086ddfd8b6056724bdb157c9ff666dbb2632ed48ef8e8af337d9c16614415610972ecce2f92a6e
|
|
7
|
+
data.tar.gz: a4813bb12472e8c40134dc46d8ae9e9e3983dda2c410ebac99277c4f2a8c847e20e17543bf0a5f8a0178a7e4f09ac77d17c0c2cab51f19c450429801b22a443e
|
|
@@ -50,7 +50,6 @@ Sample "in_tables.yml" is below.
|
|
|
50
50
|
- column1
|
|
51
51
|
- column2
|
|
52
52
|
last_id: -1
|
|
53
|
-
buffer: 10 # last 10 records don't append (default 0).
|
|
54
53
|
td_database: sample_datasets
|
|
55
54
|
|
|
56
55
|
- table_name: test_tbl2
|
|
@@ -62,6 +61,5 @@ Sample "in_tables.yml" is below.
|
|
|
62
61
|
- column1
|
|
63
62
|
- column2
|
|
64
63
|
last_id: -1
|
|
65
|
-
buffer: 10 # last 10 records don't append (default 0).
|
|
66
64
|
td_database: sample_datasets
|
|
67
65
|
```
|
|
@@ -25,7 +25,6 @@ It is a guide to replicate single mysql table to treasure data.
|
|
|
25
25
|
time_column created_at # specify TIME column.
|
|
26
26
|
limit 1000 # query limit
|
|
27
27
|
last_id -1 # specify primary_key start
|
|
28
|
-
buffer 10 # last 10 records don't append (default 0).
|
|
29
28
|
</source>
|
|
30
29
|
|
|
31
30
|
<match appender.*.*>
|
|
@@ -26,7 +26,6 @@ module Fluent
|
|
|
26
26
|
config_param :primary_key, :string, :default => 'id'
|
|
27
27
|
config_param :interval, :string, :default => '1m'
|
|
28
28
|
config_param :tag, :string, :default => nil
|
|
29
|
-
config_param :buffer, :integer, :default => 0
|
|
30
29
|
|
|
31
30
|
def configure(conf)
|
|
32
31
|
super
|
|
@@ -64,27 +63,22 @@ module Fluent
|
|
|
64
63
|
start_time = Time.now
|
|
65
64
|
select_query = @query.gsub(/"/,'') + " where #{primary_key} > #{last_id} order by #{primary_key} asc limit #{limit}"
|
|
66
65
|
rows, con = query(select_query, con)
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
66
|
+
rows.each_with_index do |row, index|
|
|
67
|
+
tag = format_tag(@tag, {:event => :insert})
|
|
68
|
+
if @time_column.nil? then
|
|
69
|
+
td_time = Engine.now
|
|
70
|
+
else
|
|
71
|
+
if row[@time_column].kind_of?(Time) then
|
|
72
|
+
td_time = row[@time_column].to_i
|
|
73
|
+
else
|
|
74
|
+
td_time = Time.parse(row[@time_column].to_s).to_i
|
|
75
|
+
end
|
|
70
76
|
end
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
if row[@time_column].kind_of?(Time) then
|
|
77
|
-
td_time = row[@time_column].to_i
|
|
78
|
-
else
|
|
79
|
-
td_time = Time.parse(row[@time_column].to_s).to_i
|
|
80
|
-
end
|
|
81
|
-
end
|
|
82
|
-
row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
|
|
83
|
-
router.emit(tag, td_time, row)
|
|
84
|
-
rows_count += 1
|
|
85
|
-
if index == rows.size - @buffer - 1
|
|
86
|
-
@last_id = row[@primary_key]
|
|
87
|
-
end
|
|
77
|
+
row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
|
|
78
|
+
router.emit(tag, td_time, row)
|
|
79
|
+
rows_count += 1
|
|
80
|
+
if index == rows.size - 1
|
|
81
|
+
@last_id = row[@primary_key]
|
|
88
82
|
end
|
|
89
83
|
end
|
|
90
84
|
con.close
|
|
@@ -29,11 +29,11 @@ module Fluent
|
|
|
29
29
|
@interval = Config.time_value(@interval)
|
|
30
30
|
|
|
31
31
|
if @yaml_path.nil?
|
|
32
|
-
raise Fluent::ConfigError, "mysql_appender_multi: missing 'yaml_path' parameter."
|
|
32
|
+
raise Fluent::ConfigError, "mysql_appender_multi: missing 'yaml_path' parameter or file not found."
|
|
33
33
|
end
|
|
34
34
|
|
|
35
35
|
if !File.exist?(@yaml_path)
|
|
36
|
-
raise Fluent::ConfigError, "mysql_appender_multi: No such file
|
|
36
|
+
raise Fluent::ConfigError, "mysql_appender_multi: 'yaml_path' No such file."
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
if @tag.nil?
|
|
@@ -66,7 +66,6 @@ module Fluent
|
|
|
66
66
|
def poll(config)
|
|
67
67
|
begin
|
|
68
68
|
tag = format_tag(config)
|
|
69
|
-
buffer = config['buffer'] || 0
|
|
70
69
|
@mutex.synchronize {
|
|
71
70
|
$log.info "mysql_replicator_multi: polling start. :tag=>#{tag}"
|
|
72
71
|
}
|
|
@@ -76,26 +75,21 @@ module Fluent
|
|
|
76
75
|
rows_count = 0
|
|
77
76
|
start_time = Time.now
|
|
78
77
|
rows, con = query(get_query(config, last_id), con)
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
router.emit(tag, td_time, row)
|
|
95
|
-
rows_count += 1
|
|
96
|
-
if index == (rows.size - buffer - 1)
|
|
97
|
-
last_id = row[config['primary_key']]
|
|
98
|
-
end
|
|
78
|
+
rows.each_with_index do |row, index|
|
|
79
|
+
if config['time_column'].nil? then
|
|
80
|
+
td_time = Engine.now
|
|
81
|
+
else
|
|
82
|
+
if row[config['time_column']].kind_of?(Time) then
|
|
83
|
+
td_time = row[config['time_column']].to_i
|
|
84
|
+
else
|
|
85
|
+
td_time = Time.parse(row[config['time_column']].to_s).to_i
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
|
|
89
|
+
router.emit(tag, td_time, row)
|
|
90
|
+
rows_count += 1
|
|
91
|
+
if index == rows.size - 1
|
|
92
|
+
last_id = row[config['primary_key']]
|
|
99
93
|
end
|
|
100
94
|
end
|
|
101
95
|
con.close
|