fluent-plugin-mysql-appender 0.2.7 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +0 -0
- data/.travis.yml +0 -0
- data/Gemfile +1 -1
- data/{LICENSE.txt → LICENSE} +1 -1
- data/README.md +62 -20
- data/Rakefile +0 -0
- data/example/mysql_multi_table_to_treasure_data.md +65 -0
- data/example/mysql_single_table_to_treasure_data.md +49 -0
- data/fluent-plugin-mysql-appender.gemspec +2 -2
- data/lib/fluent/plugin/in_mysql_appender_multi.rb +1 -1
- data/test/helper.rb +1 -1
- metadata +6 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1afa4ae7160361a720f0ada38c03abf549b017de
|
4
|
+
data.tar.gz: 5d166fe972de1046026b596318ac142b61a22e46
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dd56bd479c62a0bef34cc5443c3383b6675e6549a0adf73906e79a7a33216e9a5329c99206c468bb81c945102368a20034d418637b375c20512a9f89fd6221b9
|
7
|
+
data.tar.gz: 1074dd4909f86589bccf2ca740feaba86e3d5d66c0d90670d9842dfc439712072632a5565647a30bb01b0ea11e1c61e58e9cb60732ef16a3cc14d10e1992b76f
|
data/.gitignore
CHANGED
File without changes
|
data/.travis.yml
CHANGED
File without changes
|
data/Gemfile
CHANGED
data/{LICENSE.txt → LICENSE}
RENAMED
data/README.md
CHANGED
@@ -1,41 +1,83 @@
|
|
1
|
-
#
|
1
|
+
# fluent-plugin-mysql-appender
|
2
2
|
|
3
|
-
|
3
|
+
## Overview
|
4
4
|
|
5
|
-
|
5
|
+
Fluentd input plugin to track insert event only from MySQL database server.
|
6
|
+
Simple incremental id's insert.
|
6
7
|
|
7
8
|
## Installation
|
8
9
|
|
9
|
-
|
10
|
+
install with gem or fluent-gem command as:
|
10
11
|
|
11
|
-
|
12
|
-
|
13
|
-
|
12
|
+
`````
|
13
|
+
# for system installed fluentd
|
14
|
+
$ gem install fluent-plugin-mysql-appender
|
14
15
|
|
15
|
-
|
16
|
+
# for td-agent2
|
17
|
+
$ td-agent-gem install fluent-plugin-mysql-appender
|
18
|
+
`````
|
16
19
|
|
17
|
-
|
20
|
+
## Included plugins
|
18
21
|
|
19
|
-
|
22
|
+
* Input Plugin: mysql_appender
|
23
|
+
* Input Plugin: mysql_appender_multi
|
20
24
|
|
21
|
-
|
25
|
+
## Output example
|
22
26
|
|
23
|
-
|
27
|
+
It is a example when detecting insert events.
|
24
28
|
|
25
|
-
|
29
|
+
### sample query
|
26
30
|
|
27
|
-
|
31
|
+
`````
|
32
|
+
$ mysql -e "create database myweb"
|
33
|
+
$ mysql myweb -e "create table search_test(id int auto_increment, text text, PRIMARY KEY (id))"
|
34
|
+
$ sleep 10
|
35
|
+
$ mysql myweb -e "insert into search_test(text) values('aaa')"
|
36
|
+
`````
|
28
37
|
|
29
|
-
|
38
|
+
### result
|
30
39
|
|
31
|
-
|
40
|
+
`````
|
41
|
+
$ tail -f /var/log/td-agent/td-agent.log
|
42
|
+
2013-11-25 18:22:25 +0900 appender.myweb.search_test: {"id":"1","text":"aaa"}
|
43
|
+
`````
|
32
44
|
|
33
|
-
|
45
|
+
mysql query log is below
|
34
46
|
|
35
|
-
|
47
|
+
`````
|
48
|
+
$ tail -f /var/log/mysql/general-query.log
|
49
|
+
161108 19:25:52 4 Connect root@localhost on myweb
|
50
|
+
4 Query SELECT id, text FROM search_test where id > -1 order by id asc
|
51
|
+
4 Quit
|
52
|
+
161108 19:26:02 5 Connect root@localhost on myweb
|
53
|
+
4 Query SELECT id, text FROM search_test where id > 1 order by id asc
|
54
|
+
4 Quit
|
55
|
+
`````
|
36
56
|
|
57
|
+
## Tutorial
|
37
58
|
|
38
|
-
|
59
|
+
### mysql_appender
|
39
60
|
|
40
|
-
|
61
|
+
see example/mysql_single_table_to_tresure_data.md.
|
62
|
+
|
63
|
+
**Features**
|
64
|
+
|
65
|
+
* Table (or view table) synchronization supported.
|
66
|
+
* Replicate small record under a millons table.
|
67
|
+
|
68
|
+
### mysql_appender_multi
|
69
|
+
|
70
|
+
see example/mysql_multi_table_to_tresure_data.md.
|
71
|
+
|
72
|
+
**Features**
|
73
|
+
|
74
|
+
* table (or view table) synchronization supported.
|
75
|
+
* Multiple table synchronization supported and its DSN stored in yaml file.
|
76
|
+
|
77
|
+
## TODO
|
78
|
+
|
79
|
+
Pull requests are very welcome like below!!
|
80
|
+
|
81
|
+
* more documents
|
82
|
+
* more tests.
|
41
83
|
|
data/Rakefile
CHANGED
File without changes
|
@@ -0,0 +1,65 @@
|
|
1
|
+
## case study
|
2
|
+
|
3
|
+
It is a guide to replicate multiple mysql table to treasure data.
|
4
|
+
|
5
|
+
## configuration
|
6
|
+
|
7
|
+
```
|
8
|
+
<source>
|
9
|
+
type mysql_appender_multi
|
10
|
+
|
11
|
+
# Set connection settings for replicate source.
|
12
|
+
host localhost
|
13
|
+
username your_mysql_user
|
14
|
+
password your_mysql_password
|
15
|
+
database myweb
|
16
|
+
|
17
|
+
interval 1m # execute query interval (default: 1m)
|
18
|
+
yaml_path "in_tables.yml"
|
19
|
+
</source>
|
20
|
+
|
21
|
+
<match appender_multi.*.*>
|
22
|
+
type tdlog
|
23
|
+
endpoint your_td_endpoint
|
24
|
+
apikey your_td_apikey
|
25
|
+
|
26
|
+
auto_create_table
|
27
|
+
buffer_type file
|
28
|
+
buffer_path /var/log/td-agent/buffer/td
|
29
|
+
flush_interval 1m
|
30
|
+
use_ssl true
|
31
|
+
num_threads 8
|
32
|
+
|
33
|
+
<secondary>
|
34
|
+
@type file
|
35
|
+
path /var/log/td-agent/failed_records
|
36
|
+
compress gzip
|
37
|
+
</secondary>
|
38
|
+
</match>
|
39
|
+
```
|
40
|
+
|
41
|
+
Sample "in_tables.yml" is below.
|
42
|
+
|
43
|
+
```
|
44
|
+
- table_name: test_tbl1
|
45
|
+
primary_key: id
|
46
|
+
time_column: created_at
|
47
|
+
limit: 1000
|
48
|
+
columns:
|
49
|
+
- id
|
50
|
+
- column1
|
51
|
+
- column2
|
52
|
+
last_id: -1
|
53
|
+
td_database: sample_datasets
|
54
|
+
|
55
|
+
- table_name: test_tbl2
|
56
|
+
primary_key: id
|
57
|
+
time_column: created_at
|
58
|
+
limit: 1000
|
59
|
+
columns:
|
60
|
+
- id
|
61
|
+
- column1
|
62
|
+
- column2
|
63
|
+
last_id: -1
|
64
|
+
td_database: sample_datasets
|
65
|
+
```
|
@@ -0,0 +1,49 @@
|
|
1
|
+
## case study
|
2
|
+
|
3
|
+
It is a guide to replicate single mysql table to treasure data.
|
4
|
+
|
5
|
+
## configuration
|
6
|
+
|
7
|
+
```
|
8
|
+
<source>
|
9
|
+
type mysql_appender
|
10
|
+
|
11
|
+
# Set connection settings for replicate source.
|
12
|
+
host localhost
|
13
|
+
username your_mysql_user
|
14
|
+
password your_mysql_password
|
15
|
+
database myweb
|
16
|
+
|
17
|
+
# Set replicate query configuration.
|
18
|
+
query SELECT id, text, updated_at from search_test;
|
19
|
+
primary_key id # specify incremental unique key (default: id)
|
20
|
+
interval 1m # execute query interval (default: 1m)
|
21
|
+
|
22
|
+
# Format output tag for each events.
|
23
|
+
tag appender.myweb.your_td_database.your_td_table
|
24
|
+
|
25
|
+
time_column created_at # specify TIME column.
|
26
|
+
limit 1000 # query limit
|
27
|
+
last_id -1 # specify primary_key start
|
28
|
+
</source>
|
29
|
+
|
30
|
+
<match appender.*.*>
|
31
|
+
type tdlog
|
32
|
+
endpoint your_td_endpoint
|
33
|
+
apikey your_td_apikey
|
34
|
+
|
35
|
+
auto_create_table
|
36
|
+
buffer_type file
|
37
|
+
buffer_path /var/log/td-agent/buffer/td
|
38
|
+
flush_interval 1m
|
39
|
+
use_ssl true
|
40
|
+
num_threads 8
|
41
|
+
|
42
|
+
<secondary>
|
43
|
+
@type file
|
44
|
+
path /var/log/td-agent/failed_records
|
45
|
+
compress gzip
|
46
|
+
</secondary>
|
47
|
+
</match>
|
48
|
+
```
|
49
|
+
|
@@ -1,13 +1,13 @@
|
|
1
1
|
# coding: utf-8
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "fluent-plugin-mysql-appender"
|
4
|
-
spec.version = "0.
|
4
|
+
spec.version = "0.3.0"
|
5
5
|
spec.authors = ["TERASAKI Tsuyoshi"]
|
6
6
|
spec.email = ["tsuyoshi_terasaki@realworld.jp"]
|
7
7
|
|
8
8
|
spec.summary = %q{Fluentd input plugin to insert from MySQL database server.}
|
9
9
|
spec.description = %q{Simple incremental id's insert.}
|
10
|
-
spec.homepage = "https://github.com/
|
10
|
+
spec.homepage = "https://github.com/tetuyosi/fluent-plugin-mysql-appender"
|
11
11
|
spec.license = "MIT"
|
12
12
|
|
13
13
|
spec.files = `git ls-files`.split($\)
|
@@ -32,7 +32,7 @@ module Fluent
|
|
32
32
|
raise Fluent::ConfigError, "mysql_appender_multi: missing 'yaml_path' parameter or file not found."
|
33
33
|
end
|
34
34
|
|
35
|
-
if
|
35
|
+
if !File.exist?(@yaml_path)
|
36
36
|
raise Fluent::ConfigError, "mysql_appender_multi: 'yaml_path' No such file."
|
37
37
|
end
|
38
38
|
|
data/test/helper.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-mysql-appender
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TERASAKI Tsuyoshi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-11-
|
11
|
+
date: 2016-11-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -76,15 +76,17 @@ files:
|
|
76
76
|
- ".gitignore"
|
77
77
|
- ".travis.yml"
|
78
78
|
- Gemfile
|
79
|
-
- LICENSE
|
79
|
+
- LICENSE
|
80
80
|
- README.md
|
81
81
|
- Rakefile
|
82
|
+
- example/mysql_multi_table_to_treasure_data.md
|
83
|
+
- example/mysql_single_table_to_treasure_data.md
|
82
84
|
- fluent-plugin-mysql-appender.gemspec
|
83
85
|
- lib/fluent/plugin/in_mysql_appender.rb
|
84
86
|
- lib/fluent/plugin/in_mysql_appender_multi.rb
|
85
87
|
- test/helper.rb
|
86
88
|
- test/plugin/test_in_mysql_appender.rb
|
87
|
-
homepage: https://github.com/
|
89
|
+
homepage: https://github.com/tetuyosi/fluent-plugin-mysql-appender
|
88
90
|
licenses:
|
89
91
|
- MIT
|
90
92
|
metadata: {}
|