fluent-plugin-sql 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +77 -6
- data/VERSION +1 -1
- data/fluent-plugin-sql.gemspec +3 -5
- data/lib/fluent/plugin/in_sql.rb +16 -1
- metadata +14 -36
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f22da5fcea57030f5a458f011a543329737057a0
|
4
|
+
data.tar.gz: 9c334fb30a6ad62cc505b60006b91fbd3e2ea200
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d82f9b07974099025f990131f6d22337f71e1ef67cba5a61f634f0a476878a9294b1f87ac58124770b52aec57e9a30c03081ae8630a9f5d342f2c93005146b99
|
7
|
+
data.tar.gz: 738ea7e30cbb0ed9beecca8fa332388d2ae3b8750aa437aaa8bf63e522bf96d014f40eeb1afb99fdf38f81b52bcc596328348e1adc816e2011f0b5dd559cda77
|
data/README.md
CHANGED
@@ -2,9 +2,21 @@
|
|
2
2
|
|
3
3
|
## Overview
|
4
4
|
|
5
|
-
This
|
5
|
+
This SQL plugin has two parts:
|
6
6
|
|
7
|
-
|
7
|
+
1. SQL **input** plugin reads records from RDBMSes periodically. An example use case would be getting "diffs" of a table (based on the "updated_at" field).
|
8
|
+
2. SQL **output** plugin that writes records into RDBMes. An example use case would be aggregating server/app/sensor logs into RDBMS systems.
|
9
|
+
|
10
|
+
## Installation
|
11
|
+
|
12
|
+
$ fluent-gem install fluent-plugin-sql --no-document
|
13
|
+
$ fluent-gem install pg --no-document # for postgresql
|
14
|
+
|
15
|
+
You should install actual RDBMS driver gem together. `pg` gem for postgresql adapter or `mysql2` gem for `mysql2` adapter. Other adapters supported by [ActiveRecord](https://github.com/rails/rails/tree/master/activerecord) should work.
|
16
|
+
|
17
|
+
We recommend that mysql2 gem is higher than `0.3.12` and pg gem is higher than `0.16.0`.
|
18
|
+
|
19
|
+
## Input: How It Works
|
8
20
|
|
9
21
|
This plugin runs following SQL periodically:
|
10
22
|
|
@@ -15,14 +27,14 @@ If you omit to set *update\_column* parameter, it uses primary key.
|
|
15
27
|
|
16
28
|
It stores last selected rows to a file (named *state\_file*) to not forget the last row when Fluentd restarts.
|
17
29
|
|
18
|
-
## Configuration
|
30
|
+
## Input: Configuration
|
19
31
|
|
20
32
|
<source>
|
21
33
|
type sql
|
22
34
|
|
23
35
|
host rdb_host
|
24
36
|
database rdb_database
|
25
|
-
adapter
|
37
|
+
adapter mysql2_or_postgresql_or_etc
|
26
38
|
username myusername
|
27
39
|
password mypassword
|
28
40
|
|
@@ -54,7 +66,7 @@ It stores last selected rows to a file (named *state\_file*) to not forget the l
|
|
54
66
|
* **host** RDBMS host
|
55
67
|
* **port** RDBMS port
|
56
68
|
* **database** RDBMS database name
|
57
|
-
* **adapter** RDBMS driver name (mysql2 for
|
69
|
+
* **adapter** RDBMS driver name. You should install corresponding gem before start (mysql2 gem for mysql2 adapter, pg gem for postgresql adapter, etc.)
|
58
70
|
* **user** RDBMS login user name
|
59
71
|
* **password** RDBMS login password
|
60
72
|
* **tag_prefix** prefix of tags of events. actual tag will be this\_tag\_prefix.tables\_tag (optional)
|
@@ -70,8 +82,67 @@ It stores last selected rows to a file (named *state\_file*) to not forget the l
|
|
70
82
|
* **update_column**: see above description
|
71
83
|
* **time_column** (optional): if this option is set, this plugin uses this column's value as the the event's time. Otherwise it uses current time.
|
72
84
|
|
73
|
-
## Limitation
|
85
|
+
## Input: Limitation
|
74
86
|
|
75
87
|
You should make sure target tables have index (and/or partitions) on the *update\_column*. Otherwise SELECT causes full table scan and serious performance problem.
|
76
88
|
|
77
89
|
You can't replicate DELETEd rows.
|
90
|
+
|
91
|
+
## Output: How It Works
|
92
|
+
|
93
|
+
This plugin takes advantage of ActiveRecord underneath. For `host`, `port`, `database`, `adapter`, `username`, `password`, `socket` parameters, you can think of ActiveRecord's equivalent parameters.
|
94
|
+
|
95
|
+
## Output: Configuration
|
96
|
+
|
97
|
+
<match my.rdb.*>
|
98
|
+
type sql
|
99
|
+
host rdb_host
|
100
|
+
port 3306
|
101
|
+
database rdb_database
|
102
|
+
adapter mysql2_or_postgresql_or_etc
|
103
|
+
username myusername
|
104
|
+
password mypassword
|
105
|
+
socket path_to_socket
|
106
|
+
remove_tag_prefix my.rdb # optional, dual of tag_prefix in in_sql
|
107
|
+
|
108
|
+
<table>
|
109
|
+
table table1
|
110
|
+
# This is the default table because it has no "pattern" field
|
111
|
+
# The logic is such that if all non-default <table> blocks
|
112
|
+
# do not match, the default one is chosen.
|
113
|
+
# The default table is required.
|
114
|
+
</table>
|
115
|
+
|
116
|
+
<table>
|
117
|
+
table table2
|
118
|
+
pattern hello.* # You can pass the same pattern you use in match statements.
|
119
|
+
# This is the non-default table. It is chosen if the tag matches the pattern
|
120
|
+
# AFTER remove_tag_prefix is applied to the incoming event. For example, if
|
121
|
+
# the message comes in with the tag my.rdb.hello.world, "remove_tag_prefix my.rdb"
|
122
|
+
# makes it "hello.world", which gets matched here because of "pattern hello.*".
|
123
|
+
</table>
|
124
|
+
|
125
|
+
<table>
|
126
|
+
table table3
|
127
|
+
pattern hello.world
|
128
|
+
# This is the second non-default table. You can have as many non-default tables
|
129
|
+
# as you wish. One caveat: non-default tables are matched top-to-bottom and
|
130
|
+
# the events go into the first table it matches to. Hence, this particular table
|
131
|
+
# never gets any data, since the above "hello.*" subsumes "hello.world".
|
132
|
+
</table>
|
133
|
+
</match>
|
134
|
+
|
135
|
+
* **host** RDBMS host
|
136
|
+
* **port** RDBMS port
|
137
|
+
* **database** RDBMS database name
|
138
|
+
* **adapter** RDBMS driver name. You should install corresponding gem before start (mysql2 gem for mysql2 adapter, pg gem for postgresql adapter, etc.)
|
139
|
+
* **user** RDBMS login user name
|
140
|
+
* **password** RDBMS login password
|
141
|
+
* **socket** RDBMS socket path
|
142
|
+
* **remove_tag_prefix** remove the given prefix from the events. See "tag_prefix" in "Input: Configuration". (optional)
|
143
|
+
|
144
|
+
\<table\> sections:
|
145
|
+
|
146
|
+
* **table** RDBM table name
|
147
|
+
* **column_mapping**: Record to table schema mapping. The format is consists of `from:to` or `key` values are separated by `,`. For example, if set 'item_id:id,item_text:data,updated_at' to **column_mapping**, `item_id` field of record is stored into `id` column and `updated_at` field of record is stored into `updated_at` column.
|
148
|
+
* **pattern**: the pattern to which the incoming event's tag (after it goes through `remove_tag_prefix`, if given). The patterns should follow the same syntax as [that of <match>](http://docs.fluentd.org/articles/config-file#match-pattern-how-you-control-the-event-flow-inside-fluentd). **Exactly one <table> element must NOT have this parameter so that it becomes the default table to store data**.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.4.0
|
data/fluent-plugin-sql.gemspec
CHANGED
@@ -17,10 +17,8 @@ Gem::Specification.new do |gem|
|
|
17
17
|
gem.require_paths = ['lib']
|
18
18
|
gem.license = "Apache 2.0"
|
19
19
|
|
20
|
-
gem.add_dependency "fluentd", "
|
21
|
-
gem.add_dependency 'activerecord', "~> 4.
|
22
|
-
gem.add_dependency 'activerecord-import', "~> 0.
|
23
|
-
gem.add_dependency 'mysql2', ['~> 0.3.12']
|
24
|
-
gem.add_dependency 'pg', ['~> 0.16.0']
|
20
|
+
gem.add_dependency "fluentd", [">= 0.10.0", "< 2"]
|
21
|
+
gem.add_dependency 'activerecord', "~> 4.2"
|
22
|
+
gem.add_dependency 'activerecord-import', "~> 0.7"
|
25
23
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
26
24
|
end
|
data/lib/fluent/plugin/in_sql.rb
CHANGED
@@ -150,7 +150,7 @@ module Fluent
|
|
150
150
|
SKIP_TABLE_REGEXP = /\Aschema_migrations\Z/i
|
151
151
|
|
152
152
|
def start
|
153
|
-
@state_store = StateStore.new(@state_file)
|
153
|
+
@state_store = @state_file.nil? ? MemoryStateStore.new : StateStore.new(@state_file)
|
154
154
|
|
155
155
|
config = {
|
156
156
|
:adapter => @adapter,
|
@@ -235,6 +235,8 @@ module Fluent
|
|
235
235
|
|
236
236
|
class StateStore
|
237
237
|
def initialize(path)
|
238
|
+
require 'yaml'
|
239
|
+
|
238
240
|
@path = path
|
239
241
|
if File.exists?(@path)
|
240
242
|
@data = YAML.load_file(@path)
|
@@ -259,6 +261,19 @@ module Fluent
|
|
259
261
|
}
|
260
262
|
end
|
261
263
|
end
|
264
|
+
|
265
|
+
class MemoryStateStore
|
266
|
+
def initialize
|
267
|
+
@data = {}
|
268
|
+
end
|
269
|
+
|
270
|
+
def last_records
|
271
|
+
@data['last_records'] ||= {}
|
272
|
+
end
|
273
|
+
|
274
|
+
def update!
|
275
|
+
end
|
276
|
+
end
|
262
277
|
end
|
263
278
|
|
264
279
|
end
|
metadata
CHANGED
@@ -1,85 +1,63 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-sql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-02-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: 0.10.0
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '2'
|
20
23
|
type: :runtime
|
21
24
|
prerelease: false
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
|
-
- - "
|
27
|
+
- - ">="
|
25
28
|
- !ruby/object:Gem::Version
|
26
29
|
version: 0.10.0
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '2'
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
34
|
name: activerecord
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
30
36
|
requirements:
|
31
37
|
- - "~>"
|
32
38
|
- !ruby/object:Gem::Version
|
33
|
-
version: 4.
|
39
|
+
version: '4.2'
|
34
40
|
type: :runtime
|
35
41
|
prerelease: false
|
36
42
|
version_requirements: !ruby/object:Gem::Requirement
|
37
43
|
requirements:
|
38
44
|
- - "~>"
|
39
45
|
- !ruby/object:Gem::Version
|
40
|
-
version: 4.
|
46
|
+
version: '4.2'
|
41
47
|
- !ruby/object:Gem::Dependency
|
42
48
|
name: activerecord-import
|
43
49
|
requirement: !ruby/object:Gem::Requirement
|
44
50
|
requirements:
|
45
51
|
- - "~>"
|
46
52
|
- !ruby/object:Gem::Version
|
47
|
-
version: 0.
|
48
|
-
type: :runtime
|
49
|
-
prerelease: false
|
50
|
-
version_requirements: !ruby/object:Gem::Requirement
|
51
|
-
requirements:
|
52
|
-
- - "~>"
|
53
|
-
- !ruby/object:Gem::Version
|
54
|
-
version: 0.4.1
|
55
|
-
- !ruby/object:Gem::Dependency
|
56
|
-
name: mysql2
|
57
|
-
requirement: !ruby/object:Gem::Requirement
|
58
|
-
requirements:
|
59
|
-
- - "~>"
|
60
|
-
- !ruby/object:Gem::Version
|
61
|
-
version: 0.3.12
|
62
|
-
type: :runtime
|
63
|
-
prerelease: false
|
64
|
-
version_requirements: !ruby/object:Gem::Requirement
|
65
|
-
requirements:
|
66
|
-
- - "~>"
|
67
|
-
- !ruby/object:Gem::Version
|
68
|
-
version: 0.3.12
|
69
|
-
- !ruby/object:Gem::Dependency
|
70
|
-
name: pg
|
71
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - "~>"
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version: 0.16.0
|
53
|
+
version: '0.7'
|
76
54
|
type: :runtime
|
77
55
|
prerelease: false
|
78
56
|
version_requirements: !ruby/object:Gem::Requirement
|
79
57
|
requirements:
|
80
58
|
- - "~>"
|
81
59
|
- !ruby/object:Gem::Version
|
82
|
-
version: 0.
|
60
|
+
version: '0.7'
|
83
61
|
- !ruby/object:Gem::Dependency
|
84
62
|
name: rake
|
85
63
|
requirement: !ruby/object:Gem::Requirement
|