logstash-input-genjdbc 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +2 -0
- data/LICENSE +13 -0
- data/README.md +197 -0
- data/Rakefile +1 -0
- data/lib/logstash/inputs/genjdbc.rb +232 -0
- data/logstash-input-genjdbc.gemspec +25 -0
- data/spec/inputs/genjdbc_spec.rb +1 -0
- metadata +101 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA1:
|
|
3
|
+
metadata.gz: e219d2a3354070fc2d7efc12bf8ae55b1d143899
|
|
4
|
+
data.tar.gz: 781b742fc40750ab302a795f5c33686cd83a675e
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: deec50c999c1dfeb7e32d38b8c4ee7e9418b6c41c625a2d6a8cda4321201ade5426c27e72672f8d6ae6fef08bcb9de4b02a4e445fe2e5fddd42a8a61e2e325d5
|
|
7
|
+
data.tar.gz: 9b8fcc90609cb79bdafcf4cb40f1e787b3759299180a277351f0b1bb65a122a16c018f81c23c03a107f69230687fb153b673942102cd1bd4b680f3917473cf05
|
data/Gemfile
ADDED
data/LICENSE
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
Copyright (c) 2012–2015 IBM <http://www.ibm.com>
|
|
2
|
+
|
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
you may not use this file except in compliance with the License.
|
|
5
|
+
You may obtain a copy of the License at
|
|
6
|
+
|
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
|
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
See the License for the specific language governing permissions and
|
|
13
|
+
limitations under the License.
|
data/README.md
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
<html>
|
|
2
|
+
<head>
|
|
3
|
+
<meta charset="UTF-8">
|
|
4
|
+
<title>Input genjdbc</title>
|
|
5
|
+
<link rel="stylesheet" href="http://logstash.net/style.css">
|
|
6
|
+
</head>
|
|
7
|
+
<body>
|
|
8
|
+
<div class="container">
|
|
9
|
+
|
|
10
|
+
<div id="content_right">
|
|
11
|
+
<!--main content goes here, yo!-->
|
|
12
|
+
|
|
13
|
+
<h2>genjdbc</h2>
|
|
14
|
+
<h3>Milestone: <a href="http://logstash.net/docs/1.4.2/plugin-milestones">1</a></h3>
|
|
15
|
+
<h3> Synopsis </h3>
|
|
16
|
+
Connects to datasources using JDBC driver, executes a select query and for each record returned, emits an event.
|
|
17
|
+
<p>
|
|
18
|
+
<pre><code>input {
|
|
19
|
+
genjdbc {
|
|
20
|
+
<a href="#jdbcHost">jdbcHost</a> => ... # string (required)
|
|
21
|
+
<a href="#jdbcPort">jdbcPort</a> => ... # string (required)
|
|
22
|
+
<a href="#jdbcDBName">jdbcDBName</a> => ... # string (required)
|
|
23
|
+
<a href="#jdbcTargetDB">jdbcTargetDB</a> => ... # string (required)
|
|
24
|
+
<a href="#jdbcDriverPath">jdbcDriverPath</a> => ... # string (required)
|
|
25
|
+
<a href="#jdbcUser">jdbcUser</a> => ... # string (required)
|
|
26
|
+
<a href="#jdbcPassword">jdbcPassword</a> => ... # string (required)
|
|
27
|
+
<a href="#jdbcSQLQuery">jdbcSQLQuery</a> => ... # string (required)
|
|
28
|
+
<a href="#jdbcURL">jdbcURL</a> => ... # string (optional)
|
|
29
|
+
<a href="#jdbcTimeField">jdbcTimeField</a> => ... # string (optional)
|
|
30
|
+
<a href="#jdbcPollInterval">jdbcPollInterval</a> => ... # number (optional), default: 60
|
|
31
|
+
<a href="#jdbcCollectionStartTime">jdbcCollectionStartTime</a> => ... # string (optional)
|
|
32
|
+
<a href="#jdbcPStoreFile">jdbcPStoreFile</a> => ... # string (optional)
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
</code></pre>
|
|
36
|
+
<h3> Details </h3>
|
|
37
|
+
<p>
|
|
38
|
+
This plugin takes basic JDBC configuration information, an SQL query and some timing control. It then runs that queryr (with some additional timing information applied) in a loop, emitting events corresponding to each row in the returned table. For example
|
|
39
|
+
</p>
|
|
40
|
+
<pre><code>
|
|
41
|
+
genjdbc {
|
|
42
|
+
jdbcHost => 'X.X.X.X'
|
|
43
|
+
jdbcPort => '50000'
|
|
44
|
+
jdbcTargetDB => 'db2'
|
|
45
|
+
jdbcDBName =>'MYDB'
|
|
46
|
+
jdbcUser => 'myuser'
|
|
47
|
+
jdbcPassword => 'mypasswd'
|
|
48
|
+
jdbcDriverPath => '/path/to/my/driver/db2jcc4.jar'
|
|
49
|
+
jdbcSQLQuery => 'select * from DT_ALARM_HIST where SUPPRESSED_DESC is null'
|
|
50
|
+
jdbcTimeField => 'TSTAMP'
|
|
51
|
+
jdbcPStoreFile => './test.pstore'
|
|
52
|
+
jdbcCollectionStartTime => '2015-01-01 00:00:00.000'
|
|
53
|
+
}
|
|
54
|
+
</code></pre>
|
|
55
|
+
<p>
|
|
56
|
+
In the above example. the query will have <code>where 'TSTAMP' >= currentTime'</code> appended to it. <code>currentTime</code> is an internal variable maintained by the plugin which will be incremented on each loop with the timestamp of the last record returned from the query.
|
|
57
|
+
</p>
|
|
58
|
+
<h4>
|
|
59
|
+
<a name="jdbcHost">jdbcHost</a>
|
|
60
|
+
</h4>
|
|
61
|
+
<ul>
|
|
62
|
+
<li> Value type is <a href="http://logstash.net/docs/1.4.2/configuration#string">String</a> </li>
|
|
63
|
+
<li> There is no default for this setting </li>
|
|
64
|
+
</ul>
|
|
65
|
+
<p>jdbc host name. Used to form jdbc url</p>
|
|
66
|
+
<h4>
|
|
67
|
+
<a name="jdbcPort">jdbcPort</a>
|
|
68
|
+
</h4>
|
|
69
|
+
<ul>
|
|
70
|
+
<li> Value type is <a href="http://logstash.net/docs/1.4.2/configuration#string">String</a> </li>
|
|
71
|
+
<li> There is no default for this setting </li>
|
|
72
|
+
</ul>
|
|
73
|
+
<p>
|
|
74
|
+
jdbc port number. Used to form jdbc url
|
|
75
|
+
</p>
|
|
76
|
+
<h4><a name="jdbcDBName">jdbcDBName</a></h4>
|
|
77
|
+
<ul>
|
|
78
|
+
<li> Value type is <a href="http://logstash.net/docs/1.4.2/configuration#string">String</a> </li>
|
|
79
|
+
<li> The default value is "" </li>
|
|
80
|
+
</ul>
|
|
81
|
+
<p>
|
|
82
|
+
Name of database on server / schema name. Used to form jdbc url
|
|
83
|
+
</p>
|
|
84
|
+
<h4><a name="jdbcTargetDB">jdbcTargetDB</a></h4>
|
|
85
|
+
<ul>
|
|
86
|
+
<li> Value type is <a href="http://logstash.net/docs/1.4.2/configuration#string">String</a> </li>
|
|
87
|
+
<li> There is no default value for this item</li>
|
|
88
|
+
</ul>
|
|
89
|
+
<p>
|
|
90
|
+
Target DB name(vendor). used to select appropriate JDBC format.
|
|
91
|
+
Supported types
|
|
92
|
+
<li>postgresql </li>
|
|
93
|
+
<li>oracle</li>
|
|
94
|
+
<li>db2</li>
|
|
95
|
+
<li>mysql</li>
|
|
96
|
+
<li>derby</li>
|
|
97
|
+
<li>mssql</li>
|
|
98
|
+
</p>
|
|
99
|
+
<h4><a name="jdbcDriverPath">jdbcDriverPath</a></h4>
|
|
100
|
+
<ul>
|
|
101
|
+
<li> Value type is <a href="http://logstash.net/docs/1.4.2/configuration#string">String</a> </li>
|
|
102
|
+
<li> There is no default value for this item </li>
|
|
103
|
+
</ul>
|
|
104
|
+
<p>
|
|
105
|
+
path to jdbc driver
|
|
106
|
+
</p>
|
|
107
|
+
<h4><a name="jdbcUser">jdbcUser</a></h4>
|
|
108
|
+
<ul>
|
|
109
|
+
<li> Value type is <a href="http://logstash.net/docs/1.4.2/configuration#string">String</a> </li>
|
|
110
|
+
<li> There is no default value for this item </li>
|
|
111
|
+
</ul>
|
|
112
|
+
<p>
|
|
113
|
+
jdbc user name. Used to form jdbc url
|
|
114
|
+
</p>
|
|
115
|
+
<h4><a name="jdbcPassword">jdbcPassword</a></h4>
|
|
116
|
+
<ul>
|
|
117
|
+
<li> Value type is <a href="http://logstash.net/docs/1.4.2/configuration#string">String</a> </li>
|
|
118
|
+
<li> There is no default value for this item </li>
|
|
119
|
+
</ul>
|
|
120
|
+
<p>
|
|
121
|
+
jdbc password. Used to form jdbc url
|
|
122
|
+
</p>
|
|
123
|
+
<h4><a name="jdbcSQLQuery">jdbcSQLQuery</a></h4>
|
|
124
|
+
<ul>
|
|
125
|
+
<li> Value type is <a href="http://logstash.net/docs/1.4.2/configuration#string">String</a> </li>
|
|
126
|
+
<li> There is no default value for this item </li>
|
|
127
|
+
</ul>
|
|
128
|
+
<p>
|
|
129
|
+
SQL query to execute or the path of a text file which contains the query to execute.
|
|
130
|
+
</p>
|
|
131
|
+
<p><code>jdbcSQLQuery => "select * from myTable"</code><p>
|
|
132
|
+
In the latter case, prefix the filename with 'file:' e.g.
|
|
133
|
+
<p><code>jdbcSQLQuery => "file:/path/to/my/query.txt"</code></p>
|
|
134
|
+
|
|
135
|
+
<h4><a name="jdbcURL">jdbcURL</a></h4>
|
|
136
|
+
<ul>
|
|
137
|
+
<li> Value type is <a href="http://logstash.net/docs/1.4.2/configuration#string">String</a> </li>
|
|
138
|
+
<li> There is no default value for this item </li>
|
|
139
|
+
</ul>
|
|
140
|
+
<p>
|
|
141
|
+
jdbc URL - explicitly set the jdbc URL string. Overrides all other URL component settings (e.g jdbcHost, jdbcPort)
|
|
142
|
+
</p>
|
|
143
|
+
<h4><a name="jdbcTimeField">jdbcTimeField</a></h4>
|
|
144
|
+
<ul>
|
|
145
|
+
<li> Value type is <a href="http://logstash.net/docs/1.4.2/configuration#string">String</a> </li>
|
|
146
|
+
<li> There is no default value for this item </li>
|
|
147
|
+
</ul>
|
|
148
|
+
<p>
|
|
149
|
+
Name of the table column which contains timestamp information. This string will be used in the automatically generated SQL query as part of the <code>where</code> clause.
|
|
150
|
+
</p>
|
|
151
|
+
<h4><a name="jdbcPollInterval">jdbcPollInterval</a></h4>
|
|
152
|
+
<ul>
|
|
153
|
+
<li> Value type is <a href="http://logstash.net/docs/1.4.2/configuration#Number">Number</a> </li>
|
|
154
|
+
<li> 60 </li>
|
|
155
|
+
</ul>
|
|
156
|
+
<p>
|
|
157
|
+
The number of seconds to wait between query loops. A query loop executes the query, waits for a response, processes the response ( by emitting events). Once these steps are completed, the plugin will wait the specified amount of time before starting again and invoking the query.
|
|
158
|
+
</p>
|
|
159
|
+
<h4><a name="jdbcCollectionStartTime">jdbcCollectionStartTime</a></h4>
|
|
160
|
+
<ul>
|
|
161
|
+
<li> Value type is <a href="http://logstash.net/docs/1.4.2/configuration#string">String</a> </li>
|
|
162
|
+
<li> There is no default value for this item </li>
|
|
163
|
+
</ul>
|
|
164
|
+
<p>
|
|
165
|
+
This is a timestamp expressed as a string e.g. <code>'2015-01-01 00:00:00.000'</code>. It will be used in direct comparison operations against the specified <code>jdbcTimeField</code> so the precise format depends on the database and the nature of the time column referenced.
|
|
166
|
+
</p>
|
|
167
|
+
<h4><a name="jdbcPStoreFile">jdbcPStoreFile</a></h4>
|
|
168
|
+
<ul>
|
|
169
|
+
<li> Value type is <a href="http://logstash.net/docs/1.4.2/configuration#string">String</a> </li>
|
|
170
|
+
<li> There is no default value for this item </li>
|
|
171
|
+
</ul>
|
|
172
|
+
<p>
|
|
173
|
+
The pathname of a file which is used to maintain state for this operator. It is useful to be able to specify this as you may have multiple <code>genjdbc</code> plugins.
|
|
174
|
+
</p>
|
|
175
|
+
<hr>
|
|
176
|
+
</div>
|
|
177
|
+
<div class="clear">
|
|
178
|
+
</div>
|
|
179
|
+
</div>
|
|
180
|
+
</div>
|
|
181
|
+
<!--closes main container div-->
|
|
182
|
+
<div class="clear">
|
|
183
|
+
</div>
|
|
184
|
+
<div class="footer">
|
|
185
|
+
<p>
|
|
186
|
+
Hello! I'm your friendly footer. If you're actually reading this, I'm impressed.
|
|
187
|
+
</p>
|
|
188
|
+
</div>
|
|
189
|
+
<noscript>
|
|
190
|
+
<div style="display:inline;">
|
|
191
|
+
<img height="1" width="1" style="border-style:none;" alt="" src="//googleads.g.doubleclick.net/pagead/viewthroughconversion/985891458/?value=0&guid=ON&script=0"/>
|
|
192
|
+
</div>
|
|
193
|
+
</noscript>
|
|
194
|
+
<script src="/js/patch.js?1.4.2"></script>
|
|
195
|
+
</body>
|
|
196
|
+
</html>
|
|
197
|
+
|
data/Rakefile
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
require "logstash/devutils/rake"
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
# Date 06 July 2015 09:00:00 GMT
|
|
2
|
+
# Logstash Generic JDBC Input PlugIn
|
|
3
|
+
# Authors: Stuart Tuck & Rob McKeown
|
|
4
|
+
#
|
|
5
|
+
# This is a community contributed content pack and no explicit support, guarantee or warranties
|
|
6
|
+
# are provided by IBM nor the contributor. Feel free to engage the community on the ITOAdev
|
|
7
|
+
# forum if you need help!
|
|
8
|
+
#
|
|
9
|
+
# encoding: utf-8
|
|
10
|
+
require "logstash/inputs/base"
|
|
11
|
+
require "logstash/namespace"
|
|
12
|
+
require "java"
|
|
13
|
+
require "rubygems"
|
|
14
|
+
require "pstore"
|
|
15
|
+
|
|
16
|
+
# This Input Plug Is Intended to Read Events from a JDBC url
|
|
17
|
+
#
|
|
18
|
+
# Like stdin and file inputs, each row returned from the remote system
|
|
19
|
+
# is assumed to result in one line of output.
|
|
20
|
+
#
|
|
21
|
+
class LogStash::Inputs::Genjdbc < LogStash::Inputs::Base
|
|
22
|
+
config_name "genjdbc"
|
|
23
|
+
|
|
24
|
+
default :codec, "plain"
|
|
25
|
+
|
|
26
|
+
# Configuration Parameters of the remote instance
|
|
27
|
+
config :jdbcHost, :validate => :string, :required => true
|
|
28
|
+
config :jdbcPort, :validate => :string, :required => true
|
|
29
|
+
config :jdbcDBName, :validate => :string, :required => true
|
|
30
|
+
config :jdbcTargetDB, :validate => :string, :required => true
|
|
31
|
+
config :jdbcDriverPath, :validate => :string, :required => true
|
|
32
|
+
config :jdbcUser, :validate => :string, :required => true
|
|
33
|
+
config :jdbcPassword, :validate => :string, :required => true
|
|
34
|
+
config :jdbcSQLQuery, :validate => :string, :required => true
|
|
35
|
+
config :jdbcURL, :validate => :string, :required => false
|
|
36
|
+
config :jdbcTimeField, :validate => :string, :required => false
|
|
37
|
+
config :jdbcPollInterval, :validate => :number, :required => false, :default => 60
|
|
38
|
+
config :jdbcCollectionStartTime, :validate => :string, :required => false
|
|
39
|
+
config :jdbcPStoreFile, :validate => :string, :required => false, :default => "genjdbc.pstore"
|
|
40
|
+
|
|
41
|
+
# The 'read' timeout in seconds. If a particular connection is idle for
|
|
42
|
+
# more than this timeout period, we will assume it is dead and close it.
|
|
43
|
+
# ToDo: Implement more awareness of connection state.
|
|
44
|
+
# If you never want to timeout, use -1.
|
|
45
|
+
#config :data_timeout, :validate => :number, :default => -1
|
|
46
|
+
|
|
47
|
+
def initialize(*args)
|
|
48
|
+
super(*args)
|
|
49
|
+
end # def initialize
|
|
50
|
+
|
|
51
|
+
public
|
|
52
|
+
def register
|
|
53
|
+
@logger.info("Starting JDBC input", :address => "#{@jdbcHost}")
|
|
54
|
+
end # def register
|
|
55
|
+
|
|
56
|
+
public
|
|
57
|
+
def run(queue)
|
|
58
|
+
require 'java'
|
|
59
|
+
require 'date'
|
|
60
|
+
require @jdbcDriverPath
|
|
61
|
+
|
|
62
|
+
# Load the Driver Manager Classes required to create/operate sql connection
|
|
63
|
+
java_import java.sql.DriverManager
|
|
64
|
+
java_import java.sql.Connection
|
|
65
|
+
import java.lang.System
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
# Database Selection
|
|
69
|
+
# ----------------------------------------------------------------------------------------------------------
|
|
70
|
+
if @jdbcTargetDB == "postgresql"
|
|
71
|
+
driver = org.postgresql.Driver.new
|
|
72
|
+
driverurl = "jdbc:postgresql://"+@jdbcHost+":"+@jdbcPort+"/"+@jdbcDBName
|
|
73
|
+
# Spec. jdbc:postgresql://<server>:<5432>/<database_name>
|
|
74
|
+
end
|
|
75
|
+
if @jdbcTargetDB == "oracle"
|
|
76
|
+
driver = Java::oracle.jdbc.driver.OracleDriver.new
|
|
77
|
+
driverurl = 'jdbc:oracle:thin:@'+@jdbcHost+':'+@jdbcPort+':'+@jdbcDBName
|
|
78
|
+
# Spec. jdbc:oracle:thin:@<server>[:<1521>]:<database_name>
|
|
79
|
+
end
|
|
80
|
+
if @jdbcTargetDB == "db2"
|
|
81
|
+
driver = Java::com.ibm.db2.jcc.DB2Driver.new
|
|
82
|
+
driverurl = "jdbc:db2://"+@jdbcHost+":"+@jdbcPort+"/"+@jdbcDBName
|
|
83
|
+
# Spec. jdbc:db2://<server>:<6789>/<db-name>
|
|
84
|
+
end
|
|
85
|
+
if @jdbcTargetDB == "mysql"
|
|
86
|
+
driver = com.mysql.jdbc.Driver.new
|
|
87
|
+
driverurl = "jdbc:mysql://"+@jdbcHost+":"+@jdbcPort+"/"+@jdbcDBName+"?profileSQL=true"
|
|
88
|
+
# Spec. jdbc:mysql://<hostname>[,<failoverhost>][<:3306>]/<dbname>[?<param1>=<value1>][&<param2>=<value2>]
|
|
89
|
+
end
|
|
90
|
+
if @jdbcTargetDB == "derby"
|
|
91
|
+
driver = org.apache.derby.jdbc.ClientDriver.new
|
|
92
|
+
driverurl = "jdbc:mysql://"+@jdbcHost+":"+@jdbcPort+"/"+@jdbcDBName
|
|
93
|
+
# Spec. jdbc:derby://<server>[:<port>]/<databaseName>[;<URL attribute>=<value>]
|
|
94
|
+
end
|
|
95
|
+
if @jdbcTargetDB == "mssql"
|
|
96
|
+
driver = com.microsoft.sqlserver.jdbc.SQLServerDriver.new
|
|
97
|
+
driverurl = "jdbc:sqlserver://"+@jdbcHost+":"+@jdbcPort+";databaseName="+@jdbcDBName
|
|
98
|
+
# Spec. jdbc:sqlserver://<server_name>:1433;databaseName=<db_name>
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
# Check the jdbcURL setting to see if there is an override, and use constructed URL if not.
|
|
103
|
+
if jdbcURL.nil?
|
|
104
|
+
@jdbcURL = driverurl
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Set the connection properties
|
|
108
|
+
props = java.util.Properties.new
|
|
109
|
+
props.setProperty("user",@jdbcUser)
|
|
110
|
+
props.setProperty("password",@jdbcPassword)
|
|
111
|
+
|
|
112
|
+
# Create a new connection to the jdbc URL, using the connection properties
|
|
113
|
+
@logger.info("Creating Connection to JDBC URL", :address => "#{@jdbcURL}")
|
|
114
|
+
conn = driver.connect(@jdbcURL,props)
|
|
115
|
+
|
|
116
|
+
# Prepare the store where we'll track most recent timestamp
|
|
117
|
+
store = PStore.new(@jdbcPStoreFile)
|
|
118
|
+
|
|
119
|
+
# Set a start time
|
|
120
|
+
lastEvent = store.transaction { store.fetch(:lastEvent,DateTime.now) }
|
|
121
|
+
# If set, make an override from the config..
|
|
122
|
+
if !@jdbcCollectionStartTime.nil?
|
|
123
|
+
lastEvent = DateTime.parse @jdbcCollectionStartTime
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Read query from file if so configured
|
|
127
|
+
if @jdbcSQLQuery.start_with? "file:"
|
|
128
|
+
queryFilename = @jdbcSQLQuery
|
|
129
|
+
queryFilename.slice! "file:"
|
|
130
|
+
queryFile = File.open(queryFilename,"rb")
|
|
131
|
+
originalQuery = queryFile.read
|
|
132
|
+
queryFile.close
|
|
133
|
+
else
|
|
134
|
+
originalQuery = @jdbcSQLQuery
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Main Loop
|
|
138
|
+
while true
|
|
139
|
+
|
|
140
|
+
# Debug : puts "lastEvent : "+lastEvent.to_s
|
|
141
|
+
jdbclastEvent = lastEvent.strftime("%Y-%m-%d %H:%M:%S.%L")
|
|
142
|
+
currentTime = jdbclastEvent
|
|
143
|
+
|
|
144
|
+
stmt = conn.create_statement
|
|
145
|
+
|
|
146
|
+
# If query explicity refers to CURRENTTIME, then use that directly
|
|
147
|
+
if originalQuery.include? "%{CURRENTTIME}"
|
|
148
|
+
escapedQuery = originalQuery.gsub("%{CURRENTTIME}",currentTime)
|
|
149
|
+
else
|
|
150
|
+
# if not, we'll implicity assemble query
|
|
151
|
+
# Suggest removing this option completely and making it explicity
|
|
152
|
+
# Escape sql query provided from config file
|
|
153
|
+
begin
|
|
154
|
+
if originalQuery.include? " where " then
|
|
155
|
+
escapedQuery = originalQuery + " and "+@jdbcTimeField+" > '" + jdbclastEvent + "'" + " order by " +@jdbcTimeField
|
|
156
|
+
else
|
|
157
|
+
escapedQuery = originalQuery + " where "+@jdbcTimeField+" > '" + jdbclastEvent + "'" + " order by " +@jdbcTimeField
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
escapedQuery = escapedQuery.gsub(/\\\"/,"\"")
|
|
164
|
+
|
|
165
|
+
@logger.info("Running Query : ", :query => "#{escapedQuery}")
|
|
166
|
+
|
|
167
|
+
# Execute Query Statement
|
|
168
|
+
rs = stmt.executeQuery(escapedQuery)
|
|
169
|
+
|
|
170
|
+
rsmd = rs.getMetaData();
|
|
171
|
+
columnCount = rsmd.getColumnCount()
|
|
172
|
+
|
|
173
|
+
while (rs.next) do
|
|
174
|
+
event = LogStash::Event.new()
|
|
175
|
+
event["jdbchost"] = @jdbcHost
|
|
176
|
+
|
|
177
|
+
for i in 1..columnCount
|
|
178
|
+
columnName = rsmd.getColumnName(i)
|
|
179
|
+
value = rs.getString(columnName)
|
|
180
|
+
|
|
181
|
+
# Debug (find out columntype for each object)
|
|
182
|
+
#columnType = rsmd.getColumnTypeName(i)
|
|
183
|
+
#puts "Column Type is : "+(columnType)
|
|
184
|
+
|
|
185
|
+
if value.nil?
|
|
186
|
+
#substitute "" for <nil> returned by DB
|
|
187
|
+
value = ""
|
|
188
|
+
end
|
|
189
|
+
event[columnName] = value
|
|
190
|
+
|
|
191
|
+
# Check the column to set the latest time field
|
|
192
|
+
if columnName == @jdbcTimeField
|
|
193
|
+
# debug: puts "Time Column is : "+columnName
|
|
194
|
+
eventTime = DateTime.parse value
|
|
195
|
+
# debug: puts "Date Parsed is : "+eventTime.to_s
|
|
196
|
+
if eventTime > lastEvent
|
|
197
|
+
lastEvent = eventTime
|
|
198
|
+
store.transaction do store[:lastEvent] = lastEvent end
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
end # for
|
|
203
|
+
|
|
204
|
+
# Todo, check how many rows collected .. rowcount++
|
|
205
|
+
decorate(event)
|
|
206
|
+
queue << event
|
|
207
|
+
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
rs.close
|
|
211
|
+
stmt.close
|
|
212
|
+
|
|
213
|
+
# Now need to sleep for interval
|
|
214
|
+
@logger.info("Sleeping for ", :interval_seconds => "#{@jdbcPollInterval}")
|
|
215
|
+
sleep(@jdbcPollInterval)
|
|
216
|
+
# zzzzzZZZZ
|
|
217
|
+
|
|
218
|
+
end # While true (end loop)
|
|
219
|
+
|
|
220
|
+
rescue LogStash::ShutdownSignal
|
|
221
|
+
# nothing to do
|
|
222
|
+
ensure
|
|
223
|
+
# Close the JDBC connection
|
|
224
|
+
@logger.info("Closing Connection to JDBC URL", :address => "#{@jdbcURL}")
|
|
225
|
+
conn.close() rescue nil
|
|
226
|
+
end # def run
|
|
227
|
+
|
|
228
|
+
def teardown
|
|
229
|
+
@interrupted = true
|
|
230
|
+
end # def teardown
|
|
231
|
+
|
|
232
|
+
end # class LogStash::Inputs::Genjdbc
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
Gem::Specification.new do |s|
|
|
2
|
+
s.name = 'logstash-input-genjdbc'
|
|
3
|
+
s.version = '0.1.1'
|
|
4
|
+
s.licenses = ['Apache License (2.0)']
|
|
5
|
+
s.summary = "This example input streams a string at a definable interval."
|
|
6
|
+
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
|
7
|
+
s.authors = ["Rob Mckeown, Stuart Tuck"]
|
|
8
|
+
s.email = 'rmckeown@us.ibm.com'
|
|
9
|
+
s.homepage = "https://github.com/IBM-ITOAdev/logstash-input-genjdbc"
|
|
10
|
+
s.require_paths = ["lib"]
|
|
11
|
+
|
|
12
|
+
# Files
|
|
13
|
+
s.files = `git ls-files`.split($\)
|
|
14
|
+
# Tests
|
|
15
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
|
16
|
+
|
|
17
|
+
# Special flag to let us know this is actually a logstash plugin
|
|
18
|
+
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
|
|
19
|
+
|
|
20
|
+
# Gem dependencies
|
|
21
|
+
s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
|
|
22
|
+
s.add_runtime_dependency 'logstash-codec-plain'
|
|
23
|
+
|
|
24
|
+
s.add_development_dependency 'logstash-devutils'
|
|
25
|
+
end
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
require "logstash/devutils/rspec/spec_helper"
|
metadata
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: logstash-input-genjdbc
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.1
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Rob Mckeown, Stuart Tuck
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2015-07-02 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: logstash-core
|
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - '>='
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: 1.4.0
|
|
20
|
+
- - <
|
|
21
|
+
- !ruby/object:Gem::Version
|
|
22
|
+
version: 2.0.0
|
|
23
|
+
requirement: !ruby/object:Gem::Requirement
|
|
24
|
+
requirements:
|
|
25
|
+
- - '>='
|
|
26
|
+
- !ruby/object:Gem::Version
|
|
27
|
+
version: 1.4.0
|
|
28
|
+
- - <
|
|
29
|
+
- !ruby/object:Gem::Version
|
|
30
|
+
version: 2.0.0
|
|
31
|
+
prerelease: false
|
|
32
|
+
type: :runtime
|
|
33
|
+
- !ruby/object:Gem::Dependency
|
|
34
|
+
name: logstash-codec-plain
|
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - '>='
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: '0'
|
|
40
|
+
requirement: !ruby/object:Gem::Requirement
|
|
41
|
+
requirements:
|
|
42
|
+
- - '>='
|
|
43
|
+
- !ruby/object:Gem::Version
|
|
44
|
+
version: '0'
|
|
45
|
+
prerelease: false
|
|
46
|
+
type: :runtime
|
|
47
|
+
- !ruby/object:Gem::Dependency
|
|
48
|
+
name: logstash-devutils
|
|
49
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - '>='
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: '0'
|
|
54
|
+
requirement: !ruby/object:Gem::Requirement
|
|
55
|
+
requirements:
|
|
56
|
+
- - '>='
|
|
57
|
+
- !ruby/object:Gem::Version
|
|
58
|
+
version: '0'
|
|
59
|
+
prerelease: false
|
|
60
|
+
type: :development
|
|
61
|
+
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
|
62
|
+
email: rmckeown@us.ibm.com
|
|
63
|
+
executables: []
|
|
64
|
+
extensions: []
|
|
65
|
+
extra_rdoc_files: []
|
|
66
|
+
files:
|
|
67
|
+
- Gemfile
|
|
68
|
+
- LICENSE
|
|
69
|
+
- README.md
|
|
70
|
+
- Rakefile
|
|
71
|
+
- lib/logstash/inputs/genjdbc.rb
|
|
72
|
+
- logstash-input-genjdbc.gemspec
|
|
73
|
+
- spec/inputs/genjdbc_spec.rb
|
|
74
|
+
homepage: https://github.com/IBM-ITOAdev/logstash-input-genjdbc
|
|
75
|
+
licenses:
|
|
76
|
+
- Apache License (2.0)
|
|
77
|
+
metadata:
|
|
78
|
+
logstash_plugin: 'true'
|
|
79
|
+
logstash_group: input
|
|
80
|
+
post_install_message:
|
|
81
|
+
rdoc_options: []
|
|
82
|
+
require_paths:
|
|
83
|
+
- lib
|
|
84
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
85
|
+
requirements:
|
|
86
|
+
- - '>='
|
|
87
|
+
- !ruby/object:Gem::Version
|
|
88
|
+
version: '0'
|
|
89
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
90
|
+
requirements:
|
|
91
|
+
- - '>='
|
|
92
|
+
- !ruby/object:Gem::Version
|
|
93
|
+
version: '0'
|
|
94
|
+
requirements: []
|
|
95
|
+
rubyforge_project:
|
|
96
|
+
rubygems_version: 2.1.9
|
|
97
|
+
signing_key:
|
|
98
|
+
specification_version: 4
|
|
99
|
+
summary: This example input streams a string at a definable interval.
|
|
100
|
+
test_files:
|
|
101
|
+
- spec/inputs/genjdbc_spec.rb
|