logstash-input-gemfire 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 1d5c414717063c02177082f724f1e9f5d5240d68
4
+ data.tar.gz: 0d0a7d467725a01f564b95b421383799c7cfc5e6
5
+ SHA512:
6
+ metadata.gz: 1a64db6fb6fc2ef953cb41dd366b4c632b9f5aefce5b2f2245b8c5502d5ea3dc6a5e23af86ed5e7f5c59fb249eea5303fe9e6d23874e9c3890b3d19bcbf208eb
7
+ data.tar.gz: 6668cdbfc47ce444d9671e27162bf8c5c622b518488eb475f9cbf4109225765afcb878d71809331a5afb02ad46dc90914c19f5cbf8d77092e07d84471c603343
data/.gitignore ADDED
@@ -0,0 +1,5 @@
1
+ *.gem
2
+ Gemfile.lock
3
+ Gemfile.bak
4
+ .bundle
5
+ vendor
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
3
+ gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"
4
+
data/LICENSE ADDED
@@ -0,0 +1,13 @@
1
+ Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
data/Rakefile ADDED
@@ -0,0 +1,8 @@
1
+ @files=[]
2
+
3
+ task :default do
4
+ system("rake -T")
5
+ end
6
+
7
+ require "logstash/devutils/rake"
8
+
@@ -0,0 +1,223 @@
1
+ # encoding: utf-8
2
+ require "logstash/inputs/threadable"
3
+ require "logstash/namespace"
4
+
5
+
6
+ # Push events to a GemFire region.
7
+ #
8
+ # GemFire is an object database.
9
+ #
10
+ # To use this plugin you need to add gemfire.jar to your CLASSPATH.
11
+ # Using format=json requires jackson.jar too; use of continuous
12
+ # queries requires antlr.jar.
13
+ #
14
+ # Note: this plugin has only been tested with GemFire 7.0.
15
+ #
16
+ class LogStash::Inputs::Gemfire < LogStash::Inputs::Threadable
17
+
18
+ config_name "gemfire"
19
+ milestone 1
20
+
21
+ default :codec, "plain"
22
+
23
+ # Your client cache name
24
+ config :cache_name, :validate => :string, :default => "logstash"
25
+
26
+ # The path to a GemFire client cache XML file.
27
+ #
28
+ # Example:
29
+ #
30
+ # <client-cache>
31
+ # <pool name="client-pool" subscription-enabled="true" subscription-redundancy="1">
32
+ # <locator host="localhost" port="31331"/>
33
+ # </pool>
34
+ # <region name="Logstash">
35
+ # <region-attributes refid="CACHING_PROXY" pool-name="client-pool" >
36
+ # </region-attributes>
37
+ # </region>
38
+ # </client-cache>
39
+ #
40
+ config :cache_xml_file, :validate => :string, :default => nil
41
+
42
+ # The region name
43
+ config :region_name, :validate => :string, :default => "Logstash"
44
+
45
+ # A regexp to use when registering interest for cache events.
46
+ # Ignored if a :query is specified.
47
+ config :interest_regexp, :validate => :string, :default => ".*"
48
+
49
+ # A query to run as a GemFire "continuous query"; if specified it takes
50
+ # precedence over :interest_regexp which will be ignore.
51
+ #
52
+ # Important: use of continuous queries requires subscriptions to be enabled on the client pool.
53
+ config :query, :validate => :string, :default => nil
54
+
55
+ # How the message is serialized in the cache. Can be one of "json" or "plain"; default is plain
56
+ config :serialization, :validate => :string, :default => nil
57
+
58
+ public
59
+ def register
60
+ import com.gemstone.gemfire.cache.AttributesMutator
61
+ import com.gemstone.gemfire.cache.InterestResultPolicy
62
+ import com.gemstone.gemfire.cache.client.ClientCacheFactory
63
+ import com.gemstone.gemfire.cache.client.ClientRegionShortcut
64
+ import com.gemstone.gemfire.cache.query.CqQuery
65
+ import com.gemstone.gemfire.cache.query.CqAttributes
66
+ import com.gemstone.gemfire.cache.query.CqAttributesFactory
67
+ import com.gemstone.gemfire.cache.query.QueryService
68
+ import com.gemstone.gemfire.cache.query.SelectResults
69
+ import com.gemstone.gemfire.pdx.JSONFormatter
70
+
71
+ @logger.info("Registering input", :plugin => self)
72
+ end # def register
73
+
74
+ def run(queue)
75
+ return if terminating?
76
+ connect
77
+
78
+ @logstash_queue = queue
79
+
80
+ if @query
81
+ continuous_query(@query)
82
+ else
83
+ register_interest(@interest_regexp)
84
+ end
85
+ end # def run
86
+
87
+ def teardown
88
+ @cache.close if @cache
89
+ @cache = nil
90
+ finished
91
+ end # def teardown
92
+
93
+ protected
94
+ def connect
95
+ begin
96
+ @logger.debug("Connecting to GemFire #{@cache_name}")
97
+
98
+ @cache = ClientCacheFactory.new.
99
+ set("name", @cache_name).
100
+ set("cache-xml-file", @cache_xml_file).create
101
+ @logger.debug("Created cache #{@cache.inspect}")
102
+
103
+ rescue => e
104
+ if terminating?
105
+ return
106
+ else
107
+ @logger.error("Gemfire connection error (during connect), will reconnect",
108
+ :exception => e, :backtrace => e.backtrace)
109
+ sleep(1)
110
+ retry
111
+ end
112
+ end
113
+
114
+ @region = @cache.getRegion(@region_name);
115
+ @logger.debug("Created region #{@region.inspect}")
116
+ end # def connect
117
+
118
+ protected
119
+ def continuous_query(query)
120
+ qs = @cache.getQueryService
121
+
122
+ cqAf = CqAttributesFactory.new
123
+ cqAf.addCqListener(self)
124
+ cqa = cqAf.create
125
+
126
+ @logger.debug("Running continuous query #{query}")
127
+ cq = qs.newCq("logstashCQ" + self.object_id.to_s, query, cqa)
128
+
129
+ cq.executeWithInitialResults
130
+ end
131
+
132
+ def register_interest(interest)
133
+ @region.getAttributesMutator.addCacheListener(self)
134
+ @region.registerInterestRegex(interest, InterestResultPolicy::NONE, false, true)
135
+ end
136
+
137
+ def deserialize_message(message)
138
+ if @serialization == "json"
139
+ message ? JSONFormatter.toJSON(message) : "{}"
140
+ else
141
+ message
142
+ end
143
+ end
144
+
145
+ def process_event(event, event_name)
146
+ message = deserialize_message(event)
147
+ @codec.decode(message) do |event|
148
+ decorate(event)
149
+ @logstash_queue << event
150
+ end
151
+ end
152
+
153
+ # multiple interfaces
154
+ def close
155
+ end
156
+
157
+ #
158
+ # CqListener interface
159
+ #
160
+ def onEvent(event)
161
+ key = event.getKey
162
+ newValue = event.getNewValue
163
+ @logger.debug("onEvent #{event.getQueryOperation} #{key} #{newValue}")
164
+
165
+ process_event(event.getNewValue, "onEvent", "gemfire://query/#{key}/#{event.getQueryOperation}")
166
+ end
167
+
168
+ def onError(event)
169
+ @logger.debug("onError #{event}")
170
+ end
171
+
172
+ #
173
+ # CacheListener interface
174
+ #
175
+ protected
176
+ def afterCreate(event)
177
+ regionName = event.getRegion.getName
178
+ key = event.getKey
179
+ newValue = event.getNewValue
180
+ @logger.debug("afterCreate #{regionName} #{key} #{newValue}")
181
+
182
+ process_event(event.getNewValue, "afterCreate", "gemfire://#{regionName}/#{key}/afterCreate")
183
+ end
184
+
185
+ def afterDestroy(event)
186
+ regionName = event.getRegion.getName
187
+ key = event.getKey
188
+ newValue = event.getNewValue
189
+ @logger.debug("afterDestroy #{regionName} #{key} #{newValue}")
190
+
191
+ process_event(nil, "afterDestroy", "gemfire://#{regionName}/#{key}/afterDestroy")
192
+ end
193
+
194
+ def afterUpdate(event)
195
+ regionName = event.getRegion.getName
196
+ key = event.getKey
197
+ oldValue = event.getOldValue
198
+ newValue = event.getNewValue
199
+ @logger.debug("afterUpdate #{regionName} #{key} #{oldValue} -> #{newValue}")
200
+
201
+ process_event(event.getNewValue, "afterUpdate", "gemfire://#{regionName}/#{key}/afterUpdate")
202
+ end
203
+
204
+ def afterRegionLive(event)
205
+ @logger.debug("afterRegionLive #{event}")
206
+ end
207
+
208
+ def afterRegionCreate(event)
209
+ @logger.debug("afterRegionCreate #{event}")
210
+ end
211
+
212
+ def afterRegionClear(event)
213
+ @logger.debug("afterRegionClear #{event}")
214
+ end
215
+
216
+ def afterRegionDestroy(event)
217
+ @logger.debug("afterRegionDestroy #{event}")
218
+ end
219
+
220
+ def afterRegionInvalidate(event)
221
+ @logger.debug("afterRegionInvalidate #{event}")
222
+ end
223
+ end # class LogStash::Inputs::Gemfire
@@ -0,0 +1,29 @@
1
+ Gem::Specification.new do |s|
2
+
3
+ s.name = 'logstash-input-gemfire'
4
+ s.version = '0.1.0'
5
+ s.licenses = ['Apache License (2.0)']
6
+ s.summary = "Push events to a GemFire region."
7
+ s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
8
+ s.authors = ["Elasticsearch"]
9
+ s.email = 'info@elasticsearch.com'
10
+ s.homepage = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
11
+ s.require_paths = ["lib"]
12
+
13
+ # Files
14
+ s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
15
+
16
+ # Tests
17
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
18
+
19
+ # Special flag to let us know this is actually a logstash plugin
20
+ s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
21
+
22
+ # Gem dependencies
23
+ s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
24
+
25
+ s.add_runtime_dependency 'ftw', ['~> 0.0.40']
26
+
27
+ s.add_development_dependency 'logstash-devutils'
28
+ end
29
+
@@ -0,0 +1 @@
1
+ require "logstash/devutils/rspec/spec_helper"
metadata ADDED
@@ -0,0 +1,101 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-input-gemfire
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Elasticsearch
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-11-19 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: logstash
15
+ version_requirements: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '>='
18
+ - !ruby/object:Gem::Version
19
+ version: 1.4.0
20
+ - - <
21
+ - !ruby/object:Gem::Version
22
+ version: 2.0.0
23
+ requirement: !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - '>='
26
+ - !ruby/object:Gem::Version
27
+ version: 1.4.0
28
+ - - <
29
+ - !ruby/object:Gem::Version
30
+ version: 2.0.0
31
+ prerelease: false
32
+ type: :runtime
33
+ - !ruby/object:Gem::Dependency
34
+ name: ftw
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - ~>
38
+ - !ruby/object:Gem::Version
39
+ version: 0.0.40
40
+ requirement: !ruby/object:Gem::Requirement
41
+ requirements:
42
+ - - ~>
43
+ - !ruby/object:Gem::Version
44
+ version: 0.0.40
45
+ prerelease: false
46
+ type: :runtime
47
+ - !ruby/object:Gem::Dependency
48
+ name: logstash-devutils
49
+ version_requirements: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - '>='
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ requirement: !ruby/object:Gem::Requirement
55
+ requirements:
56
+ - - '>='
57
+ - !ruby/object:Gem::Version
58
+ version: '0'
59
+ prerelease: false
60
+ type: :development
61
+ description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
62
+ email: info@elasticsearch.com
63
+ executables: []
64
+ extensions: []
65
+ extra_rdoc_files: []
66
+ files:
67
+ - .gitignore
68
+ - Gemfile
69
+ - LICENSE
70
+ - Rakefile
71
+ - lib/inputs/gemfire.rb
72
+ - logstash-input-gemfire.gemspec
73
+ - spec/inputs/gemfire_spec.rb
74
+ homepage: http://www.elasticsearch.org/guide/en/logstash/current/index.html
75
+ licenses:
76
+ - Apache License (2.0)
77
+ metadata:
78
+ logstash_plugin: 'true'
79
+ logstash_group: input
80
+ post_install_message:
81
+ rdoc_options: []
82
+ require_paths:
83
+ - lib
84
+ required_ruby_version: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - '>='
87
+ - !ruby/object:Gem::Version
88
+ version: '0'
89
+ required_rubygems_version: !ruby/object:Gem::Requirement
90
+ requirements:
91
+ - - '>='
92
+ - !ruby/object:Gem::Version
93
+ version: '0'
94
+ requirements: []
95
+ rubyforge_project:
96
+ rubygems_version: 2.2.2
97
+ signing_key:
98
+ specification_version: 4
99
+ summary: Push events to a GemFire region.
100
+ test_files:
101
+ - spec/inputs/gemfire_spec.rb