logstash-input-gemfire 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1d5c414717063c02177082f724f1e9f5d5240d68
+  data.tar.gz: 0d0a7d467725a01f564b95b421383799c7cfc5e6
+SHA512:
+  metadata.gz: 1a64db6fb6fc2ef953cb41dd366b4c632b9f5aefce5b2f2245b8c5502d5ea3dc6a5e23af86ed5e7f5c59fb249eea5303fe9e6d23874e9c3890b3d19bcbf208eb
+  data.tar.gz: 6668cdbfc47ce444d9671e27162bf8c5c622b518488eb475f9cbf4109225765afcb878d71809331a5afb02ad46dc90914c19f5cbf8d77092e07d84471c603343

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+Gemfile.lock
+Gemfile.bak
+.bundle
+vendor

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+gemspec
+gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"
+

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -0,0 +1,8 @@
+@files=[]
+
+task :default do
+  system("rake -T")
+end
+
+require "logstash/devutils/rake"
+

data/lib/inputs/gemfire.rb

@@ -0,0 +1,223 @@
+# encoding: utf-8
+require "logstash/inputs/threadable"
+require "logstash/namespace"
+
+
+# Push events to a GemFire region.
+#
+# GemFire is an object database.
+#
+# To use this plugin you need to add gemfire.jar to your CLASSPATH.
+# Using format=json requires jackson.jar too; use of continuous
+# queries requires antlr.jar.
+#
+# Note: this plugin has only been tested with GemFire 7.0.
+#
+class LogStash::Inputs::Gemfire < LogStash::Inputs::Threadable
+
+  config_name "gemfire"
+  milestone 1
+
+  default :codec, "plain"
+
+  # Your client cache name
+  config :cache_name, :validate => :string, :default => "logstash"
+
+  # The path to a GemFire client cache XML file.
+  #
+  # Example:
+  #
+  #      <client-cache>
+  #        <pool name="client-pool" subscription-enabled="true" subscription-redundancy="1">
+  #            <locator host="localhost" port="31331"/>
+  #        </pool>
+  #        <region name="Logstash">
+  #            <region-attributes refid="CACHING_PROXY" pool-name="client-pool" >
+  #            </region-attributes>
+  #        </region>
+  #      </client-cache>
+  #
+  config :cache_xml_file, :validate => :string, :default => nil
+
+  # The region name
+  config :region_name, :validate => :string, :default => "Logstash"
+
+  # A regexp to use when registering interest for cache events.
+  # Ignored if a :query is specified.
+  config :interest_regexp, :validate => :string, :default => ".*"
+
+  # A query to run as a GemFire "continuous query"; if specified it takes
+  # precedence over :interest_regexp which will be ignore.
+  #
+  # Important: use of continuous queries requires subscriptions to be enabled on the client pool.
+  config :query, :validate => :string, :default => nil
+
+  # How the message is serialized in the cache. Can be one of "json" or "plain"; default is plain
+  config :serialization, :validate => :string, :default => nil
+
+  public
+  def register
+    import com.gemstone.gemfire.cache.AttributesMutator
+    import com.gemstone.gemfire.cache.InterestResultPolicy
+    import com.gemstone.gemfire.cache.client.ClientCacheFactory
+    import com.gemstone.gemfire.cache.client.ClientRegionShortcut
+    import com.gemstone.gemfire.cache.query.CqQuery
+    import com.gemstone.gemfire.cache.query.CqAttributes
+    import com.gemstone.gemfire.cache.query.CqAttributesFactory
+    import com.gemstone.gemfire.cache.query.QueryService
+    import com.gemstone.gemfire.cache.query.SelectResults
+    import com.gemstone.gemfire.pdx.JSONFormatter
+
+    @logger.info("Registering input", :plugin => self)
+  end # def register
+
+  def run(queue)
+    return if terminating?
+    connect
+
+    @logstash_queue = queue
+
+    if @query
+      continuous_query(@query)
+    else
+      register_interest(@interest_regexp)
+    end
+  end # def run
+
+  def teardown
+    @cache.close if @cache
+    @cache = nil
+    finished
+  end # def teardown
+
+  protected
+  def connect
+    begin
+      @logger.debug("Connecting to GemFire #{@cache_name}")
+
+      @cache = ClientCacheFactory.new.
+        set("name", @cache_name).
+        set("cache-xml-file", @cache_xml_file).create
+      @logger.debug("Created cache #{@cache.inspect}")
+
+    rescue => e
+      if terminating?
+        return
+      else
+        @logger.error("Gemfire connection error (during connect), will reconnect",
+                      :exception => e, :backtrace => e.backtrace)
+        sleep(1)
+        retry
+      end
+    end
+
+    @region = @cache.getRegion(@region_name);
+    @logger.debug("Created region #{@region.inspect}")
+  end # def connect
+
+  protected
+  def continuous_query(query)
+    qs = @cache.getQueryService
+
+    cqAf = CqAttributesFactory.new
+    cqAf.addCqListener(self)
+    cqa = cqAf.create
+
+    @logger.debug("Running continuous query #{query}")
+    cq = qs.newCq("logstashCQ" + self.object_id.to_s, query, cqa)
+
+    cq.executeWithInitialResults
+  end
+
+  def register_interest(interest)
+    @region.getAttributesMutator.addCacheListener(self)
+    @region.registerInterestRegex(interest, InterestResultPolicy::NONE, false, true)
+  end
+
+  def deserialize_message(message)
+    if @serialization == "json"
+      message ? JSONFormatter.toJSON(message) : "{}"
+    else
+      message
+    end
+  end
+
+  def process_event(event, event_name)
+    message = deserialize_message(event)
+    @codec.decode(message) do |event|
+      decorate(event)
+      @logstash_queue << event
+    end
+  end
+
+  # multiple interfaces
+  def close
+  end
+
+  #
+  # CqListener interface
+  #
+  def onEvent(event)
+    key = event.getKey
+    newValue = event.getNewValue
+    @logger.debug("onEvent #{event.getQueryOperation} #{key} #{newValue}")
+
+    process_event(event.getNewValue, "onEvent", "gemfire://query/#{key}/#{event.getQueryOperation}")
+  end
+
+  def onError(event)
+    @logger.debug("onError #{event}")
+  end
+
+  #
+  # CacheListener interface
+  #
+  protected
+  def afterCreate(event)
+    regionName = event.getRegion.getName
+    key = event.getKey
+    newValue = event.getNewValue
+    @logger.debug("afterCreate #{regionName} #{key} #{newValue}")
+
+    process_event(event.getNewValue, "afterCreate", "gemfire://#{regionName}/#{key}/afterCreate")
+  end
+
+  def afterDestroy(event)
+    regionName = event.getRegion.getName
+    key = event.getKey
+    newValue = event.getNewValue
+    @logger.debug("afterDestroy #{regionName} #{key} #{newValue}")
+
+    process_event(nil, "afterDestroy", "gemfire://#{regionName}/#{key}/afterDestroy")
+  end
+
+  def afterUpdate(event)
+    regionName = event.getRegion.getName
+    key = event.getKey
+    oldValue = event.getOldValue
+    newValue = event.getNewValue
+    @logger.debug("afterUpdate #{regionName} #{key} #{oldValue} -> #{newValue}")
+
+    process_event(event.getNewValue, "afterUpdate", "gemfire://#{regionName}/#{key}/afterUpdate")
+  end
+
+  def afterRegionLive(event)
+    @logger.debug("afterRegionLive #{event}")
+  end
+
+  def afterRegionCreate(event)
+    @logger.debug("afterRegionCreate #{event}")
+  end
+
+  def afterRegionClear(event)
+    @logger.debug("afterRegionClear #{event}")
+  end
+
+  def afterRegionDestroy(event)
+    @logger.debug("afterRegionDestroy #{event}")
+  end
+
+  def afterRegionInvalidate(event)
+    @logger.debug("afterRegionInvalidate #{event}")
+  end
+end # class LogStash::Inputs::Gemfire

data/logstash-input-gemfire.gemspec

@@ -0,0 +1,29 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-input-gemfire'
+  s.version         = '0.1.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Push events to a GemFire region."
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Elasticsearch"]
+  s.email           = 'info@elasticsearch.com'
+  s.homepage        = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
+
+  s.add_runtime_dependency 'ftw', ['~> 0.0.40']
+
+  s.add_development_dependency 'logstash-devutils'
+end
+

data/spec/inputs/gemfire_spec.rb

@@ -0,0 +1 @@
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-gemfire
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Elasticsearch
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-11-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: ftw
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.40
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.40
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: info@elasticsearch.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- Rakefile
+- lib/inputs/gemfire.rb
+- logstash-input-gemfire.gemspec
+- spec/inputs/gemfire_spec.rb
+homepage: http://www.elasticsearch.org/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.2.2
+signing_key:
+specification_version: 4
+summary: Push events to a GemFire region.
+test_files:
+- spec/inputs/gemfire_spec.rb