logstash-filter-redis 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 77dfa528ca95df82aad46d238a1dcf466ed9475e
-  data.tar.gz: ce51f3975f89e9f08b075f46a63787cde84015d8
+  metadata.gz: 380ebbff9adc91207fe0fad3aa9818e11378cc45
+  data.tar.gz: 6377159bf5e178ba6e68d91a4a02c062581ca170
 SHA512:
-  metadata.gz: ad2a1e2cd6cfa0bfd2ded1cd5e56308249cf7f0c71696165732f1e3835eaea6dfa9c36794bdcf552e5649626a7eba0e34f9c6820abf28ba903f76eb6e8e1432f
-  data.tar.gz: 969d23e7084ca18c701f79ccca26c2e15cc2b6fabfe3d6faa341451b17b42abb851f30695332b22e587ad6ccccf17d93e22ffaa81726ed5b1dfa16b4431971e0
+  metadata.gz: 8d934621faec917dfc1f2ab4e21a472e01f94b6e95ad5ca23a81cd073790764878abe6581dbcf16cab1afd72c50ae304b9b534ea7820c6cd96a4dd8d8626eeaf
+  data.tar.gz: 062615bbd46143c35b5c786a8cb48e166d6c85dd7adad450c18342fd6399e9cf0eb9c2716bc5a391a9fe4c36c10881878d6cb0f4c2f4310326e0e8dd53cb5db8

data/CHANGELOG.md

@@ -0,0 +1,7 @@
+# 0.2.0
+  - Removed data store feature
+  - Renamed configuration option "key" into "field" to match configuration of translate plugin
+  - Changed main functionality and configuration similar to translate filter (logstash-plugins/logstash-filter-translate)
+    - Added field, destination and override configuration options and their handling from logstash-plugins/logstash-filter-translate/blob/master/lib/logstash/filters/translate.rb
+# 0.1.0
+  - forked from meulop/logstash-filter-redis

data/CONTRIBUTORS

@@ -4,6 +4,7 @@ reports, or in general have helped logstash along its way.
 Contributors:
 * Aaron Mildenstein (untergeek)
 * Pier-Hugues Pellerin (ph)
+* Markus Paaso (make)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/README.md

@@ -1,17 +1,15 @@
-# TODO: remove/alter Elasticsearch example text from this file and the licence/copyright/etc...
-
 # Logstash Plugin
 
-This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
 ## Documentation
 
-Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
 
 - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
-- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
 
 ## Need Help?
 
@@ -55,7 +53,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
 ```
 - Install plugin
 ```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
 bin/plugin install --no-verify
+
 ```
 - Run Logstash with your plugin
 ```sh
@@ -73,7 +76,12 @@ gem build logstash-filter-awesome.gemspec
 ```
 - Install the plugin from the Logstash home
 ```sh
-bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
 ```
 - Start Logstash and proceed to test the plugin
 
@@ -85,4 +93,4 @@ Programming is not a required skill. Whatever you've seen about open source and
 
 It is more important to the community that you are able to contribute.
 
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/filters/redis.rb

@@ -2,6 +2,21 @@
 require "logstash/filters/base"
 require "logstash/namespace"
 
+# A general search and replace tool which queries replacement values from a redis instance.
+#
+# This is actually a redis version of a translate plugin. <https://www.elastic.co/guide/en/logstash/current/plugins-filters-translate.html>
+#
+# Operationally, if the event field specified in the "field" configuration
+# matches the EXACT contents of a redis key, the field's value will be substituted
+# with the matched key's value from the redis GET <key> command.
+#
+# By default, the redis filter will replace the contents of the 
+# matching event field (in-place). However, by using the "destination"
+# configuration item, you may also specify a target event field to
+# populate with the new translated value.
+# 
+# Alternatively, for simple string search and replacements for just a few values
+# you might consider using the gsub function of the mutate filter.
 
 class LogStash::Filters::Redis < LogStash::Filters::Base
 
@@ -19,80 +34,63 @@ class LogStash::Filters::Redis < LogStash::Filters::Base
   # The Redis database number.
   config :db, :validate => :number, :default => 0
   
-  # Tag to store key
-  config :store_tag, :validate => :string, :required => true
-
-  # Tag to retrieve key
-  config :retrieve_tag, :validate => :string, :required => true
-
-  # Redis key name
-  config :key, :validate => :string, :required => true
-
-  # Delete on retrieval
-  config :delete, :validate => :boolean, :default => false
-
-  # Fields to store in the Redis value
-  config :fields, :validate => :array, :default => ["message"]
-
-  # Field prefix
-  config :prefix, :validate => :string, :default => "old_"
+  # The name of the logstash event field containing the value to be compared for a
+  # match by the translate filter (e.g. "message", "host", "response_code"). 
+  # 
+  # If this field is an array, only the first value will be used.
+  config :field, :validate => :string, :required => true
+
+  # If the destination (or target) field already exists, this configuration item specifies
+  # whether the filter should skip translation (default) or overwrite the target field
+  # value with the new translation value.
+  config :override, :validate => :boolean, :default => false
+
+  # The destination field you wish to populate with the translated code. The default
+  # is a field named "redis". Set this to the same value as source if you want
+  # to do a substitution, in this case filter will allways succeed. This will clobber
+  # the old value of the source field! 
+  config :destination, :validate => :string, :default => "redis"
+
+  # In case no translation occurs in the event (no matches), this will add a default
+  # translation string, which will always populate "field", if the match failed.
+  #
+  # For example, if we have configured `fallback => "no match"`, using this dictionary:
+  #
+  #     foo: bar
+  #
+  # Then, if logstash received an event with the field `foo` set to "bar", the destination
+  # field would be set to "bar". However, if logstash received an event with `foo` set to "nope",
+  # then the destination field would still be populated, but with the value of "no match".
+  config :fallback, :validate => :string
 
   # Connection timeout
   config :timeout, :validate => :number, :required => false, :default => 5
 
-  # Key expiry time
-  config :expiry, :validate => :number, :default => 1800
-
   public
   def register
     require 'redis'
     require 'json'
     @redis = nil
-    @redis_url = "redis://#{@password}@#{@host}:#{@port}/#{@db}"
   end # def register
 
   public
   def filter(event)
-    return unless filter?(event)
-
-    # Allow dynamic key names using fields etc
-    key = event.sprintf(@key)
-
-    relevant = [@store_tag, @retrieve_tag].select do |t|
-       event["tags"].include?(t)
-    end
- 
-    @logger.debug(relevant)
-    return unless [] != relevant
- 
-    # Do we retrieve data from a prior event?
-    # N.b. we retrieve before we store so that we can do both if we want!
-    # (e.g. packet sequence numbers? ...)
-    #
-    if event["tags"].include?(@retrieve_tag)
-      @logger.debug("Found retrieve tag %{retrieve_tag}")
-      @redis ||= connect
-      val = @redis.get(key)
-      if val != nil
-        @logger.debug("Found key in Redis")
-        JSON.parse(val).each do |k,v|
-          event[prefix + k] = v
-        end
-        if @delete
-          @redis.del(key) && @logger.debug("Deleted key")
-        end
-      else
-        @logger.debug("Key not found in Redis")
+    return unless event.include?(@field)
+    return if event.include?(@destination) and not @override
+
+    source = event[@field].is_a?(Array) ? event[@field].first.to_s : event[@field].to_s
+    @redis ||= connect
+    val = @redis.get(source)
+    if val
+      begin
+        event[@destination] = JSON.parse(val)
+      rescue JSON::ParserError => e
+        event[@destination] = val
       end
+    elsif @fallback
+      event[@destination] = @fallback
     end
-    # Do we store data? 
-    if event["tags"].include?(@store_tag)
-      @logger.debug("Found store tag %{store_tag}")
-      val = event.to_hash().select { |name,value| fields.include?(name) }
-      @redis ||= connect
-      @redis.set(key, val.to_json) && @logger.debug("Stored key")
-      @redis.expire(key, @expiry) && @logger.debug("Set expiry key")
-    end
+      
     # filter_matched should go in the last line of our successful code
     filter_matched(event)
   end # def filter
@@ -103,7 +101,7 @@ class LogStash::Filters::Redis < LogStash::Filters::Base
        :host => @host,
        :port => @port, 
        :timeout => @timeout,
-       :db => db,
+       :db => @db,
        :password => @password.nil? ? nil : @password.value
      )
   end #def connect

data/logstash-filter-redis.gemspec

@@ -1,24 +1,28 @@
 Gem::Specification.new do |s|
+
   s.name = 'logstash-filter-redis'
-  s.version = '0.1.2'
+  s.version = '0.2.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This filter allows the storage of event fields in a redis key to be retrieved by a later event"
-  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors = ["meulop","make"]
-  s.email = '[info@meulop.uk]'
-  s.homepage = "https://github.com/make/lostash-redis-filter"
+  s.email = 'markus.paaso@gmail.com'
+  s.homepage = "https://github.com/make/logstash-redis-filter"
   s.require_paths = ["lib"]
 
   # Files
   s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
-   # Tests
+
+  # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
-  s.add_development_dependency 'logstash-devutils'
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
   s.add_runtime_dependency "redis", '>= 3.0.0', '< 4.0.0'
+
+  s.add_development_dependency 'logstash-devutils'
 end
+

data/spec/filters/redis_spec.rb

@@ -5,134 +5,58 @@ require "pp"
 require "json"
 
 describe LogStash::Filters::Redis do
+
   before(:all) do
     @redis = Redis.new()
+    @redis.set("somekey", "somevalue")
   end
 
-  after(:each) do
-    sleep(0.2)
-    @redis.keys("logstash-filter-redis-test*").each do |k|
-      @redis.del(k)
-    end
-  end
-
-  describe "Ignores irrelevant tag" do
+  describe "Retrieves data from redis" do
     config <<-CONFIG
       filter {
         redis {
-          add_tag => "STORED"
-          store_tag => "BEGIN"
-          retrieve_tag => "END"
-          key => "logstash-filter-redis-test"
+          field => "redis-key"
+          destination => "redis-value"
         }
       }
     CONFIG
 
-    sample({"message" => "Test message", "tags" => ["APACHE"]}) do
-      insist { subject["tags"].include?("STORED") } == false
-      insist { @redis.get("logstash-filter-redis-test") } == nil
+    sample({"message" => "Test message", "redis-key" => "somekey"}) do
+      insist { subject["redis-value"] } == "somevalue"
+      insist { @redis.get("somekey") } == "somevalue"
     end
-
   end
 
-  describe "Stores key and values" do
+  describe "Retrieves data from redis when field is an array" do
     config <<-CONFIG
       filter {
         redis {
-          add_tag => "STORED"
-          store_tag => "BEGIN"
-	  retrieve_tag => "END"
-	  key => "logstash-filter-redis-test"
+          field => ["redis-key"]
+          destination => "redis-value"
         }
       }
     CONFIG
 
-    sample({"message" => "Storing message", "tags" => ["BEGIN"]}) do
-      # Did we add tag to the event?
-      insist { subject["tags"].include?("STORED") } == true
-      # Did the Redis value get set correctly?
-      @stored = @redis.get("logstash-filter-redis-test")
-      insist { JSON.parse(@stored)["message"] } == "Storing message"
-    end
-  end
-
-  describe "Retrieves the stored data" do
-    config <<-CONFIG
-    filter{
-      redis {
-        store_tag => "BEGIN"
-	retrieve_tag => "END"
-	key => "logstash-filter-redis-test"
-      }
-    }
-    CONFIG
-
-    eventstore = {
-      "message" => "Storing message",
-      "tags"    => ["BEGIN"]
-    }
-    eventretrieve = {
-      "message" => "Retrieving message",
-      "tags"    => ["END"]
-    }
-
-    sample([eventstore, eventretrieve]) do
-      insist{ subject[1]["old_message"] } == "Storing message"
-    end
-  end
-
-  describe "Handles overlapping sets of events" do
-    # Fill me in
-
-    config <<-CONFIG
-    filter {
-      redis {
-        store_tag => "BEGIN"
-	retrieve_tag => "END"
-	key => "logstash-filter-redis-test-%{message}"
-      }
-    }
-    CONFIG
-
-    eventsstore = []
-    eventsretrieve = []
-    5.times do |i|
-     eventsstore << {
-       "message" => i.to_s,
-       "tags"    => ["BEGIN"]
-     }
-     eventsretrieve << {
-       "message" => i.to_s,
-       "tags"    => ["END"]
-     }
-    end
-    eventsretrieve.shuffle!
-    sample(eventsstore + eventsretrieve) do
-      insist {
-        subject[5..9].select { |e| e["old_message"] == e["message"] }
-      } != []
+    sample({"message" => "Test message", "redis-key" => "somekey"}) do
+      insist { subject["redis-value"] } == "somevalue"
+      insist { @redis.get("somekey") } == "somevalue"
     end
   end
 
-
-  describe "Deletes after retrieval" do
+  describe "Skips non-existing key" do
     config <<-CONFIG
       filter {
         redis {
-	  store_tag      => "BEGIN"
-	  retrieve_tag   => "END"
-	  delete         => true
-	  key            => "logstash-filter-redis-test"
-	}
+          field => "redis-key"
+          destination => "redis-value"
+        }
       }
     CONFIG
-    estore = { "message" => "Store", "tags" => ["BEGIN"] }
-    ereceive1 = { "message" => "Receive1", "tags" => ["END"] }
-    ereceive2 = { "message" => "Receive2", "tags" => ["END"] }
 
-    sample([estore,ereceive1,ereceive2]) do
-      insist { subject[1]["old_message"] } == "Store"
-      insist { subject[2]["old_message"] } == nil
+    sample({"message" => "Test message", "redis-key" => "notakey"}) do
+      insist { subject.include?("redis-value") } == false
     end
   end
+
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-redis
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - meulop
@@ -9,42 +9,22 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-03 00:00:00.000000000 Z
+date: 2016-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
-  name: logstash-core
+        version: '1.0'
+  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 2.0.0
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: logstash-devutils
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -65,15 +45,28 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 4.0.0
-description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
-email: "[info@meulop.uk]"
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+email: markus.paaso@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - CHANGELOG.md
 - CONTRIBUTORS
-- DEVELOPER.md
 - Gemfile
 - LICENSE
 - NOTICE.TXT
@@ -82,7 +75,7 @@ files:
 - logstash-filter-redis.gemspec
 - spec/filters/redis_spec.rb
 - spec/spec_helper.rb
-homepage: https://github.com/make/lostash-redis-filter
+homepage: https://github.com/make/logstash-redis-filter
 licenses:
 - Apache License (2.0)
 metadata:

data/DEVELOPER.md

@@ -1,2 +0,0 @@
-# logstash-filter-example
-Example filter plugin. This should help bootstrap your effort to write your own filter plugin!