logstash-filter-rediss 0.3.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7f19ec0db128b3d2fab457046c9cef823897d334dc142c17230b9ee4d634f6dc
+  data.tar.gz: ecf92c21293d5a49d37fb3a958cdeb3cdbe14591732c1e5bef892b1aa034c452
+SHA512:
+  metadata.gz: e162d6c193d0f035a4b23bf0a0e5a9bb960d1cce69883b286a0706f265d7a40d676b478976c6e39c8a3d82eaea91657352dec4fc16644c0833e20d6d37f20acc
+  data.tar.gz: '04908bcb693b5beefa287da14e49926958bb05100e617f606582c5f5cb0348e0bef8193992c6e1df46c8591991c7a71470ea84ee497f2d700fb48c60d76c32ef'

data/CHANGELOG.md

@@ -0,0 +1,19 @@
+## 0.3.2
+ - Introduce redis command `sismember`
+
+## 0.3.1
+ - Prevent infinite loops for retries after errors or failed lock acquisitions
+ - Introduce optional configuration options for controlling the count and
+   the interval of retries - see :max_retries, :lock_retry_interval and
+   :max_lock_retries
+
+## 0.3.0
+ - Support for GET, SET, EXISTS, DEL, SADD, SMEMBERS and SCARD
+ - Introduce new control field `cmd_key_is_formatted` for declaring commands
+   to be resolved through - see %{foo} handling
+
+## 0.2.0
+ - Support for HSET and HGET
+
+## 0.1.0
+ - Initial version; simple caching with redis

data/CONTRIBUTORS

@@ -0,0 +1,10 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash-filter-cache-redis along its way.
+
+Contributors:
+* David Robakowski (drobakowski)
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+logstash-filter-cache-redis, and you aren't on the list above and want to be,
+please let us know and we'll make sure you're here. Contributions from folks
+like you are what make open source awesome.

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gem 'coveralls', require: false
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016, Synlay Technologies UG & Co. KG.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,27 @@
+# Logstash Filter Redis Cache Plugin
+
+[![Build Status](https://travis-ci.org/synlay/logstash-filter-cache-redis.svg)](https://travis-ci.org/synlay/logstash-filter-cache-redis)
+[![Coverage Status](https://coveralls.io/repos/github/synlay/logstash-filter-cache-redis/badge.svg?branch=master)](https://coveralls.io/github/synlay/logstash-filter-cache-redis?branch=master)
+[![Gem Version](https://badge.fury.io/rb/logstash-filter-cache-redis.svg)](https://badge.fury.io/rb/logstash-filter-cache-redis)
+[![GitHub license](https://img.shields.io/github/license/synlay/logstash-filter-cache-redis.svg)](https://github.com/synlay/logstash-filter-cache-redis)
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is MIT, see [LICENSE](http://github.com/synlay/logstash-filter-cache-redis/LICENSE) for further infos.
+
+## Documentation
+
+This filter will store and retrieve data from Redis data cache. The fields `source` and `target` are used alternatively as data in- or output fields, while the value of a defined command like `rpush` will look for the corresponding event and use that value as the key. The following example for instance will store data from the event `ProductEntity` under the key based upon the data from the event `ProductId`:
+
+```ruby
+filter {
+    cache_redis {
+        rpush => "ProductId"
+        source => "ProductEntity"
+    }
+}
+```
+
+## Developing
+
+For further instructions on howto develop on logstash plugins, please see the documentation of the official [logstash-filter-example](https://github.com/logstash-plugins/logstash-filter-example#developing).

data/lib/logstash/filters/rediss.rb

@@ -0,0 +1,284 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+require "redis"
+require "redlock"
+
+class LogStash::Filters::Rediss < LogStash::Filters::Base
+    config_name "rediss"
+
+    # The field to perform filter
+    #
+    config :source, :validate => :string, :default => "message"
+
+    # The name of the container to put the result
+    #
+    config :target, :validate => :string, :default => "message"
+
+    config :field, :validate => :string
+
+    # Expire time in seconds
+    config :ttl => :number
+
+    # Informs if the connection is to be made with SSL or not
+    config :ssl => :boolean
+
+    # For now only working for rpushnx and llen!
+    config :cmd_key_is_formatted, :validate => :boolean, :default => false
+
+    # The hostname(s) of your Redis server(s). Ports may be specified on any
+    # hostname, which will override the global port config.
+    # If the hosts list is an array, Logstash will pick one random host to connect to,
+    # if that host is disconnected it will then pick another.
+    #
+    # For example:
+    # [source,ruby]
+    #     "127.0.0.1"
+    #     ["127.0.0.1", "127.0.0.2"]
+    #     ["127.0.0.1:6380", "127.0.0.1"]
+    config :host, :validate => :array, :default => ["127.0.0.1"]
+
+    # Shuffle the host list during Logstash startup.
+    config :shuffle_hosts, :validate => :boolean, :default => true
+
+    # The default port to connect on. Can be overridden on any hostname.
+    config :port, :validate => :number, :default => 6379
+
+    # The Redis database number.
+    config :db, :validate => :number, :default => 0
+
+    # Redis initial connection timeout in seconds.
+    config :timeout, :validate => :number, :default => 5
+
+    # Password to authenticate with.  There is no authentication by default.
+    config :password, :validate => :password
+
+    # Interval for reconnecting to failed Redis connections
+    config :reconnect_interval, :validate => :number, :default => 1
+
+    # Maximal count of command retries after a crash because of a failure
+    config :max_retries, :validate => :number, :default => 3
+
+    # Interval for retrying to acquire a lock
+    config :lock_retry_interval, :validate => :number, :default => 1
+
+    # Maximal count of retries to acquire a lock
+    config :max_lock_retries, :validate => :number, :default => 3
+
+    config :get, :validate => :string
+
+    config :set, :validate => :string
+
+    config :setex, :validate => :string
+
+    config :exists, :validate => :string
+
+    config :del, :validate => :string
+
+    # # Sets the action. If set to true, it will get the data from redis cache
+    # config :get, :validate => :boolean, :default => false
+    config :llen, :validate => :string
+
+    # # Sets the action. If set to true, it will get the data from redis cache
+    # config :get, :validate => :boolean, :default => false
+    config :rpush, :validate => :string
+
+    # # Sets the action. If set to true, it will get the data from redis cache
+    # config :get, :validate => :boolean, :default => false
+    config :rpushnx, :validate => :string
+
+    # Sets the action. If set to true, it will get the data from redis cache
+    config :hset, :validate => :string
+
+    # Sets the action. If set to true, it will get the data from redis cache
+    config :hget, :validate => :string
+
+    config :sadd, :validate => :string
+
+    config :sismember, :validate => :string
+
+    config :smembers, :validate => :string
+
+    config :scard, :validate => :string
+
+    # config :get, :validate => :boolean, :default => false
+    config :lock_timeout, :validate => :number, :default => 5000
+
+    # # Sets the action. If set to true, it will get the data from redis cache
+    # config :get, :validate => :boolean, :default => false
+    config :rpop, :validate => :string
+
+    # # Sets the action. If set to true, it will get the data from redis cache
+    # config :get, :validate => :boolean, :default => false
+    config :lpop, :validate => :string
+
+    # # Sets the action. If set to true, it will get the data from redis cache
+    # config :get, :validate => :boolean, :default => false
+    # O(N)
+    config :lget, :validate => :string
+
+
+    public
+    def register
+        @redis = nil
+        @lock_manager = nil
+        if @shuffle_hosts
+            @host.shuffle!
+        end
+        @host_idx = 0
+    end # def register
+
+
+    def filter(event)
+
+        # TODO: Maybe refactor the interface into a more flexible one with two
+        #       main configs 'cmd' & 'args'. Then it would be possible to eliminate
+        #       all if clauses and replace it through one hashmap call, where
+        #       the hashmap would be a mapping from 'cmd' -> <cmd_function_ref>
+        #       E.q.: cmds.fetch(event.get(@llen), &method(:cmd_not_found_err))
+        max_retries = @max_retries
+        begin
+            @redis ||= connect
+
+            if @get
+                event.set(@target, @redis.get(event.get(@get)))
+            end
+
+            if @set
+                @redis.set(event.get(@set), event.get(@source))
+            end
+
+            if @setex
+                @redis.setex(event.get(@setex), event.get(@ttl), event.get(@source))
+            end
+
+            if @exists
+                @redis.exists(event.get(@exists))
+            end
+
+            if @del
+                @redis.del(event.get(@del))
+            end
+
+            if @hget
+                event.set(@target, @redis.hget(event.get(@hget), event.get(@source)))
+            end
+
+            if @hset
+                @redis.hset(event.get(@hset), event.get(@field), event.get(@source))
+            end
+
+            if @sadd
+                @redis.sadd(event.get(@sadd), event.get(@source))
+            end
+
+            if @sismember
+                event.set(@target, @redis.sismember(event.get(@sismember), event.get(@source)))
+            end
+
+            if @smembers
+                @redis.smembers(event.get(@smembers))
+            end
+
+            if @scard
+                event.set(@target, @redis.scard(event.get(@scard)))
+            end
+
+            if @llen
+                key = @cmd_key_is_formatted ? event.sprintf(@llen) : event.get(@llen)
+                event.set(@target, @redis.llen(key))
+            end
+
+            if @rpush
+                @redis.rpush(event.get(@rpush), event.get(@source))
+            end
+
+            if @rpushnx
+                key = @cmd_key_is_formatted ? event.sprintf(@rpushnx) : event.get(@rpushnx)
+                max_lock_retries = @max_lock_retries
+                begin
+                    @lock_manager ||= connect_lockmanager
+                    @lock_manager.lock!("lock_#{key}", @lock_timeout) do
+                        @redis.rpush(key, event.get(@source)) unless @redis.exists(key)
+                    end
+                rescue Redlock::LockError => e
+                    @logger.warn("Failed to lock section 'rpushnx' for key: #{key}",
+                                 :event => event, :exception => e)
+                    sleep @lock_retry_interval
+                    max_lock_retries -= 1
+                    unless max_lock_retries < 0
+                        retry
+                    else
+                        @logger.error("Max retries reached for trying to lock section 'rpushnx' for key: #{key}",
+                                      :event => event, :exception => e)
+                    end
+                end
+            end
+
+            if @rpop
+                event.set(@target, @redis.rpop(event.get(@rpop)))
+            end
+
+            if @lget
+                event.set(@target, @redis.lrange(event.get(@lget), 0, -1))
+            end
+
+        rescue => e
+            @logger.warn("Failed to send event to Redis, retrying after #{@reconnect_interval} seconds...", :event => event,
+                         :exception => e, :backtrace => e.backtrace)
+            sleep @reconnect_interval
+            @redis = nil
+            @lock_manager = nil
+            max_retries -= 1
+            unless max_retries < 0
+                retry
+            else
+                @logger.error("Max retries reached for trying to execute a command",
+                              :event => event, :exception => e)
+            end
+        end
+
+        # filter_matched should go in the last line of our successful code
+        filter_matched(event)
+    end # def filter
+
+
+    private
+    def connect
+        @current_host, @current_port = @host[@host_idx].split(':')
+        @host_idx = @host_idx + 1 >= @host.length ? 0 : @host_idx + 1
+
+        if not @current_port
+            @current_port = @port
+        end
+
+        params = {
+            :host => @current_host,
+            :port => @current_port,
+            :timeout => @timeout,
+            :db => @db,
+            :ssl => @ssl
+        }
+
+        @logger.debug("connection params", params)
+
+        if @password
+            params[:password] = @password.value
+        end
+
+        Redis.new(params)
+    end # def connect
+
+    def connect_lockmanager
+        @protocol =  @ssl ? 'rediss://' : 'redis://'
+
+        hosts = Array(@host).map { |host|
+            host.prepend(@protocol) unless host.start_with?(@protocol) 
+        }
+
+        @logger.debug("lock_manager hosts", hosts)
+
+        Redlock::Client.new(hosts)
+    end # def connect
+
+end # class LogStash::Filters::Example

data/logstash-filter-rediss.gemspec

@@ -0,0 +1,25 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-filter-rediss'
+  s.version         = '0.3.4'
+  s.licenses        = ['MIT']
+  s.summary         = "Redis Cache Filter for Logstash"
+  s.description     = "A Logstash filter plugin for storing and retrieving data from redis cache. This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gem_name. This gem is not a stand-alone program."
+  s.authors         = ["David Robakowski"]
+  s.email           = 'david.robakowski@synlay.com'
+  s.homepage        = "https://github.com/bruno-sales/logstash-filter-cache-redis"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_runtime_dependency 'redis', '~> 3.3', '>= 3.3.3'
+  s.add_runtime_dependency 'redlock', '~> 0.2', '>= 0.2.0'
+  s.add_development_dependency 'logstash-devutils', '~> 0'
+end

metadata

@@ -0,0 +1,122 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-rediss
+version: !ruby/object:Gem::Version
+  version: 0.3.4
+platform: ruby
+authors:
+- David Robakowski
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-10-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.3.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.3.3
+- !ruby/object:Gem::Dependency
+  name: redlock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A Logstash filter plugin for storing and retrieving data from redis cache.
+  This gem is a Logstash plugin required to be installed on top of the Logstash core
+  pipeline using $LS_HOME/bin/logstash-plugin install gem_name. This gem is not a
+  stand-alone program.
+email: david.robakowski@synlay.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/filters/rediss.rb
+- logstash-filter-rediss.gemspec
+homepage: https://github.com/bruno-sales/logstash-filter-cache-redis
+licenses:
+- MIT
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Redis Cache Filter for Logstash
+test_files: []