logstash-input-redis 3.2.2 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3166f0d0790fde3501ba2bbe462c826548d3f4f3796354cb528ec7dec9b250db
-  data.tar.gz: 64a5bd0542681731513c830c611b051bf445d729c5b9259a31801f70fb07d825
+  metadata.gz: 8e3e4321bea3cf53703a9cb4a730e72757c84293359a0b8bf627489900124994
+  data.tar.gz: 404ca9d392bb01710e8f5617a4b583761667b7f8a933ec49a9fed5c4adc9a0ba
 SHA512:
-  metadata.gz: c6d0529bd28db81ea0a134a504dd79b1ad499cbefae933a20946abafa7e80c95ff478941c6739d855325629ff0d2ab028e7fd30227a8994a23d1b1e6153f5388
-  data.tar.gz: da5342d81946331dec0fef00fdc132c62255f837467dbd2b582e503ff701e8f9813a05b167c580b53fc56bd8f4f361e19283f601cdc02269b79189f20feaeaeb
+  metadata.gz: a5a2b7347085f5e45f4318b8deb23e87df06b7a6726ffd6c4d51740a9574c226b043ca8afec8a8c8a0d3918959d60d91797efd6da3add028891f1280846b7b67
+  data.tar.gz: 2cd45bba305f53142589737821c8e99402dd020fa1c845bd7c4399406e6569dd82e38683c562a33c4aa203e39572a201d02085864aea7b1afc8a71f53d46b9d4

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.4.0
+  - Added support for renamed redis commands [#29](https://github.com/logstash-plugins/logstash-input-redis/issues/29)
+
+## 3.3.0
+  - Add channel to the event #46
+
 ## 3.2.2
   - Docs: Set the default_codec doc attribute.
 

data/docs/index.asciidoc

@@ -53,6 +53,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-threads>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-timeout>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-command_map>> |<<hash,hash>>|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -154,7 +155,15 @@ Enable SSL support.
 
 Initial connection timeout in seconds.
 
+[id="plugins-{type}s-{plugin}-command_map"]
+===== `command_map`
 
+  * Value type is <<hash,hash>>
+  * There is no default value for this setting.
+  * key is the default command name, value is the renamed command.
+
+Configure renamed redis commands in the form of "oldname" => "newname".
+Redis allows for the renaming or disabling of commands in its protocol, see:  https://redis.io/topics/security
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/inputs/redis.rb

@@ -56,9 +56,12 @@ module LogStash module Inputs class Redis < LogStash::Inputs::Threadable
   # The number of events to return from Redis using EVAL.
   config :batch_count, :validate => :number, :default => 125
 
+  # Redefined Redis commands to be passed to the Redis client.
+  config :command_map, :validate => :hash, :default => {}
+
   public
   # public API
-  # use to store a proc that can provide a redis instance or mock
+  # use to store a proc that can provide a Redis instance or mock
   def add_external_redis_builder(builder) #callable
     @redis_builder = builder
     self
@@ -151,6 +154,15 @@ module LogStash module Inputs class Redis < LogStash::Inputs::Threadable
   # private
   def connect
     redis = new_redis_instance
+
+    # register any renamed Redis commands
+    if @command_map.any?
+      client_command_map = redis.client.command_map
+      @command_map.each do |name, renamed|
+        client_command_map[name.to_sym] = renamed.to_sym
+      end
+    end
+
     load_batch_script(redis) if batched? && is_list_type?
     redis
   end # def connect
@@ -158,20 +170,21 @@ module LogStash module Inputs class Redis < LogStash::Inputs::Threadable
   # private
   def load_batch_script(redis)
     #A Redis Lua EVAL script to fetch a count of keys
-      redis_script = <<EOF
-        local batchsize = tonumber(ARGV[1])
-        local result = redis.call('lrange', KEYS[1], 0, batchsize)
-        redis.call('ltrim', KEYS[1], batchsize + 1, -1)
-        return result
+    redis_script = <<EOF
+      local batchsize = tonumber(ARGV[1])
+      local result = redis.call(\'#{@command_map.fetch('lrange', 'lrange')}\', KEYS[1], 0, batchsize)
+      redis.call(\'#{@command_map.fetch('ltrim', 'ltrim')}\', KEYS[1], batchsize + 1, -1)
+      return result
 EOF
     @redis_script_sha = redis.script(:load, redis_script)
   end
 
   # private
-  def queue_event(msg, output_queue)
+  def queue_event(msg, output_queue, channel=nil)
     begin
       @codec.decode(msg) do |event|
         decorate(event)
+        event.set("[@metadata][redis_channel]", channel) if !channel.nil?
         output_queue << event
       end
     rescue => e # parse or event creation error
@@ -297,7 +310,7 @@ EOF
       end
 
       on.message do |channel, message|
-        queue_event(message, output_queue)
+        queue_event(message, output_queue, channel)
       end
 
       on.unsubscribe do |channel, count|
@@ -320,7 +333,7 @@ EOF
       end
 
       on.pmessage do |pattern, channel, message|
-        queue_event(message, output_queue)
+        queue_event(message, output_queue, channel)
       end
 
       on.punsubscribe do |channel, count|

data/logstash-input-redis.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-redis'
-  s.version         = '3.2.2'
+  s.version         = '3.4.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads events from a Redis instance"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/redis_spec.rb

@@ -76,6 +76,7 @@ describe LogStash::Inputs::Redis do
   let(:cfg) { {'key' => 'foo', 'data_type' => data_type, 'batch_count' => batch_count} }
   let(:quit_calls) { [:quit] }
   let(:accumulator) { [] }
+  let(:command_map) { {} }
 
   subject do
     LogStash::Plugin.lookup("input", "redis")
@@ -88,6 +89,71 @@ describe LogStash::Inputs::Redis do
     end
   end
 
+  context 'renamed redis commands' do
+    let(:cfg) {
+      {'key' => 'foo',
+      'data_type' => data_type,
+      'command_map' =>
+        {
+        'blpop' => 'testblpop',
+        'evalsha' => 'testevalsha',
+        'lrange' => 'testlrange',
+        'ltrim' => 'testltrim',
+        'script' => 'testscript',
+        'subscribe' => 'testsubscribe',
+        'psubscribe' => 'testpsubscribe',
+        },
+        'batch_count' => 2
+      }
+    }
+
+    before do
+      subject.register
+      allow(redis).to receive(:connected?)
+      allow(redis).to receive(:client).and_return(connection)
+      allow(connection).to receive(:command_map).and_return(command_map)
+    end
+
+    it 'sets the renamed commands in the command map' do
+      allow(redis).to receive(:script)
+      allow(redis).to receive(:evalsha).and_return([])
+
+      tt = Thread.new do
+        sleep 0.01
+        subject.do_stop
+      end
+
+      subject.run(accumulator)
+      tt.join
+
+      expect(command_map[:blpop]).to eq cfg['command_map']['blpop'].to_sym
+      expect(command_map[:evalsha]).to eq cfg['command_map']['evalsha'].to_sym
+      expect(command_map[:lrange]).to eq cfg['command_map']['lrange'].to_sym
+      expect(command_map[:ltrim]).to eq cfg['command_map']['ltrim'].to_sym
+      expect(command_map[:script]).to eq cfg['command_map']['script'].to_sym
+      expect(command_map[:subscribe]).to eq cfg['command_map']['subscribe'].to_sym
+      expect(command_map[:psubscribe]).to eq cfg['command_map']['psubscribe'].to_sym
+    end
+
+    it 'loads the batch script with the renamed command' do
+      capture = nil
+      allow(redis).to receive(:script) { |load, lua_script| capture = lua_script }
+      allow(redis).to receive(:evalsha).and_return([])
+
+      tt = Thread.new do
+        sleep 0.01
+        subject.do_stop
+      end
+
+      subject.run(accumulator)
+      tt.join
+
+      expect(capture).to include "redis.call('#{cfg['command_map']['lrange']}', KEYS[1], 0, batchsize)"
+      expect(capture).to include "redis.call('#{cfg['command_map']['ltrim']}', KEYS[1], batchsize + 1, -1)"
+    end
+  end
+
+
   context 'runtime for list data_type' do
     before do
       subject.register
@@ -272,6 +338,18 @@ describe LogStash::Inputs::Redis do
 
           expect(accumulator.size).to eq(2)
         end
+        it 'events had redis_channel' do
+          #simulate the input thread
+          rt = run_it_thread(instance)
+          #simulate the other system thread
+          publish_thread(instance.new_redis_instance, 'c').join
+          #simulate the pipeline thread
+          close_thread(instance, rt).join
+          e1 = accumulator.pop
+          e2 = accumulator.pop
+          expect(e1.get('[@metadata][redis_channel]')).to eq('foo')
+          expect(e2.get('[@metadata][redis_channel]')).to eq('foo')
+        end
       end
     end
 
@@ -299,6 +377,18 @@ describe LogStash::Inputs::Redis do
 
           expect(accumulator.size).to eq(2)
         end
+        it 'events had redis_channel' do
+          #simulate the input thread
+          rt = run_it_thread(instance)
+          #simulate the other system thread
+          publish_thread(instance.new_redis_instance, 'pc').join
+          #simulate the pipeline thread
+          close_thread(instance, rt).join
+          e1 = accumulator.pop
+          e2 = accumulator.pop
+          expect(e1.get('[@metadata][redis_channel]')).to eq('foo')
+          expect(e2.get('[@metadata][redis_channel]')).to eq('foo')
+        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-redis
 version: !ruby/object:Gem::Version
-  version: 3.2.2
+  version: 3.4.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-06 00:00:00.000000000 Z
+date: 2018-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -112,7 +112,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Reads events from a Redis instance