gitlab-fluent-plugin-redis-slowlog 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c29fe816731c72b94242e6ea8debf36746f08f6f6d750b88eb95dcab727a006
-  data.tar.gz: 26792f5c673c1a6ea403e52b0be1473e94802564f3ec2298897348cd3bc5c863
+  metadata.gz: d38f425f2f17bdb5aa44efb1c3901240bb1f41f340cef86c9cca6834716e975b
+  data.tar.gz: e6739b5077e8d9c6b6482c7de64c80fea53d960cceb22f100ea55237269320f3
 SHA512:
-  metadata.gz: a5021486e250895773ffcfb3740cde86fc39cd909f0de98f85069c6592bcb383fd8382256639fa13a5492f5ee8def005197ba3e7e71dafd1e3452543c31b466e
-  data.tar.gz: 163fdfbbb4461a6fbe78558aa8930a030077fd459bcdc2bcfdc114cf27cf34e247d25192707f104d24bb70e5a55e7c6cb6573c7c65a2da910bce8f4665d90b1b
+  metadata.gz: ae4c9a0fafe66b7df1d9c49f639f882c5c834ad368dab29a9d0599a5af75ae4e73f4f113f4b4e3e6bc2f71b9aa9358403c48188865ffabd3797302e9add5ea53
+  data.tar.gz: 3d748722aa7ae4690f9298c455ee91fbbd4106f1fc505fa98b28f5198722086286a8343b907d10fbea0dab6bd9ad55a368b9f8af3bb1794f7f11b244d1614ff8

data/lib/fluent/plugin/in_redis_slowlog.rb

@@ -6,6 +6,8 @@ module Fluent
     class RedisSlowlogInput < Fluent::Plugin::Input
       Fluent::Plugin.register_input("redis_slowlog", self)
 
+      Entry = Struct.new(:id, :timestamp, :exec_time_us, :command)
+
       config_param :tag,      :string
       config_param :url,      :string,  default: nil
       config_param :path,     :string,  default: nil
@@ -15,12 +17,6 @@ module Fluent
       config_param :logsize,  :integer, default: 128
       config_param :interval, :integer, default: 10
 
-      def initialize
-        super
-
-        @current_log_id = 0
-      end
-
       def configure(conf)
         super
 
@@ -55,39 +51,49 @@ module Fluent
       private
 
       attr_reader :redis, :interval
-      attr_accessor :watcher, :watching, :current_log_id
+      attr_accessor :watcher, :watching
 
       def watch
+        # Check the current id of the slowlog, and start logging from there
+        current_log_id = get_slowlogs(1).first&.id || -1
+
         while watching
           sleep interval
 
-          self.current_log_id = output(current_log_id)
+          current_log_id = output(current_log_id)
         end
       end
 
-      def output(last_id = 0)
-        slowlogs = redis.slowlog("get", logsize)
+      def output(last_id)
+        slowlogs = get_slowlogs(logsize)
         return last_id if slowlogs.empty?
 
-        emit_slowlog(slowlogs)
+        # If the latest entry is smaller than what we last saw, redis was restarted
+        # Restart logging from the beginning.
+        last_id = -1 if slowlogs.first.id < last_id
+
+        emit_slowlog(slowlogs, last_id)
 
         # Return the id of the last slowlog entry we've logged
         # The first entry is the one that occurred last
-        slowlogs.first.first
+        slowlogs.first.id
       end
 
-      def emit_slowlog(slowlogs)
+      def emit_slowlog(slowlogs, last_id)
         slowlogs.reverse_each do |log|
           # Don't emit logs for entries we've already logged
-          next if log.first <= current_log_id
+          next if log.id <= last_id
 
-          log_hash = { "id" => log[0],
-                       "timestamp" => log[1].to_i,
-                       "exec_time" => log[2],
-                       "command" => log[3] }
-          router.emit(tag, Time.now.to_i, log_hash)
+          log_hash = { "id" => log.id,
+                       "exec_time" => log.exec_time_us,
+                       "command" => log.command }
+          router.emit(tag, Fluent::EventTime.new(log.timestamp.to_i), log_hash)
         end
       end
+
+      def get_slowlogs(size)
+        redis.slowlog("get", size).map { |slowlog_entry| Entry.new(*slowlog_entry.first(4)) }
+      end
     end
   end
 end

data/spec/fluent/plugin/redis_slowlog_input_spec.rb

@@ -6,21 +6,24 @@ describe Fluent::Plugin::RedisSlowlogInput do
   end
 
   let(:plugin) { driver.instance }
-  let(:slowlog) { [] }
-  let(:fake_redis) { instance_double(Redis, ping: "PONG", quit: "OK", slowlog: slowlog) }
+  let(:fake_redis) { instance_double(Redis, ping: "PONG", quit: "OK", slowlog: []) }
 
   subject(:driver) do
     config_string = config.map { |key, value| "#{key} #{value}" }.join("\n")
     Fluent::Test::Driver::Input.new(described_class).configure(config_string)
   end
 
+  def run(**options)
+    driver.run({ timeout: 1 }.merge(options))
+  end
+
   before do
     Fluent::Test.setup
   end
 
   context "when redis can't be reached" do
     it "raises an error" do
-      expect { driver.run }.to raise_error(Redis::CannotConnectError)
+      expect { run }.to raise_error(Redis::CannotConnectError)
     end
   end
 
@@ -40,7 +43,7 @@ describe Fluent::Plugin::RedisSlowlogInput do
 
       expect(Redis).to receive(:new).with(redis_params).and_return(fake_redis)
 
-      driver.run
+      run
     end
   end
 
@@ -63,10 +66,11 @@ describe Fluent::Plugin::RedisSlowlogInput do
     end
 
     it "does not raise errors" do
-      expect { driver.run }.not_to raise_error
+      expect { run }.not_to raise_error
     end
 
     it "polls the slowlog with the configured interval and size" do
+      expect(fake_redis).to receive(:slowlog).with("get", 1).ordered
       expect(plugin).to receive(:sleep).with(0).ordered
       expect(fake_redis).to receive(:slowlog).with("get", 10).ordered
       expect(plugin).to receive(:sleep).with(0).ordered
@@ -75,51 +79,76 @@ describe Fluent::Plugin::RedisSlowlogInput do
       # Limit to 2 cycles
       allow(plugin).to receive(:watching).thrice.and_return(true, true, false)
 
-      driver.run
+      run
     end
 
-    context "when the slowlog returns entries" do
-      let(:slowlog) do
+    context "when the slowlog returns entries", :aggregate_failures do
+      let(:startup_slowlog) { [[25637, 1590522108, 5, %w[SCAN 0]]] }
+      let(:logged_slowlog) do
         [
           [25640, 1590522258, 1, %w[ping]],
           [25639, 1590522249, 1, %w[ping]],
           [25638, 1590522208, 5, %w[SCAN 0]]
         ]
       end
+      let(:slowlog) { logged_slowlog + startup_slowlog }
 
       let(:expected_entries) do
-        slowlog.map(&method(:log_entry)).sort_by { |event| event["id"] }
+        logged_slowlog.map(&method(:log_entry)).sort_by { |event| event["id"] }
       end
 
       let(:emitted_entries) { driver.events.map(&:last) }
 
       it "emits an event for each slowlog in reverse order" do
-        driver.run(expect_emits: 3)
+        expect(fake_redis).to receive(:slowlog).and_return(startup_slowlog, slowlog)
+
+        run(expect_emits: 3)
 
         expect(driver.events.size).to eq(3)
         expect(emitted_entries).to eq(expected_entries)
       end
 
       it "does not log the same event twice" do
-        expect(fake_redis).to receive(:slowlog).and_return(slowlog.last(1), slowlog)
+        expect(fake_redis).to receive(:slowlog).and_return(startup_slowlog, slowlog.last(2), slowlog)
 
-        driver.run(expect_emits: 3)
+        run(expect_emits: 3)
 
         expect(driver.events.size).to eq(3)
         expect(emitted_entries).to eq(expected_entries)
       end
 
       it "emits results that can be streamed to fluent using MessagePack" do
-        driver.run(expect_emits: 3)
+        expect(fake_redis).to receive(:slowlog).and_return(startup_slowlog, slowlog)
+
+        run(expect_emits: 3)
 
         expect { driver.events.map(&:to_msgpack) }.not_to raise_error
       end
+
+      context "when redis restarts" do
+        let(:logged_slowlog) { [[25638, 1590522208, 5, %w[SCAN 0]]] }
+        let(:reset_slowlog) do
+          [
+            [2, 1590522258, 1, %w[ping]],
+            [1, 1590522249, 1, %w[ping]]
+          ]
+        end
+
+        it "emits all items" do
+          reset_entries = reset_slowlog.map(&method(:log_entry)).sort_by { |e| e["id"] }
+          expect(fake_redis).to receive(:slowlog).and_return(startup_slowlog, slowlog, [], reset_slowlog)
+
+          run(expect_emits: 3)
+
+          expect(emitted_entries.size).to eq(3)
+          expect(emitted_entries).to eq(expected_entries + reset_entries)
+        end
+      end
     end
   end
 
   def log_entry(slowlog_entry)
     { "id" => slowlog_entry.first,
-      "timestamp" => slowlog_entry[1].to_i,
       "exec_time" => slowlog_entry[2],
       "command" => slowlog_entry.last }
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-fluent-plugin-redis-slowlog
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - Bob Van Landuyt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-05 00:00:00.000000000 Z
+date: 2020-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler