gitlab-fluent-plugin-redis-slowlog 0.0.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8fc7a7afc37728bd3e9ae94eeaa408103bf28f00a87c6e931a4568f675bab927
-  data.tar.gz: 3e7fc4b3bc8dbb082f84f211f4de09ceb9b415b2d9d7c0c59b67316498586730
+  metadata.gz: 3c1ce26c15184cd4542891a3175434b81ed44b295a6ffd2a8c96222d4fd16da8
+  data.tar.gz: 632613b3d6b036a6f94c66bea02dc08d5dff119202b1614f5df4ff1efd3c1c58
 SHA512:
-  metadata.gz: 4e315c9db4c5d3b70c196fcb9dd912304ea8619766f8d9bbc8c00eeb1d4389f96fd37f1279f061e1c3d977944a972209763cadf417bb1dc18c98b12aceeeb9c9
-  data.tar.gz: 59f9771844039b0a250c9f6bcbf342ea42163d2710d79b9121f222c39736d3cb8f40e2978cf40a2e86496b507ef1da549fdfbeadea47098c9c6dc86efffb8739
+  metadata.gz: aea063934370511db16b64743463868b909d4685c141e0d56365f2edb604688125851d815b5c241f0a721be42e17785aea0f6ce862e726b050e1812d70a0bd2a
+  data.tar.gz: 99ff9bf22f51fc3b5d6bffc55db8c3979b67bc41a4ef0ba69bb4ab136332a6de938694f39a9f4fc4409a2f41dcfdcb84a14bc8aec6fc353544a14dd9578713c2

data/.gitignore

@@ -1 +1,2 @@
 *.gem
+Gemfile.lock

data/.gitlab-ci.yml

@@ -1,19 +1,40 @@
-test:
-  image: ruby:2.6
+workflow:
+  rules:
+    # For merge requests, create a pipeline.
+    - if: '$CI_MERGE_REQUEST_IID'
+    # For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    # For tags, create a pipeline.
+    - if: '$CI_COMMIT_TAG'
+
+.test_template: &test_definition
   stage: test
   script:
     - gem install bundler
     - bundle install
     - bundle exec rake
 
+test2.6:
+  image: ruby:2.6
+  <<: *test_definition
+test2.5:
+  image: ruby:2.5
+  <<: *test_definition
+test2.4:
+  image: ruby:2.4
+  <<: *test_definition
+
 deploy:
+  image: ruby:2.6
   stage: deploy
   script:
+    - gem install bundler
     - tools/deploy-rubygem.sh
   only:
     - tags
 
 release:
+  image: ruby:2.6
   stage: deploy
   script:
     - tools/update-changelog.rb "${CI_COMMIT_TAG}"

data/.rubocop.yml

@@ -3,14 +3,14 @@ inherit_gem:
     - rubocop-default.yml
 
 AllCops:
-  TargetRubyVersion: 2.6
+  TargetRubyVersion: 2.4
 
 # Let's just pick something from the beginning
 Style/StringLiterals:
   Enabled: true
   EnforcedStyle: double_quotes
 
-# This isn't Rails, so diable these
+# This isn't Rails, so disable these
 Rails/RakeEnvironment:
   Enabled: false
 CodeReuse/ActiveRecord:

data/README.md

@@ -118,16 +118,15 @@ rake verify               # Run RuboCop
 rake verify:auto_correct  # Auto-correct RuboCop offenses
 ```
 
-
 ### Releasing a new version
 
 Releasing a new version can be done by pushing a new tag, or creating
 it from the
 [interface](https://gitlab.com/gitlab-org/fluent-plugin-redis-slowlog/-/tags).
 
-A new changelog will automatically be added to the release on Gitlab.
+A new changelog will automatically be added to the release on GitLab.
 
 The new version will automatically be published to
-[rubygems](https://rubygems.org/gems/gitlab-labkit) when the pipeline
-for the tag completes. It might take a few minutes before the update
-is available.
+[rubygems](https://rubygems.org/gems/gitlab-fluent-plugin-redis-slowlog)
+when the pipeline for the tag completes. It might take a few minutes
+before the update is available.

data/gitlab-fluent-plugin-redis-slowlog.gemspec

@@ -3,12 +3,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "gitlab-fluent-plugin-redis-slowlog"
-  spec.version = "0.0.0" # `git describe --tags`.chomp.gsub(/^v/, "")
+  spec.version = `git describe --tags`.chomp.gsub(/^v/, "")
   spec.metadata = { "source_code_uri" => "https://gitlab.com/gitlab-org/fluent-plugin-redis-slowlog" }
   spec.authors = ["Bob Van Landuyt"]
   spec.email   = ["bob@gitlab.com"]
 
-  spec.required_ruby_version = ">= 2.6"
+  spec.required_ruby_version = ">= 2.4"
 
   spec.summary       = "Emit redis slowlog entries into fluentd"
   spec.homepage      = "http://gitlab.com/gitlab-org/gitlab-fluent-plugin-redis-slowlog"

data/lib/fluent/plugin/in_redis_slowlog.rb

@@ -6,6 +6,8 @@ module Fluent
     class RedisSlowlogInput < Fluent::Plugin::Input
       Fluent::Plugin.register_input("redis_slowlog", self)
 
+      Entry = Struct.new(:id, :timestamp, :exec_time_us, :command)
+
       config_param :tag,      :string
       config_param :url,      :string,  default: nil
       config_param :path,     :string,  default: nil
@@ -15,12 +17,6 @@ module Fluent
       config_param :logsize,  :integer, default: 128
       config_param :interval, :integer, default: 10
 
-      def initialize
-        super
-
-        @current_log_id = 0
-      end
-
       def configure(conf)
         super
 
@@ -55,39 +51,50 @@ module Fluent
       private
 
       attr_reader :redis, :interval
-      attr_accessor :watcher, :watching, :current_log_id
+      attr_accessor :watcher, :watching
 
       def watch
+        # Check the current id of the slowlog, and start logging from there
+        current_log_id = get_slowlogs(1).first&.id || -1
+
         while watching
           sleep interval
 
-          self.current_log_id = output(current_log_id)
+          current_log_id = output(current_log_id)
         end
       end
 
-      def output(last_id = 0)
-        slowlogs = redis.slowlog("get", logsize)
+      def output(last_id)
+        slowlogs = get_slowlogs(logsize)
         return last_id if slowlogs.empty?
 
-        emit_slowlog(slowlogs)
+        # If the latest entry is smaller than what we last saw, redis was restarted
+        # Restart logging from the beginning.
+        last_id = -1 if slowlogs.first.id < last_id
+
+        emit_slowlog(slowlogs, last_id)
 
         # Return the id of the last slowlog entry we've logged
         # The first entry is the one that occurred last
-        slowlogs.first.first
+        slowlogs.first.id
       end
 
-      def emit_slowlog(slowlogs)
+      def emit_slowlog(slowlogs, last_id)
         slowlogs.reverse_each do |log|
           # Don't emit logs for entries we've already logged
-          next if log.first <= current_log_id
+          next if log.id <= last_id
 
-          log_hash = { "id" => log[0],
-                       "timestamp" => Time.at(log[1]),
-                       "exec_time" => log[2],
-                       "command" => log[3] }
-          router.emit(tag, Time.now.to_i, log_hash)
+          log_hash = { "id" => log.id,
+                       "time" => Time.at(log.timestamp.to_i).utc.iso8601(3),
+                       "exec_time" => log.exec_time_us,
+                       "command" => log.command }
+          router.emit(tag, Fluent::EventTime.new(log.timestamp.to_i), log_hash)
         end
       end
+
+      def get_slowlogs(size)
+        redis.slowlog("get", size).map { |slowlog_entry| Entry.new(*slowlog_entry.first(4)) }
+      end
     end
   end
 end

data/spec/fluent/plugin/redis_slowlog_input_spec.rb

@@ -6,21 +6,24 @@ describe Fluent::Plugin::RedisSlowlogInput do
   end
 
   let(:plugin) { driver.instance }
-  let(:slowlog) { [] }
-  let(:fake_redis) { instance_double(Redis, ping: "PONG", quit: "OK", slowlog: slowlog) }
+  let(:fake_redis) { instance_double(Redis, ping: "PONG", quit: "OK", slowlog: []) }
 
   subject(:driver) do
     config_string = config.map { |key, value| "#{key} #{value}" }.join("\n")
     Fluent::Test::Driver::Input.new(described_class).configure(config_string)
   end
 
+  def run(**options)
+    driver.run({ timeout: 1 }.merge(options))
+  end
+
   before do
     Fluent::Test.setup
   end
 
   context "when redis can't be reached" do
     it "raises an error" do
-      expect { driver.run }.to raise_error(Redis::CannotConnectError)
+      expect { run }.to raise_error(Redis::CannotConnectError)
     end
   end
 
@@ -35,11 +38,12 @@ describe Fluent::Plugin::RedisSlowlogInput do
     end
 
     it "delegates all of them to redis-rb" do
-      redis_params = [:url, :path, :host, :port, :password]
+      # In ruby 2.5+ this could be config.slice(*[])
+      redis_params = config.select { |name, _| [:url, :path, :host, :port, :password].include?(name) }
 
-      expect(Redis).to receive(:new).with(config.slice(*redis_params)).and_return(fake_redis)
+      expect(Redis).to receive(:new).with(redis_params).and_return(fake_redis)
 
-      driver.run
+      run
     end
   end
 
@@ -62,10 +66,11 @@ describe Fluent::Plugin::RedisSlowlogInput do
     end
 
     it "does not raise errors" do
-      expect { driver.run }.not_to raise_error
+      expect { run }.not_to raise_error
     end
 
-    it "polls the slowlog at with the configured interval and size" do
+    it "polls the slowlog with the configured interval and size" do
+      expect(fake_redis).to receive(:slowlog).with("get", 1).ordered
       expect(plugin).to receive(:sleep).with(0).ordered
       expect(fake_redis).to receive(:slowlog).with("get", 10).ordered
       expect(plugin).to receive(:sleep).with(0).ordered
@@ -74,45 +79,77 @@ describe Fluent::Plugin::RedisSlowlogInput do
       # Limit to 2 cycles
       allow(plugin).to receive(:watching).thrice.and_return(true, true, false)
 
-      driver.run
+      run
     end
 
-    context "when the slowlog returns entries" do
-      let(:slowlog) do
+    context "when the slowlog returns entries", :aggregate_failures do
+      let(:startup_slowlog) { [[25637, 1590522108, 5, %w[SCAN 0]]] }
+      let(:logged_slowlog) do
         [
           [25640, 1590522258, 1, %w[ping]],
           [25639, 1590522249, 1, %w[ping]],
           [25638, 1590522208, 5, %w[SCAN 0]]
         ]
       end
+      let(:slowlog) { logged_slowlog + startup_slowlog }
 
       let(:expected_entries) do
-        slowlog.map(&method(:log_entry)).sort_by { |event| event["id"] }
+        logged_slowlog.map(&method(:log_entry)).sort_by { |event| event["id"] }
       end
 
       let(:emitted_entries) { driver.events.map(&:last) }
 
       it "emits an event for each slowlog in reverse order" do
-        driver.run(expect_emits: 3)
+        expect(fake_redis).to receive(:slowlog).and_return(startup_slowlog, slowlog)
+
+        run(expect_emits: 3)
 
         expect(driver.events.size).to eq(3)
         expect(emitted_entries).to eq(expected_entries)
       end
 
       it "does not log the same event twice" do
-        expect(fake_redis).to receive(:slowlog).and_return(slowlog.last(1), slowlog)
+        expect(fake_redis).to receive(:slowlog).and_return(startup_slowlog, slowlog.last(2), slowlog)
 
-        driver.run(expect_emits: 3)
+        run(expect_emits: 3)
 
         expect(driver.events.size).to eq(3)
         expect(emitted_entries).to eq(expected_entries)
       end
+
+      it "emits results that can be streamed to fluent using MessagePack" do
+        expect(fake_redis).to receive(:slowlog).and_return(startup_slowlog, slowlog)
+
+        run(expect_emits: 3)
+
+        expect { driver.events.map(&:to_msgpack) }.not_to raise_error
+      end
+
+      context "when redis restarts" do
+        let(:logged_slowlog) { [[25638, 1590522208, 5, %w[SCAN 0]]] }
+        let(:reset_slowlog) do
+          [
+            [2, 1590522258, 1, %w[ping]],
+            [1, 1590522249, 1, %w[ping]]
+          ]
+        end
+
+        it "emits all items" do
+          reset_entries = reset_slowlog.map(&method(:log_entry)).sort_by { |e| e["id"] }
+          expect(fake_redis).to receive(:slowlog).and_return(startup_slowlog, slowlog, [], reset_slowlog)
+
+          run(expect_emits: 3)
+
+          expect(emitted_entries.size).to eq(3)
+          expect(emitted_entries).to eq(expected_entries + reset_entries)
+        end
+      end
     end
   end
 
   def log_entry(slowlog_entry)
     { "id" => slowlog_entry.first,
-      "timestamp" => Time.at(slowlog_entry[1]),
+      "time" => Time.at(slowlog_entry[1].to_i).utc.iso8601(3),
       "exec_time" => slowlog_entry[2],
       "command" => slowlog_entry.last }
   end

data/tools/deploy-rubygem.sh

@@ -27,5 +27,5 @@ set -x
 rm -rf pkg/
 mkdir -p pkg/
 bundle install
-bundle exec rake verify build
+bundle exec rake build
 gem push pkg/*.gem

data/tools/update-changelog.rb

@@ -26,8 +26,8 @@ def gitlab_client(path, method, body = nil)
               raise "Unknown method: #{method}"
             end
 
-  request["content-type"] = "application/json"
-  request["PRIVATE-TOKEN"] = ENV["GITLAB_TOKEN"]
+  request["Content-Type"] = "application/json"
+  request["Private-Token"] = ENV["GITLAB_TOKEN"]
   request.body = JSON.dump(body) if body
 
   response = http.request(request)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-fluent-plugin-redis-slowlog
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Bob Van Landuyt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-28 00:00:00.000000000 Z
+date: 2020-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -145,7 +145,6 @@ files:
 - ".gitlab-ci.yml"
 - ".rubocop.yml"
 - Gemfile
-- Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
@@ -168,7 +167,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/Gemfile.lock

@@ -1,103 +0,0 @@
-PATH
-  remote: .
-  specs:
-    gitlab-fluent-plugin-redis-slowlog (0.0.0)
-      fluentd (>= 0.14.10, < 2)
-      redis (>= 4.1.3, < 5)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    ast (2.4.0)
-    coderay (1.1.2)
-    concurrent-ruby (1.1.6)
-    cool.io (1.6.0)
-    diff-lcs (1.3)
-    fluentd (1.10.4)
-      cool.io (>= 1.4.5, < 2.0.0)
-      http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.0.4, < 3.0.0)
-      sigdump (~> 0.2.2)
-      strptime (>= 0.2.2, < 1.0.0)
-      tzinfo (>= 1.0, < 3.0)
-      tzinfo-data (~> 1.0)
-      yajl-ruby (~> 1.0)
-    gitlab-styles (3.2.0)
-      rubocop (~> 0.74.0)
-      rubocop-gitlab-security (~> 0.1.0)
-      rubocop-performance (~> 1.4.1)
-      rubocop-rails (~> 2.0)
-      rubocop-rspec (~> 1.36)
-    http_parser.rb (0.6.0)
-    jaro_winkler (1.5.4)
-    method_source (0.9.2)
-    msgpack (1.3.3)
-    parallel (1.19.1)
-    parser (2.7.1.2)
-      ast (~> 2.4.0)
-    power_assert (1.1.3)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    rack (2.2.2)
-    rainbow (3.0.0)
-    rake (12.3.3)
-    redis (4.1.4)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
-    rubocop (0.74.0)
-      jaro_winkler (~> 1.5.1)
-      parallel (~> 1.10)
-      parser (>= 2.6)
-      rainbow (>= 2.2.2, < 4.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 1.7)
-    rubocop-gitlab-security (0.1.1)
-      rubocop (>= 0.51)
-    rubocop-performance (1.4.1)
-      rubocop (>= 0.71.0)
-    rubocop-rails (2.4.0)
-      rack (>= 1.1)
-      rubocop (>= 0.72.0)
-    rubocop-rspec (1.37.0)
-      rubocop (>= 0.68.1)
-    ruby-progressbar (1.10.1)
-    serverengine (2.2.1)
-      sigdump (~> 0.2.2)
-    sigdump (0.2.4)
-    strptime (0.2.4)
-    test-unit (3.2.9)
-      power_assert
-    tzinfo (2.0.2)
-      concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2020.1)
-      tzinfo (>= 1.0.0)
-    unicode-display_width (1.6.1)
-    yajl-ruby (1.4.1)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler (~> 2.1.4)
-  gitlab-fluent-plugin-redis-slowlog!
-  gitlab-styles (~> 3.2.0)
-  pry (~> 0.12.2)
-  rake (~> 12.0)
-  rspec (~> 3.9.0)
-  test-unit (~> 3.2.9)
-
-BUNDLED WITH
-   2.1.4