gitlab-fluent-plugin-redis-slowlog 0.0.1.pre → 0.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8104bc9262ab90f07e653d801a21ed7384f8d53a3bbc7644d3f568c9cd2a0be
-  data.tar.gz: cb9da92f38150d81e5927b0ffb9710de9eb9dfff1b9a5b9233fa42dc40e508c2
+  metadata.gz: 849d65a25ba670f60d324188cb543a9b5b618cde5683d27e95cecd862290bddc
+  data.tar.gz: 12557c95bfd6029bbe899c826dad3038e7d285e3b5fe1b7d0747a200d91319bc
 SHA512:
-  metadata.gz: ae18b72578831c8ba6d58a6c82df91c75a9c9dca51ed5c6546765afba1193090dc4583999fa30be0d508611e414cc1f5cffa1ebe3e6cc927434dede4ba06ec95
-  data.tar.gz: c33ebe9e818ea120eac6f900c62c1bc61c91fe5d9dde5a367b84f65f45895bc19c673337b184ffe3ca9f8f62c56d63dec9c227a2f6e7bc0e19d01dfabd8e9c9e
+  metadata.gz: e79d5c3a510092ce501cd67294eb2faaa4edd333bfb706d9508e2a3a8917d861ab6e87eeb1f4cf04e9d3d0698159911e1462e745258bc726a1881c2f9f62ba35
+  data.tar.gz: b4c8d26df8598d9700168b1eccf0316db60b3ca5864339c4c72bd927124180174d6de9c2723bd569702cdc7386ebde74bce9b7d54a5d02cf37dcaace20d152f6

data/.gitignore

@@ -1 +1,2 @@
 *.gem
+Gemfile.lock

data/.gitlab-ci.yml

@@ -0,0 +1,42 @@
+workflow:
+  rules:
+    # For merge requests, create a pipeline.
+    - if: '$CI_MERGE_REQUEST_IID'
+    # For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    # For tags, create a pipeline.
+    - if: '$CI_COMMIT_TAG'
+
+.test_template: &test_definition
+  stage: test
+  script:
+    - gem install bundler
+    - bundle install
+    - bundle exec rake
+
+test2.6:
+  image: ruby:2.6
+  <<: *test_definition
+test2.5:
+  image: ruby:2.5
+  <<: *test_definition
+test2.4:
+  image: ruby:2.4
+  <<: *test_definition
+
+deploy:
+  image: ruby:2.6
+  stage: deploy
+  script:
+    - gem install bundler
+    - tools/deploy-rubygem.sh
+  only:
+    - tags
+
+release:
+  image: ruby:2.6
+  stage: deploy
+  script:
+    - tools/update-changelog.rb "${CI_COMMIT_TAG}"
+  only:
+    - tags

data/.rubocop.yml

@@ -0,0 +1,17 @@
+inherit_gem:
+  gitlab-styles:
+    - rubocop-default.yml
+
+AllCops:
+  TargetRubyVersion: 2.4
+
+# Let's just pick something from the beginning
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+# This isn't Rails, so disable these
+Rails/RakeEnvironment:
+  Enabled: false
+CodeReuse/ActiveRecord:
+  Enabled: false

data/README.md

@@ -94,3 +94,39 @@ Default value: `128`.
 The time in seconds to wait between `SLOWLOG` commands.
 
 Default value: `10`.
+
+## Development
+
+After checking out the repository, run `bundle install` to install all
+dependencies.
+
+After that, you can run `bundle exec rake` to run all lints and
+specs.
+
+Other rake tasks:
+
+```
+rake build                # Build gitlab-fluent-plugin-redis-slowlog-0.gem into the pkg directory
+rake clean                # Remove any temporary products
+rake clobber              # Remove any generated files
+rake console              # Start an interactive console with the gem loaded
+rake install              # Build and install gitlab-fluent-plugin-redis-slowlog-0.gem into system gems
+rake install:local        # Build and install gitlab-fluent-plugin-redis-slowlog-0.gem into system gems without network access
+rake release[remote]      # Create tag v0 and build and push gitlab-fluent-plugin-redis-slowlog-0.gem to rubygems.org
+rake spec                 # Run RSpec code examples
+rake verify               # Run RuboCop
+rake verify:auto_correct  # Auto-correct RuboCop offenses
+```
+
+### Releasing a new version
+
+Releasing a new version can be done by pushing a new tag, or creating
+it from the
+[interface](https://gitlab.com/gitlab-org/fluent-plugin-redis-slowlog/-/tags).
+
+A new changelog will automatically be added to the release on GitLab.
+
+The new version will automatically be published to
+[rubygems](https://rubygems.org/gems/gitlab-fluent-plugin-redis-slowlog)
+when the pipeline for the tag completes. It might take a few minutes
+before the update is available.

data/Rakefile

@@ -1,12 +1,17 @@
 require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+require "rubocop/rake_task"
 
-require "rake/testtask"
+RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new(:verify)
 
-Rake::TestTask.new(:test) do |t|
-  t.libs.push("lib", "test")
-  t.test_files = FileList["test/**/test_*.rb"]
-  t.verbose = true
-  t.warning = true
+desc "Start an interactive console with the gem loaded"
+task :console do
+  require "pry"
+  require "fluent/env"
+  require "fluent/plugin/in_redis_slowlog"
+
+  Pry.start
 end
 
-task default: [:test]
+task default: [:verify, :spec]

data/gitlab-fluent-plugin-redis-slowlog.gemspec

@@ -3,12 +3,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "gitlab-fluent-plugin-redis-slowlog"
-  spec.version = "0.0.1.pre" # `git describe --tags`.chomp.gsub(/^v/, "")
+  spec.version = `git describe --tags`.chomp.gsub(/^v/, "")
   spec.metadata = { "source_code_uri" => "https://gitlab.com/gitlab-org/fluent-plugin-redis-slowlog" }
   spec.authors = ["Bob Van Landuyt"]
   spec.email   = ["bob@gitlab.com"]
 
-  spec.required_ruby_version = ">= 2.6"
+  spec.required_ruby_version = ">= 2.4"
 
   spec.summary       = "Emit redis slowlog entries into fluentd"
   spec.homepage      = "http://gitlab.com/gitlab-org/gitlab-fluent-plugin-redis-slowlog"

data/spec/fluent/plugin/redis_slowlog_input_spec.rb

@@ -0,0 +1,120 @@
+require "spec_helper"
+
+describe Fluent::Plugin::RedisSlowlogInput do
+  let(:config) do
+    { tag: "redis.slowlog" }
+  end
+
+  let(:plugin) { driver.instance }
+  let(:slowlog) { [] }
+  let(:fake_redis) { instance_double(Redis, ping: "PONG", quit: "OK", slowlog: slowlog) }
+
+  subject(:driver) do
+    config_string = config.map { |key, value| "#{key} #{value}" }.join("\n")
+    Fluent::Test::Driver::Input.new(described_class).configure(config_string)
+  end
+
+  before do
+    Fluent::Test.setup
+  end
+
+  context "when redis can't be reached" do
+    it "raises an error" do
+      expect { driver.run }.to raise_error(Redis::CannotConnectError)
+    end
+  end
+
+  context "when specifying redis connection attributes" do
+    let(:config) do
+      { tag: "redis.slowlog",
+        url: "redis://:p4ssw0rd@10.0.1.1:6380/15",
+        path: "/path/to/redis.sock",
+        host: "localhost",
+        port: 1234,
+        password: "5iveL!fe" }
+    end
+
+    it "delegates all of them to redis-rb" do
+      # In ruby 2.5+ this could be config.slice(*[])
+      redis_params = config.select { |name, _| [:url, :path, :host, :port, :password].include?(name) }
+
+      expect(Redis).to receive(:new).with(redis_params).and_return(fake_redis)
+
+      driver.run
+    end
+  end
+
+  context "when redis is available" do
+    let(:config) do
+      {
+        tag: "redis.slowlog",
+        interval: 0,
+        logsize: 10
+      }
+    end
+
+    before do
+      allow(plugin).to receive(:redis).and_return(fake_redis)
+    end
+
+    after do
+      # Wait for the thread polling redis to finish
+      plugin.__send__(:watcher).join
+    end
+
+    it "does not raise errors" do
+      expect { driver.run }.not_to raise_error
+    end
+
+    it "polls the slowlog with the configured interval and size" do
+      expect(plugin).to receive(:sleep).with(0).ordered
+      expect(fake_redis).to receive(:slowlog).with("get", 10).ordered
+      expect(plugin).to receive(:sleep).with(0).ordered
+      expect(fake_redis).to receive(:slowlog).with("get", 10).ordered
+
+      # Limit to 2 cycles
+      allow(plugin).to receive(:watching).thrice.and_return(true, true, false)
+
+      driver.run
+    end
+
+    context "when the slowlog returns entries" do
+      let(:slowlog) do
+        [
+          [25640, 1590522258, 1, %w[ping]],
+          [25639, 1590522249, 1, %w[ping]],
+          [25638, 1590522208, 5, %w[SCAN 0]]
+        ]
+      end
+
+      let(:expected_entries) do
+        slowlog.map(&method(:log_entry)).sort_by { |event| event["id"] }
+      end
+
+      let(:emitted_entries) { driver.events.map(&:last) }
+
+      it "emits an event for each slowlog in reverse order" do
+        driver.run(expect_emits: 3)
+
+        expect(driver.events.size).to eq(3)
+        expect(emitted_entries).to eq(expected_entries)
+      end
+
+      it "does not log the same event twice" do
+        expect(fake_redis).to receive(:slowlog).and_return(slowlog.last(1), slowlog)
+
+        driver.run(expect_emits: 3)
+
+        expect(driver.events.size).to eq(3)
+        expect(emitted_entries).to eq(expected_entries)
+      end
+    end
+  end
+
+  def log_entry(slowlog_entry)
+    { "id" => slowlog_entry.first,
+      "timestamp" => Time.at(slowlog_entry[1]),
+      "exec_time" => slowlog_entry[2],
+      "command" => slowlog_entry.last }
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+require "rspec"
+require "fluent/test"
+require "fluent/test/driver/input"
+require "fluent/test/helpers"
+require "pry"
+
+Dir.glob(File.expand_path("../lib/**/*.rb", __dir__)).each(&method(:require))
+
+# Test::Unit is required for fluent/test, but we don't want it to autorun when
+# required
+Test::Unit::AutoRunner.need_auto_run = false
+
+RSpec.configure do |config|
+  config.include Fluent::Test::Helpers
+end

data/tools/deploy-rubygem.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+IFS=$'\n\t'
+
+mkdir -p ~/.gem/
+temp_file=$(mktemp)
+
+# Revert change the credentials file
+function revert_gem_cred_switch {
+  mv "$temp_file" ~/.gem/credentials
+}
+
+if [[ -f ~/.gem/credentials ]]; then
+  echo "Temporarily moving existing credentials to $temp_file"
+  mv ~/.gem/credentials "$temp_file"
+  trap revert_gem_cred_switch EXIT
+fi
+
+cat <<EOD > ~/.gem/credentials
+---
+:rubygems_api_key: $RUBYGEMS_API_KEY
+EOD
+chmod 600 ~/.gem/credentials
+
+set -x
+rm -rf pkg/
+mkdir -p pkg/
+bundle install
+bundle exec rake build
+gem push pkg/*.gem

data/tools/update-changelog.rb

@@ -0,0 +1,106 @@
+#!/usr/bin/env ruby
+
+require "uri"
+require "net/http"
+require "openssl"
+require "json"
+require "optparse"
+
+GitlabError = Class.new(StandardError)
+GitlabClientError = Class.new(GitlabError)
+
+def gitlab_client(path, method, body = nil)
+  url = URI("https://gitlab.com#{path}")
+
+  http = Net::HTTP.new(url.host, url.port)
+  http.use_ssl = true
+
+  request = case method
+            when :get
+              Net::HTTP::Get.new(url)
+            when :post
+              Net::HTTP::Post.new(url)
+            when :put
+              Net::HTTP::Put.new(url)
+            else
+              raise "Unknown method: #{method}"
+            end
+
+  request["Content-Type"] = "application/json"
+  request["Private-Token"] = ENV["GITLAB_TOKEN"]
+  request.body = JSON.dump(body) if body
+
+  response = http.request(request)
+
+  case response
+  when Net::HTTPSuccess
+    JSON.parse(response.read_body)
+  when Net::HTTPClientError
+    raise GitlabClientError, "HTTP #{response.code} for #{method} #{url}: #{response.read_body}"
+  else
+    raise GitLabError, "HTTP #{response.code} for #{method} #{url}: #{response.read_body}"
+  end
+end
+
+def list_commits_for_tag(tag)
+  commits = `git rev-list --no-merges $(git describe --tags --abbrev=0 #{tag}^)..#{tag}`.lines
+  commits.map(&:chomp)
+end
+
+def get_mrs_for_commit(commit_id)
+  gitlab_client("/api/v4/projects/gitlab-org%2ffluent-plugin-redis-slowlog/repository/commits/#{commit_id}/merge_requests", :get)
+end
+
+def create_release_for_tag(tag, description)
+  req_body = { tag_name: tag, description: description }
+  gitlab_client("/api/v4/projects/gitlab-org%2ffluent-plugin-redis-slowlog/releases", :post, req_body)
+end
+
+def update_release_for_tag(tag, description)
+  req_body = { description: description }
+  gitlab_client("/api/v4/projects/gitlab-org%2ffluent-plugin-redis-slowlog/releases/#{tag}", :put, req_body)
+end
+
+def get_unique_mrs_for_tag(tag)
+  commit_ids = list_commits_for_tag(tag)
+  dup_mrs = commit_ids.map { |commit_id| get_mrs_for_commit(commit_id) }.flatten
+  uniq_mrs = dup_mrs.uniq { |mr| mr["iid"] }
+
+  uniq_mrs.sort { |mr| mr["iid"] }.reverse
+end
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: update-changelog.rb [options] tag"
+
+  opts.on("-d", "--[no-]dry-run", "Display only. Do not update release notes.") do |d|
+    options[:dry_run] = d
+  end
+end.parse!
+
+# Check required conditions
+unless ARGV.length == 1
+  puts optparse
+  exit(-1)
+end
+
+tag = ARGV.first
+
+changelog = get_unique_mrs_for_tag(tag).map { |mr| "* #{mr['web_url']}: #{mr['title']}" }.join("\n")
+
+raise "No new changes found" if changelog.empty?
+
+changelog = "## Changes in #{tag}\n\n" + changelog
+
+if options[:dry_run]
+  puts changelog
+  exit
+end
+
+begin
+  create_release_for_tag(tag, changelog)
+  puts "Created release: https://gitlab.com/gitlab-org/fluent-plugin-redis-slowlog/-/releases##{tag}"
+rescue GitlabClientError
+  update_release_for_tag(tag, changelog)
+  puts "Updated release: https://gitlab.com/gitlab-org/fluent-plugin-redis-slowlog/-/releases##{tag}"
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-fluent-plugin-redis-slowlog
 version: !ruby/object:Gem::Version
-  version: 0.0.1.pre
+  version: 0.0.1
 platform: ruby
 authors:
 - Bob Van Landuyt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-28 00:00:00.000000000 Z
+date: 2020-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -142,15 +142,18 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".gitlab-ci.yml"
+- ".rubocop.yml"
 - Gemfile
-- Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
 - gitlab-fluent-plugin-redis-slowlog.gemspec
 - lib/fluent/plugin/in_redis_slowlog.rb
-- test/helper.rb
-- test/plugin/test_in_redis_slowlog.rb
+- spec/fluent/plugin/redis_slowlog_input_spec.rb
+- spec/spec_helper.rb
+- tools/deploy-rubygem.sh
+- tools/update-changelog.rb
 homepage: http://gitlab.com/gitlab-org/gitlab-fluent-plugin-redis-slowlog
 licenses:
 - MIT
@@ -164,17 +167,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Emit redis slowlog entries into fluentd
 test_files:
-- test/helper.rb
-- test/plugin/test_in_redis_slowlog.rb
+- spec/fluent/plugin/redis_slowlog_input_spec.rb
+- spec/spec_helper.rb

data/Gemfile.lock

@@ -1,110 +0,0 @@
-PATH
-  remote: .
-  specs:
-    gitlab-fluent-plugin-redis-slowlog (0.0.1.pre)
-      fluentd (>= 0.14.10, < 2)
-      redis (>= 4.1.3, < 5)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    activesupport (3.2.22.5)
-      i18n (~> 0.6, >= 0.6.4)
-      multi_json (~> 1.0)
-    ast (2.4.0)
-    coderay (1.1.2)
-    concurrent-ruby (1.1.6)
-    cool.io (1.6.0)
-    diff-lcs (1.3)
-    fluentd (1.10.4)
-      cool.io (>= 1.4.5, < 2.0.0)
-      http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.0.4, < 3.0.0)
-      sigdump (~> 0.2.2)
-      strptime (>= 0.2.2, < 1.0.0)
-      tzinfo (>= 1.0, < 3.0)
-      tzinfo-data (~> 1.0)
-      yajl-ruby (~> 1.0)
-    gitlab-styles (3.2.0)
-      rubocop (~> 0.74.0)
-      rubocop-gitlab-security (~> 0.1.0)
-      rubocop-performance (~> 1.4.1)
-      rubocop-rails (~> 2.0)
-      rubocop-rspec (~> 1.36)
-    http_parser.rb (0.6.0)
-    i18n (0.9.5)
-      concurrent-ruby (~> 1.0)
-    jaro_winkler (1.5.4)
-    method_source (0.9.2)
-    msgpack (1.3.3)
-    multi_json (1.14.1)
-    parallel (1.19.1)
-    parser (2.7.1.3)
-      ast (~> 2.4.0)
-    power_assert (1.1.3)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    rack (2.2.2)
-    rainbow (3.0.0)
-    rake (12.3.3)
-    redis (4.1.4)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
-    rubocop (0.74.0)
-      jaro_winkler (~> 1.5.1)
-      parallel (~> 1.10)
-      parser (>= 2.6)
-      rainbow (>= 2.2.2, < 4.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 1.7)
-    rubocop-gitlab-security (0.1.1)
-      rubocop (>= 0.51)
-    rubocop-performance (1.4.1)
-      rubocop (>= 0.71.0)
-    rubocop-rails (2.5.2)
-      activesupport
-      rack (>= 1.1)
-      rubocop (>= 0.72.0)
-    rubocop-rspec (1.39.0)
-      rubocop (>= 0.68.1)
-    ruby-progressbar (1.10.1)
-    serverengine (2.2.1)
-      sigdump (~> 0.2.2)
-    sigdump (0.2.4)
-    strptime (0.2.4)
-    test-unit (3.2.9)
-      power_assert
-    tzinfo (2.0.2)
-      concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2020.1)
-      tzinfo (>= 1.0.0)
-    unicode-display_width (1.6.1)
-    yajl-ruby (1.4.1)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler (~> 2.1.4)
-  gitlab-fluent-plugin-redis-slowlog!
-  gitlab-styles (~> 3.2.0)
-  pry (~> 0.12.2)
-  rake (~> 12.0)
-  rspec (~> 3.9.0)
-  test-unit (~> 3.2.9)
-
-BUNDLED WITH
-   2.1.4

data/test/helper.rb

@@ -1,8 +0,0 @@
-$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
-require "test-unit"
-require "fluent/test"
-require "fluent/test/driver/input"
-require "fluent/test/helpers"
-
-Test::Unit::TestCase.include(Fluent::Test::Helpers)
-Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_in_redis_slowlog.rb

@@ -1,18 +0,0 @@
-require "helper"
-require "fluent/plugin/in_redis_slowlog.rb"
-
-class RedisSlowlogInputTest < Test::Unit::TestCase
-  setup do
-    Fluent::Test.setup
-  end
-
-  test "failure" do
-    flunk
-  end
-
-  private
-
-  def create_driver(conf)
-    Fluent::Test::Driver::Input.new(Fluent::Plugin::RedisSlowlogInput).configure(conf)
-  end
-end