redis-cleaner 0.0.1

data/.gitignore

@@ -0,0 +1,6 @@
+.DS_Store
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.rvmrc

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 1.9.2
+  - 1.9.3
+  - 2.0.0

data/Gemfile

@@ -0,0 +1,8 @@
+source "http://rubygems.org"
+
+gemspec
+
+group :development, :test do
+  gem 'rake'
+  gem 'rspec'
+end

data/README.md

@@ -0,0 +1,57 @@
+Redis-Cleaner [![Build Status](https://travis-ci.org/lloydmeta/redis-cleaner.png?branch=master)](https://travis-ci.org/lloydmeta/redis-cleaner) [![Code Climate](https://codeclimate.com/github/lloydmeta/redis-cleaner.png)](https://codeclimate.com/github/lloydmeta/redis-cleaner)
+-------------
+
+A simple way of cleaning up a large number of Redis keys via [pattern matching](http://redis.io/commands/keys)
+
+Example Usage
+=========
+
+Instantiating a redis_cleaner
+
+```ruby
+# Timeout needs to be set, because KEYS is run on the Redis-server
+# side, which is potentially slow -> O(n) where n is the number of keys
+redis_config = {
+  host: "127.0.0.1",
+  port: 6390,
+  timeout: 60
+}
+
+redis_cleaner = RedisKeyCleaner.new(Redis.new(redis_config), "./borked_keys")
+```
+
+Separate dumping and cleaning
+
+```ruby
+redis_cleaner.dump_matching_keys_to_temp_file("resque:resque-retry*") #<-- can be skipped if you already have a file to read from
+cleanup_job_stats = redis_cleaner.delete_keys_in_temp_file(verbose: false)
+puts "Deleted #{cleanup_job_stats[:deleted_keys_count]} keys out of #{cleanup_job_stats[:total_keys_count]}"
+```
+
+Do everything in one go
+
+```ruby
+redis_cleaner.dump_matching_keys_to_temp_file("resque:resque-retry*", delete_temp_file: false, verbose: false, batch_size: 200)
+```
+
+## License
+
+Copyright (c) 2013 by Lloyd Chan
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, and to permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,8 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new do |t|
+end
+
+task :default  => :spec

data/lib/redis_cleaner.rb

@@ -0,0 +1,83 @@
+#encoding: utf-8
+
+require 'rubygems'
+require 'bundler/setup'
+
+class RedisCleaner
+
+  attr_accessor :redis, :temp_file_path
+
+  def initialize(redis, temp_file_path)
+    self.redis = redis
+    self.temp_file_path = temp_file_path
+  end
+
+  # returns number of keys deleted if
+  # everything was succesfully deleted otherwise
+  # return false
+  def remove_from_redis(keys_group = [])
+    if redis.del(keys_group) == keys_group.size
+      keys_group.size
+    else
+      false
+    end
+  end
+
+  # def delete_in_groups
+  # end
+  # deleted_keys_count = 0
+  # total_keys_count = 0
+
+  # takes params inside a hash
+  #   :batch_size defaults to 20
+  #   :verbose defaults to true
+  def delete_keys_in_temp_file(params = {})
+    batch_size = params.fetch(:batch_size, 20)
+    verbose = params.fetch(:verbose, true)
+
+    total_keys_count = 0
+    deleted_keys_count = 0
+
+    File.foreach(temp_file_path).each_slice(batch_size) do |keys_group|
+      total_keys_count = total_keys_count + keys_group.size
+      if removed_keys_count = remove_from_redis(strip_keys_group(keys_group))
+        puts "Successfully deleted key group of size #{keys_group.size}" if verbose
+        deleted_keys_count = deleted_keys_count + removed_keys_count
+      else
+        puts "Failed to delete: #{keys_group.inspect}" if verbose
+      end
+    end
+
+    {total_keys_count: total_keys_count, deleted_keys_count: deleted_keys_count}
+  end
+
+  def dump_matching_keys_to_temp_file(pattern)
+    resque_keys = redis.keys(pattern)
+    File.open(temp_file_path, "w+") do |file|
+      resque_keys.each do |key|
+        file.puts key
+      end
+    end
+  end
+
+  def dump_and_delete(pattern, params = {})
+    verbose = params.fetch(:verbose, true)
+    delete_temp_file = params.fetch(:delete_temp_file, true)
+
+    puts "Dumping ..."
+    dump_matching_keys_to_temp_file(pattern)
+    puts "Dumping finished !"
+    puts "Deleting dumped keys ..."
+    cleanup_job_stats = delete_keys_in_temp_file(params)
+
+    File.delete(temp_file_path) if delete_temp_file
+    puts "Deleted #{cleanup_job_stats[:deleted_keys_count]} keys out of #{cleanup_job_stats[:total_keys_count]}"
+  end
+
+  private
+
+    def strip_keys_group(keys_group = [])
+      keys_group.map{|element| element.strip}
+    end
+
+end

data/redis-cleaner.gemspec

@@ -0,0 +1,18 @@
+Gem::Specification.new do |gem|
+    gem.name        = %q{redis-cleaner}
+    gem.version = "0.0.1"
+    gem.date = %q{2013-05-07}
+    gem.authors     = ["Lloyd Meta"]
+    gem.email       = ["lloydmeta@gmail.com"]
+    gem.homepage    = "http://github.com/lloydmeta/redis-cleaner"
+    gem.description = %q{A Ruby gem for cleaning up of Redis keys by pattern matching. Handles huge amounts of keys. Can be used with any Redis client that responds to #del and #keys}
+    gem.summary     = gem.description
+
+    gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+    gem.files         = `git ls-files`.split("\n")
+    gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+    gem.require_paths = ["lib"]
+
+    gem.add_development_dependency 'rake'
+    gem.add_development_dependency 'rspec'
+end

data/spec/redis-cleaner/redis_cleaner_spec.rb

@@ -0,0 +1,116 @@
+require 'spec_helper'
+require 'tempfile'
+
+describe RedisCleaner do
+
+  let(:fake_keys){%w[key1 key2 key3 key4 key5 a_key1 a_key2]}
+  let(:redis_mock){
+    r = double("Redis mock", :keys => fake_keys)
+    r.stub(:del) do |arg|
+      arg.size
+    end
+    r
+  }
+  let(:temp_file){Tempfile.new('test_file')}
+  let(:temp_file_path){temp_file.path}
+  let(:redis_cleaner){RedisCleaner.new(redis_mock, temp_file_path)}
+
+  after(:all) do
+    temp_file.unlink
+  end
+
+  describe "initialisation" do
+
+    it "should not error out" do
+      expect{
+        redis_cleaner = RedisCleaner.new(redis_mock, temp_file_path)
+        }.to_not raise_error
+    end
+
+  end
+
+  describe "#remove_from_redis" do
+
+    it "should receive key_group as an argument" do
+      redis_cleaner.should_receive(:remove_from_redis).with(fake_keys)
+      redis_cleaner.remove_from_redis(fake_keys)
+    end
+
+    it "should return the number of keys deleted if everything was deleted successfully" do
+      redis_cleaner.remove_from_redis(fake_keys).should eq(fake_keys.size)
+    end
+
+    it "should return false if the number of keys deleted does not equal the number of keys passed in" do
+      redis_mock.stub(:del) do |arg|
+        arg.size - 1
+      end
+      redis_cleaner.remove_from_redis(fake_keys).should be_false
+    end
+
+  end
+
+  describe "#dump_matching_keys_to_temp_file" do
+
+    it "should receive a pattern as an argument" do
+      pattern = "asdf1234"
+      redis_cleaner.should_receive(:dump_matching_keys_to_temp_file).with(pattern)
+      redis_cleaner.dump_matching_keys_to_temp_file(pattern)
+    end
+
+    it "should call #keys on the redis object" do
+      pattern = "asdf1234"
+      redis_mock.should_receive(:keys).with(pattern)
+      redis_cleaner.dump_matching_keys_to_temp_file(pattern)
+    end
+
+    it "should dump the keys into the temp file" do
+      pattern = "asdf1234"
+      redis_cleaner.dump_matching_keys_to_temp_file(pattern)
+      temp_file_array = File.readlines(temp_file_path)
+      temp_file_array.zip(fake_keys).each do |(t, f)|
+        t.strip.should eq f
+      end
+    end
+
+  end
+
+  describe "#delete_keys_in_temp_file" do
+
+    before(:each) do
+      redis_cleaner.dump_matching_keys_to_temp_file("bleh")
+    end
+
+    it "should call #remove_from_redis" do
+      redis_cleaner.should_receive(:remove_from_redis)
+      redis_cleaner.delete_keys_in_temp_file(verbose: false)
+    end
+
+  end
+
+  describe "#dump_and_delete" do
+
+    let(:pattern){"lala"}
+    let(:fake_params){{asdf: 3, verbose: false}}
+
+    before(:each) do
+      redis_cleaner.stub(:delete_keys_in_temp_file).and_return({deleted_keys_count: fake_keys.size, total_keys_count: fake_keys.size})
+    end
+
+    it "should call receive pattern and params as arguments" do
+      redis_cleaner.should_receive(:dump_and_delete).with(pattern, fake_params)
+      redis_cleaner.dump_and_delete(pattern, fake_params)
+    end
+
+    it "should call #dump_matching_keys_to_temp_file" do
+      redis_cleaner.should_receive(:dump_matching_keys_to_temp_file)
+      redis_cleaner.dump_and_delete(pattern, fake_params)
+    end
+
+    it "should call #delete_keys_in_temp_file" do
+      redis_cleaner.should_receive(:delete_keys_in_temp_file)
+      redis_cleaner.dump_and_delete(pattern, fake_params)
+    end
+
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+require 'bundler/setup'
+require 'rspec'
+require 'rspec/mocks'
+require 'redis_cleaner'
+
+Dir[File.expand_path('../support/**/*', __FILE__)].each { |f| require f }
+
+RSpec.configure do |config|
+
+  #nothing yet
+
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: redis-cleaner
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Lloyd Meta
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ! 'A Ruby gem for cleaning up of Redis keys by pattern matching. Handles
+  huge amounts of keys. Can be used with any Redis client that responds to #del and
+  #keys'
+email:
+- lloydmeta@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- Gemfile
+- README.md
+- Rakefile
+- lib/redis_cleaner.rb
+- redis-cleaner.gemspec
+- spec/redis-cleaner/redis_cleaner_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/lloydmeta/redis-cleaner
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: ! 'A Ruby gem for cleaning up of Redis keys by pattern matching. Handles
+  huge amounts of keys. Can be used with any Redis client that responds to #del and
+  #keys'
+test_files:
+- spec/redis-cleaner/redis_cleaner_spec.rb
+- spec/spec_helper.rb