prune_cloudfiles_db_backups 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+out/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - '2.0.0'
+  - 1.9.3

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in prune_cloudfiles_db_backups.gemspec
+gemspec
+
+group :development do
+  gem 'parallel'
+end
+

data/LICENSE.txt

@@ -0,0 +1,7 @@
+Copyright (C) 2013 SmartReceipt, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,35 @@
+# PruneCfDbBackups
+
+[![Code Climate](https://codeclimate.com/github/SmartReceipt/prune_cloudfiles_db_backups.png)](https://codeclimate.com/github/SmartReceipt/prune_cloudfiles_db_backups)
+
+This gem helps prune useless database backups from Cloudfiles.
+
+The current backup retention defaults enforced by this tool are:
+
+* Keep hourly files for two weeks.
+* Keep weekly backups for 8 weeks on the Sundays.
+* Keep monthy backups for 12 months on the Sundays.
+
+## Usage
+
+To see the usage prompt, run:
+
+    prune_cloudfiles_db_backups --help
+
+The first deletion run, if not run for a year, will take a very long time.
+
+## Installation
+
+    $ gem install prune_cloudfiles_db_backups
+
+Or stick it in bundler, a chef recipe, or whatever!
+
+## Usage
+
+To see the usage prompt, run:
+
+    prune_cloudfiles_db_backups --help
+
+## Development Tools
+
+There is a `populate_dummy_container` ruby script inside the `test-util` folder. Also run it with `-h` to see the help banner. This tool is to aid in copying the object names in a container to a test container for manual integration testing. It does not copy the content and only copies the object names.

data/Rakefile

@@ -0,0 +1,5 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+task default: :spec

data/bin/prune_cloudfiles_db_backups

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+
+require 'prune_cloudfiles_db_backups/cli'
+
+PruneCloudfilesDbBackups::CLI.start

data/lib/prune_cloudfiles_db_backups/backup.rb

@@ -0,0 +1,66 @@
+require 'date'
+
+module PruneCloudfilesDbBackups
+  class Backup
+    attr_reader :daily, :weekly, :monthly
+    attr_reader :datetime
+    attr_reader :objects
+    attr_reader :name
+
+    # Creates a new instance of backup with a grouped set of related files
+    # @param [Hash] options in the form of a hash
+    # @option options [Set] :objects
+    # @option options [String] :name is optional and will be gleaned from objects if not specified
+    # @option options [DateTime] :date is optional and will be gleaned from objects if not specified
+    # @option options [Boolean] :monthly
+    # @option options [Boolean] :weekly
+    # @option options [Boolean] :daily
+    # @return [Backup]
+    def initialize(options = {})
+      @objects = options[:objects] || (raise ArgumentError, 'Objects not specified')
+      @name = options[:name] || self.class.parse_for_name(@objects)
+      @datetime = options[:datetime] || self.class.parse_for_datetime(@objects)
+      @monthly = options[:monthly]
+      @weekly = options[:weekly]
+      @daily = options[:daily]
+    end
+
+    # @param [Enumerable] objects
+    def self.parse_for_name(objects)
+      objects.first[/(^.+\.pgdump).*/, 1]
+    end
+
+    # @param [Enumerable] objects
+    def self.parse_for_datetime(objects)
+      str_date = objects.first[/\d{14}/]
+      DateTime.strptime(str_date, '%Y%m%d%H%M%S')
+    end
+
+    def dbname
+      @name[/(.+)-\d{14}/, 1]
+    end
+
+    def deletable?
+      !(@daily|@weekly|@monthly)
+    end
+
+    # @param [Backup] other_backup
+    def ==(other_backup)
+      self.objects == other_backup.objects
+    end
+
+    #noinspection RubyResolve
+    def to_s
+      mwd = '['
+      mwd << 'd' if @daily
+      mwd << '-' unless @daily
+      mwd << 'w' if @weekly
+      mwd << '-' unless @weekly
+      mwd << 'm' if @monthly
+      mwd << '-' unless @monthly
+      mwd << ']'
+
+      "#{mwd} #{dbname} #{@datetime.rfc822} (#{objects.size} item#{'s' if @objects.size > 1})"
+    end
+  end
+end

data/lib/prune_cloudfiles_db_backups/cli.rb

@@ -0,0 +1,43 @@
+require 'slop'
+require 'prune_cloudfiles_db_backups/pruner'
+require 'ruby-progressbar'
+
+
+module PruneCloudfilesDbBackups
+  class CLI
+    def self.start
+      opts = Slop.parse(strict: true, help: true) do
+        banner 'Usage: prune_cloudfiles_db_backups [options]'
+        on :u, :user=, 'Rackspace user to use. Default: ENV["N_RACKSPACE_API_USERNAME"]', default: ENV['N_RACKSPACE_API_USERNAME']
+        on :k, :key=, 'Rackspace key to use. Default: ENV["N_RACKSPACE_API_KEY"]', as: String, default: ENV['N_RACKSPACE_API_KEY']
+        on :authurl=, 'Auth URL Default: https://identity.api.rackspacecloud.com/v1.0', default: 'https://identity.api.rackspacecloud.com/v1.0'
+        on :c, :container=, 'Rackspace container to prune. Default: dummy_container', default: 'dummy_container'
+        on :allow_deletion, 'WARNING: Actually delete files from Rackspace Cloud. Without this option, only a listing of files to be deleted are given.'
+        on :d, :daily, 'Set daily retention amount. Default: 14', default: 14, as: Integer
+        on :w, :weekly, 'Set weekly retention amount. Default: 12', default: 12, as: Integer
+        on :m, :monthly, 'Set monthly retention amount. Default: 8', default: 8, as: Integer
+      end
+
+      pruner = PruneCloudfilesDbBackups::Pruner.new(opts)
+
+      puts pruner.date_sorted_delete_list.map { |backup| "Delete: #{backup.to_s}" }
+      puts pruner.date_sorted_keep_list.map { |backup| "Keep: #{backup.to_s}" }
+
+      if opts[:allow_deletion]
+        puts "Commencing deletion in #{opts[:container]}!"
+        progress = ProgressBar.create(title: 'Objects for deletion deleted',
+                                      total: pruner.list_to_delete.size,
+                                      format: '%e %a |%b>%i| %p%% %t'
+        )
+        pruner.delete! do |_|
+          progress.increment
+        end
+        progress.finish
+        puts 'Deletion completed.'
+      else
+        puts "Dry run on #{opts[:container]}. No files deleted. Run with --allow_deletion to commence deletion."
+      end
+
+    end
+  end
+end

data/lib/prune_cloudfiles_db_backups/pruner.rb

@@ -0,0 +1,49 @@
+require 'openstack'
+require 'logger'
+require 'prune_cloudfiles_db_backups/retention_calculator'
+
+module PruneCloudfilesDbBackups
+  class Pruner
+    attr_reader :list_to_keep, :list_to_delete
+
+    def initialize(opts = {})
+      cf = OpenStack::Connection.create(username: opts[:user],
+                                      api_key: opts[:key],
+                                      auth_url: 'https://identity.api.rackspacecloud.com/v1.0',
+                                      service_type:'object-store')
+      @container = cf.container(opts[:container])
+
+      calc = RetentionCalculator.new(@container.objects,
+                                     opts[:daily],
+                                     opts[:weekly],
+                                     opts[:monthly]
+      )
+      @list_to_delete = calc.list_to_delete
+      @list_to_keep = calc.list_to_keep
+    end
+
+    def delete!(&block)
+      @list_to_delete.map do |backup|
+        if block
+          block.call(backup)
+        end
+        backup.objects.map do |object|
+          begin
+            @container.delete_object(object)
+          rescue OpenStack::Exception::ItemNotFound
+            # ignored
+          end
+        end
+      end
+    end
+
+    def date_sorted_keep_list
+      @list_to_keep.sort_by {|a| a.datetime}
+    end
+
+    def date_sorted_delete_list
+      @list_to_delete.sort_by {|a| a.datetime}
+    end
+
+  end
+end

data/lib/prune_cloudfiles_db_backups/retention_calculator.rb

@@ -0,0 +1,86 @@
+require 'active_support/time'
+require 'prune_cloudfiles_db_backups/backup'
+require 'set'
+
+module PruneCloudfilesDbBackups
+  class RetentionCalculator
+
+    # Initialize the Retention Calculator with a list of cloudfile
+    # container objects
+    # @param [Array[String]] objects from cloudfiles
+    # @param [Integer] day_retention
+    # @param [Integer] week_retention
+    # @param [Integer] month_retention
+    def initialize(objects,
+        day_retention = 14,
+        week_retention = 8,
+        month_retention = 12)
+      @day_retention = day_retention || 14
+      @week_retention = week_retention || 8
+      @month_retention = month_retention || 12
+
+      @now = DateTime.now
+
+      sets = objects.group_by do |o|
+        /(^.+\.pgdump).*/.match(o)[1]
+      end
+
+      @backup_sets = sets.map do |_, v|
+        objects = Set.new(v)
+        datetime = Backup.parse_for_datetime(objects)
+
+        daily = keep_daily_dates.include?(datetime.to_date)
+        weekly = keep_weekly_dates.include?(datetime.to_date)
+        monthly = keep_monthly_dates.include?(datetime.to_date)
+        daily = true if datetime.to_date > @now
+
+        Backup.new(objects: objects,
+                   datetime: datetime,
+                   daily: daily,
+                   weekly: weekly,
+                   monthly: monthly)
+      end.to_set
+
+    end
+
+    # Based off of http://www.infi.nl/blog/view/id/23/Backup_retention_script
+
+    # @return [Set[DateTime]] a calculated array of days to be kept
+    def keep_daily_dates
+      @daily_dates ||= (0...@day_retention).map do |i|
+        @now.at_midnight.advance(days: -i).to_date
+      end.to_set
+    end
+
+    # @return [Set[DateTime]] a calculated array of days to be kept
+    def keep_weekly_dates
+      @weekly_dates ||= (0...@week_retention).map do |i|
+        # Sunday on the last couple of weeks
+        @now.at_beginning_of_week(:sunday).advance(weeks: -i).to_date
+      end.to_set
+    end
+
+    # @return [Set[DateTime]] a calculated array of days to be kept
+    def keep_monthly_dates
+      @monthly_dates ||= (0...@month_retention).map do |i|
+        # First Sunday of every month
+        @now.at_beginning_of_month.advance(months: -i).advance(days: 6).beginning_of_week(:sunday).to_date
+      end.to_set
+    end
+
+    # @return [Set[Backup]] a set of files for deletion
+    def list_to_delete
+      @backup_sets.select do |set|
+        set.deletable?
+      end
+    end
+
+    # @return [Set[Backup]] a set of files that will be kept
+    def list_to_keep
+      @backup_sets.select do |set|
+        !set.deletable?
+      end
+    end
+
+  end
+end

data/lib/prune_cloudfiles_db_backups/version.rb

@@ -0,0 +1,3 @@
+module PruneCloudfilesDbBackups
+  VERSION = '0.0.1'
+end

data/prune_cloudfiles_db_backups.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'prune_cloudfiles_db_backups/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'prune_cloudfiles_db_backups'
+  spec.version       = PruneCloudfilesDbBackups::VERSION
+  spec.authors       = ['Nelson Chen']
+  spec.email         = %w(nelson.chen@receipt.com)
+  spec.description   = 'Prunes SmartReceipts DB Backups from Cloudfiles'
+  spec.summary       = spec.description
+  spec.homepage      = 'https://github.com/SmartReceipt/prune_cloudfiles_db_backups'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = %w(lib)
+
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+
+  spec.add_dependency 'openstack', '~> 1.0.9'
+  spec.add_dependency 'slop', '~> 3.4.4'
+  spec.add_dependency 'activesupport', '~> 3.2.13'
+  spec.add_dependency 'ruby-progressbar', '~> 1.0.2'
+end

data/spec/prune_cloudfiles_db_backups/backup_spec.rb

@@ -0,0 +1,64 @@
+require 'rspec'
+require 'prune_cloudfiles_db_backups/backup'
+
+module PruneCloudfilesDbBackups
+  describe Backup do
+    before(:each) do
+      @object_pile = %w{
+          reports_production-20120819000003.pgdump
+          reports_production-20120819000003.pgdump.000
+          reports_production-20120819000003.pgdump.001
+        }.to_set
+      @datetime = DateTime.strptime('20120819000003', '%Y%m%d%H%M%S')
+      @name = 'reports_production-20120819000003'
+    end
+
+    describe '#initialize' do
+      it 'is created with a hash' do
+        Backup.new({ objects: @object_pile,
+                     name: @name,
+                     datetime: @datetime,
+                     monthly: false,
+                     weekly: false,
+                     daily: false
+                   })
+      end
+    end
+
+    describe '#deletable?' do
+      it 'returns true if none of the daily, weekly, or monthly flags are
+ false' do
+        backup = Backup.new({ objects: @object_pile,
+                              name: @name,
+                              datetime: @datetime,
+                              monthly: false,
+                              weekly: false,
+                              daily: false
+                            })
+
+        backup.deletable?.should be_true
+      end
+
+      it 'returns false if a certain flag (say, monthly) is set' do
+        backup = Backup.new({ objects: @object_pile,
+                              name: @name,
+                              datetime: @datetime,
+                              monthly: true,
+                              weekly: false,
+                              daily: false
+                            })
+
+        backup.deletable?.should be_false
+      end
+    end
+
+    describe '#==' do
+      it 'allows an equality comparison of backups that checks each field' do
+        one = Backup.new(objects: @object_pile, name: @name, datetime: @datetime)
+        two= Backup.new(objects: @object_pile, name: @name, datetime: @datetime)
+        one.should == two
+      end
+    end
+
+  end
+end