pg_s3_dumper 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8b14600ad7c17cf7a10564e4072cebeedc7c7b3a
+  data.tar.gz: 3043449edb6e21f11cb1b015e708e8cca7675d4b
+SHA512:
+  metadata.gz: 85933b6c92087a99f1c491678eabebb2fbb7d50e2ccd349f9ee0a2666500d1cae553035ba34a32ce36581832afc18a83f268011bb6648e07117dfaaa5393a39c
+  data.tar.gz: 86f5b988f8fc4ba8d8998b0923ee8d89148496ae8dbb6c88064a38775e1aa668944c6e64ec6c56b6efa57f67749d08bed72a225cf39b58cf86858d3233c910dc

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/.envrc

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,13 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the [Contributor Covenant](http:contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/)

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in pg_s3_dumper.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Nicolas Goy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,40 @@
+# PG S3 Dumper
+
+This simple tool creates and expires postgresql backups on an S3 bucket.
+
+In this document, "PG S3 Dumper" will be refered simply as "Dumper".
+
+NOTE: This is alpha quality software, use at your own risks.
+
+WARNING: Ensure you create a bucket and prefix specifically for this tool, with
+proper IAM permissions to avoid dataloss in case of failure.
+
+## Installation
+
+Just install the gem.
+
+    $ gem install pg_s3_dumper
+
+## Usage
+
+### Configuration
+
+Dumper can be configured by command line arguments or environment variables.
+
+For usage information:
+
+    $ pg_s3_dumper -h
+
+## TODO
+
+- Make backup keep count configurable
+- Write tests
+
+
+## Contributing
+
+1. Fork it ( https://github.com/kuon/pg_s3_dumper/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "pg_s3_dumper"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/pg_s3_dumper

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "pg_s3_dumper"
+
+PgS3Dumper::CLI.start

data/lib/pg_s3_dumper.rb

@@ -0,0 +1,15 @@
+require "time"
+require "tempfile"
+require "aws-sdk"
+require "securerandom"
+require "active_support"
+require "active_support/core_ext"
+
+require "pg_s3_dumper/version"
+require "pg_s3_dumper/cli"
+require "pg_s3_dumper/dumper"
+require "pg_s3_dumper/backup"
+
+module PgS3Dumper
+  # Your code goes here...
+end

data/lib/pg_s3_dumper/backup.rb

@@ -0,0 +1,59 @@
+class Backup
+
+  def self.generate_id
+    SecureRandom.hex(32)
+  end
+
+  include Comparable
+
+  attr_reader :s3_object, :ts, :keep
+
+  def <=>(other)
+    ts <=> other.ts
+  end
+
+  def initialize(s3_object)
+    @s3_object = s3_object
+    @ts = Time.parse(s3_object.key.split('/').last)
+    @keep = false
+  end
+
+  def to_s
+    "id: #{short_id}, key: #{basename}"
+  end
+
+  def on_day?(date)
+    ts.strftime('%F') == date.strftime('%F')
+  end
+
+  def keep_on_day(date)
+    @keep = true if on_day?(date)
+  end
+
+  def prune
+    if keep
+      puts "Keeping backup: #{self}"
+    else
+      delete
+    end
+  end
+
+  def delete
+    puts "Deleting backup: #{self}"
+    s3_object.delete
+  end
+
+  def id
+    @id ||= s3_object.metadata['backup_id']
+  end
+
+  def short_id
+    id[0..8]
+  end
+
+  def basename
+    File.basename(s3_object.key)
+  end
+
+end
+

data/lib/pg_s3_dumper/cli.rb

@@ -0,0 +1,87 @@
+module PgS3Dumper
+  module CLI
+    module_function
+
+    def start
+      options = {}
+
+      OptionParser.new do |opts|
+        opts.banner = "Usage: pg_s3_dumper options"
+
+
+        opts.separator "Commands options:"
+        opts.on("-c", "--command=COMMAND", "Command to execute",
+               " - list - List all backups",
+               " - backup - Create a new backup",
+               " - cleanup - Delete all backups") do |v|
+          options[:command] = v.to_sym
+        end
+
+        opts.on("-p", "--[no-]prune", "Prune (delete) old backups.",
+                                "When pruning is off all backups are kept.",
+                                "When pruning is on, the following backups are kept:",
+                                " - all backups for today",
+                                " - 1 per day for the last week",
+                                " - 1 per week for the last month",
+                                " - 1 per month for the last year") do |v|
+          options[:prune] = v
+        end
+
+        opts.separator ""
+        opts.separator "Configuration options:"
+
+        opts.on("-d", "--database URL", "Database to use, must be an URL in the form:",
+                                        "'postgres://username:password@hostname:port/database',",
+                                        "it will be passed directly to the pg_dump command.",
+                                        "If not set, the environment variable DATABASE_URL is used.") do |v|
+          options[:database_url] = v
+        end
+
+        opts.on("-k", "--aws-key KEY", "AWS key, must have read write access to the bucket.",
+                                       "If not set, the environment variable AWS_ACCESS_KEY_ID is used.") do |v|
+          options[:aws_key] = v
+        end
+
+        opts.on("-w", "--aws-secret SECRET", "AWS secret key.",
+                                             "If not set, the environment variable AWS_SECRET_ACCESS_KEY is used.") do |v|
+          options[:aws_secret] = v
+        end
+
+        opts.on("-r", "--aws-region REGION", "AWS region your bucket resides in.",
+                                             "If not set, the environment variable AWS_REGION is used.") do |v|
+          options[:aws_region] = v
+        end
+
+        opts.on("-u", "--aws-url URL", "AWS destination, must be an URL in the form:",
+                                             "'s3://bucket/prefix'.",
+                                             "If not set, the environment variable AWS_URL is used.") do |v|
+          options[:aws_url] = v
+        end
+
+        opts.separator ""
+        opts.separator "General options:"
+
+        opts.on("-v", "--version", "Output version information, then exit.") do
+          puts PgS3Dumper::VERSION
+          exit
+        end
+
+        opts.on("-h", "--help", "Show this help, then exit.") do
+          puts opts.help
+          exit
+        end
+
+
+      end.parse!
+
+      begin
+        dumper = PgS3Dumper::Dumper.new(options)
+        dumper.run(options[:command])
+      rescue PgS3Dumper::Error => e
+        puts "ERROR: #{e.message}."
+      end
+
+    end
+  end
+end
+

data/lib/pg_s3_dumper/dumper.rb

@@ -0,0 +1,140 @@
+require "aws-sdk"
+
+module PgS3Dumper
+  class Error < ::StandardError
+  end
+
+  class Dumper
+
+    attr_reader          :database_url,
+                         :database_name,
+                         :prefix,
+                         :bucket
+
+    def prune?
+      @prune
+    end
+
+    def initialize(options)
+      @database_url = options[:database_url] || ENV['DATABASE_URL']
+      aws_region = options[:aws_region] || ENV['AWS_REGION']
+      aws_key = options[:aws_key] || ENV['AWS_ACCESS_KEY_ID']
+      aws_secret = options[:aws_secret] || ENV['AWS_SECRET_ACCESS_KEY']
+      aws_url = options[:aws_url] || ENV['AWS_URL']
+      @prune = options.has_key?(:prune) ? options[:prune] : false
+
+      v = `pg_dump --version`
+      v =~ /pg_dump \(PostgreSQL\) 9\.\d\.\d/ or raise Error, "pg_dump version 9.x.x is required and must be in the PATH"
+      database_url or raise Error, "Database URL required."
+      aws_url =~ %r{^s3://([^/]+)/(\S*?)/?$} or raise Error, "Invalid AWS URL"
+
+      bucket_name = $1
+      @prefix = "#{$2}/"
+
+      database_url =~ %r{postgres://[^/]+/(.+)} or raise Error, "Invalid database URL"
+      @database_name = $1
+      @prefix = File.join(prefix, database_name)
+
+      aws_key && aws_secret or raise Error, "AWS key and secret required"
+
+      cred = Aws::Credentials.new(aws_key, aws_secret)
+      client = Aws::S3::Client.new(:credentials => cred, :region => aws_region)
+      s3 = Aws::S3::Resource.new(:client => client)
+      @bucket = s3.bucket(bucket_name)
+    end
+
+    def run(command)
+      case command
+      when :list
+        list
+      when :backup
+        backup
+      when :cleanup
+        cleanup
+      else
+        raise Error, "Invalid command '#{command}'"
+      end
+    end
+
+    def list
+      f = "% 12s% 30s"
+      puts f % ['id', 'date']
+      puts "-" * 42
+      find_backups.each do |b|
+        puts f % [b.short_id, b.ts]
+      end
+    end
+
+    def cleanup
+      find_backups.each do |b|
+        b.delete
+      end
+    end
+
+    def backup
+      backups = find_backups
+
+      # Keep daily backups for one week
+      # Keep weekly backups for one month
+      # Keep monthly backups for one year
+      days =
+      7.times.map{|i| i.days.ago} +
+      4.times.map{|i| i.weeks.ago} +
+      12.times.map{|i| i.months.ago}
+
+      days.each do |ts|
+        backups.each do |b|
+          b.keep_on_day(ts)
+          # We kept one backup on day ts, go to next day
+          break if b.keep
+        end
+      end
+
+      # Keep all backups for today
+      backups.each do |b|
+        b.keep_on_day(Date.today)
+      end
+
+      # Make new backup
+      key = "#{now.iso8601}-#{database_name.split('/').last}.dmp"
+      file = Tempfile.new(key)
+
+      puts "## Creating backup"
+
+      system "pg_dump -Fc --no-owner --no-acl -d #{database_url} > #{file.path}" or fail 'Cannot create dump'
+
+      obj = bucket.object(File.join(prefix, key))
+
+      obj.upload_file(file.path, :metadata => {:backup_id => Backup.generate_id})
+      file.close
+      file.unlink
+      bck = Backup.new(obj)
+      puts "Created backup: #{bck}"
+
+      if prune?
+        puts "## Pruning old backups"
+        backups.each do |b|
+          b.prune
+        end
+      end
+    end
+
+    def now
+      Time.now.utc
+    end
+
+    private
+    def find_backups
+      backups = []
+
+      # Collect existing backups
+      bucket.objects(:prefix => prefix).each do |o|
+        backups << Backup.new(o.object)
+      end
+
+      backups.sort
+    end
+
+  end
+end
+

data/lib/pg_s3_dumper/version.rb

@@ -0,0 +1,3 @@
+module PgS3Dumper
+  VERSION = "0.1.0"
+end

data/pg_s3_dumper.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pg_s3_dumper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "pg_s3_dumper"
+  spec.version       = PgS3Dumper::VERSION
+  spec.authors       = ["Nicolas Goy"]
+  spec.email         = ["kuon@goyman.com"]
+
+
+  spec.summary       = %q{Simple tool to dump postgresql database to an S3 bucket.}
+  spec.homepage      = "http://github.com/kuon/pg_s3_dumper"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "aws-sdk", "~> 2.0"
+  spec.add_dependency "activesupport", "~> 4.0"
+  spec.add_development_dependency "bundler", "~> 1.8"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification
+name: pg_s3_dumper
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Nicolas Goy
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-03-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: 
+email:
+- kuon@goyman.com
+executables:
+- pg_s3_dumper
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/pg_s3_dumper
+- lib/pg_s3_dumper.rb
+- lib/pg_s3_dumper/backup.rb
+- lib/pg_s3_dumper/cli.rb
+- lib/pg_s3_dumper/dumper.rb
+- lib/pg_s3_dumper/version.rb
+- pg_s3_dumper.gemspec
+homepage: http://github.com/kuon/pg_s3_dumper
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Simple tool to dump postgresql database to an S3 bucket.
+test_files: []