s3ckup 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ab5c9438cd5dcd2044b8de936e70cd11c676d400
+  data.tar.gz: dd1d6f66846bcb0891bda487ea9645792e121b2b
+SHA512:
+  metadata.gz: 5159afdaace5641384d99967281fa705f551da0622b16f79e5b1388d9fe3bc0412c52806e217f63aa801a3dfa520d07da920c85d9dfc7d7039c05da2ebf4d747
+  data.tar.gz: 302145498e57f9c2c06417dae0cd91e4d59d90bd528c45cd7b85bc0ece2e8c994a8d16ce8ec0d2032e16cc5e5c14acaacef85daf67fb134a3abb99002d3df53c

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2012 Tiago Scolari
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,43 @@
+s3ckup
+====
+
+Ruby gem/command line tool for Incremental backuping to AWS S3.
+
+My motivation for this was to be used in a backup container for docker. But it's a command line that can be used in any
+platform.
+
+It will store a `manifest.yml` file in the root of the bucket, containing all
+file names and digests. On backup this file will be matched with a on time
+generated one, only updated/new files will be pushed to the bucket, and files
+that no longer exists will be deleted.
+
+There's no versioning, but you could use s3 built in versioning.
+
+INSTALLATION
+-----------
+
+```
+  gem install s3ckup
+```
+
+
+USAGE
+-----
+
+```
+Usage: s3ckup OPTIONS --folders folder1,folder2,...
+    -k, --aws-key-id KEY_ID          AWS key id
+    -a, --aws-access-key ACCESS_KEY  AWS access key
+    -b, --s3-bucket BUCKET_NAME      S3 bucket name
+    -r, --recursive                  Recursive mode
+    -v, --verbose                    Verbose
+    -f FOLDER1,FOLDER2,FOLDER3...,   Folder list, separated by ','
+        --folders
+```
+
+Example:
+
+```
+s3ckup -k $AWS_KEY -a $AWS_SECRET -b my_backup_bucket -r -f /myapp/uploads,/mydb/data
+```
+

data/Rakefile

@@ -0,0 +1,9 @@
+#!/usr/bin/env rake
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+Bundler::GemHelper.install_tasks
+

data/bin/s3ckup

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 's3ckup'
+require 's3ckup/command_line'
+S3ckup::CommandLine.new.run(ARGV)

data/lib/s3ckup.rb

@@ -0,0 +1,4 @@
+require 'rubygems'
+require_relative 's3ckup/manifest'
+require_relative 's3ckup/backup'
+require_relative 's3ckup/s3_manager'

data/lib/s3ckup/backup.rb

@@ -0,0 +1,37 @@
+module S3ckup
+  class Backup
+
+    def initialize(key_id, access_key, logger = Logger.new)
+      @logger = logger
+      @s3_manager = S3Mananger.new(key_id, access_key, @logger)
+    end
+
+    def run(bucket_name, folders)
+      local_files = Manifest.new(folders.flatten)
+      remote_files = @s3_manager.fetch_manifest(bucket_name)
+      backup(bucket_name, remote_files, local_files)
+    end
+
+    private
+
+    def fetch_remote_manifest(bucket)
+      manifest_yml = @s3_manager.fetch_manifest(bucket)
+      data = YAML.load(manifest_yml.value)
+      Manifest.new(data)
+    rescue AWS::S3::NoSuchKey
+      Manifest.new({})
+    end
+
+    def backup(bucket, remote_files, local_files)
+      removed_files = remote_files - local_files
+      @s3_manager.delete(bucket, removed_files)
+
+      remote_files = remote_files - removed_files
+      files_to_upload = local_files.diff(remote_files)
+
+      @s3_manager.update_manifest(bucket, local_files)
+      @s3_manager.upload(bucket, files_to_upload)
+    end
+
+  end
+end

data/lib/s3ckup/command_line.rb

@@ -0,0 +1,58 @@
+module S3ckup
+  class CommandLine
+    require 'optparse'
+    require 'logger'
+
+    def run(args)
+      logger = Logger.new($stdout)
+      logger.level = Logger::WARN
+
+      options = fetch_options(args, logger)
+      S3ckup::Backup.new(options[:key_id], options[:access_key], logger).
+        run(options[:bucket], options[:folders])
+    end
+
+    private
+
+    def fetch_options(args, logger)
+      Hash.new.tap do |options|
+        OptionParser.new do |opts|
+          opts.banner = "Usage: s3ckup OPTIONS --folders folder1,folder2,..."
+
+          opts.on("-k", "--aws-key-id KEY_ID", "AWS key id") do |key_id|
+            options[:key_id] = key_id
+          end
+
+          opts.on("-a", "--aws-access-key ACCESS_KEY", "AWS access key") do |access_key|
+            options[:access_key] = access_key
+          end
+
+          opts.on("-b", "--s3-bucket BUCKET_NAME", "S3 bucket name") do |bucket_name|
+            options[:bucket] = bucket_name
+          end
+
+          opts.on("-r", "--recursive", "Recursive mode") do
+            options[:recursive] = true
+          end
+
+          opts.on("-v", "--verbose", "Verbose") do
+            logger.level = Logger::INFO
+          end
+
+          opts.on("-f", "--folders FOLDER1,FOLDER2,FOLDER3...", Array, "Folder list, separated by ','") do |folder_list|
+            options[:folders] = []
+            folder_list.each do |folder|
+              if options[:recursive]
+                options[:folders] << File.join(folder, "**", "*")
+              else
+                options[:folders] << File.join(folder, "*")
+              end
+            end
+          end
+
+        end.parse!(args)
+      end
+    end
+
+  end
+end

data/lib/s3ckup/manifest.rb

@@ -0,0 +1,67 @@
+require 'digest/md5'
+require 'yaml'
+
+module S3ckup
+  class Manifest
+    attr_reader :files
+
+    def initialize(*args)
+      if args.first.is_a?(Hash)
+        @files = args.first
+      else
+        @files = create_manifest(Array(args).flatten)
+      end
+    end
+
+    def to_yaml
+      @files.to_yaml
+    end
+
+    def file_list
+      @files.keys.sort
+    end
+
+    def -(manifest)
+      raise ArgumentError unless manifest.is_a?(Manifest)
+      remaining_files = file_list - manifest.file_list
+      remaining_files = @files.clone.keep_if { |file| remaining_files.include?(file) }
+      Manifest.new(remaining_files)
+    end
+
+    def diff(manifest)
+      files_to_upload = @files.to_a - manifest.files.to_a
+      Manifest.new(Hash[*files_to_upload.flatten])
+    end
+
+    private
+
+    def create_manifest(folders)
+      Hash.new.tap do |files|
+        folders.each do |folder|
+          files.merge! scan_folder(folder)
+        end
+      end
+    end
+
+    def scan_folder(folder)
+      Hash.new.tap do |folder_files|
+        Dir[folder].each do |file_name|
+          if should_backup?(file_name)
+            folder_files[file_name] = file_digest(file_name)
+          end
+        end
+      end
+    end
+
+    def should_backup?(file_name)
+      ! (File.directory?(file_name) ||
+       File.symlink?(file_name) ||
+       File.socket?(file_name))
+    end
+
+    def file_digest(file_name)
+      Digest::MD5.file(file_name).hexdigest
+    end
+
+  end
+end

data/lib/s3ckup/s3_manager.rb

@@ -0,0 +1,49 @@
+require 'aws/s3'
+
+module S3ckup
+  class S3Mananger
+    MANIFEST_FILE_NAME = 'manifest.yml'
+
+    def initialize(key_id, access_key, logger = Logger.new)
+      connect!(key_id, access_key)
+      @logger = logger
+    end
+
+    def fetch_manifest(bucket)
+      manifest = AWS::S3::S3Object.find MANIFEST_FILE_NAME, bucket
+      Manifest.new(YAML.load(manifest.value))
+    rescue
+      @logger.warn("Failed to fetch remote manifest. Ignoring.")
+      Manifest.new({})
+    end
+
+    def upload(bucket, manifest)
+      manifest.file_list.each do |file_name|
+        @logger.info("Uploading file: #{file_name}")
+        AWS::S3::S3Object.store(file_name, open(file_name), bucket)
+      end
+    end
+
+    def update_manifest(bucket, manifest)
+      @logger.info("Updating remote manifest")
+      AWS::S3::S3Object.store(MANIFEST_FILE_NAME, manifest.to_yaml, bucket)
+    end
+
+    def delete(bucket, manifest)
+      manifest.file_list.each do |file_name|
+        @logger.info("Deleting remote file: #{file_name}")
+        AWS::S3::S3Object.delete file_name, bucket
+      end
+    end
+
+    private
+
+    def connect!(key_id, access_key)
+      AWS::S3::Base.establish_connection!(
+        :access_key_id     => key_id,
+        :secret_access_key => access_key
+      )
+    end
+
+  end
+end

data/lib/s3ckup/version.rb

@@ -0,0 +1,3 @@
+module S3ckup
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: s3ckup
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Tiago Scolari
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Manages incremental backups of files, soring on s3.
+email:
+- tscolari@gmail.com
+executables:
+- s3ckup
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- bin/s3ckup
+- lib/s3ckup.rb
+- lib/s3ckup/backup.rb
+- lib/s3ckup/command_line.rb
+- lib/s3ckup/manifest.rb
+- lib/s3ckup/s3_manager.rb
+- lib/s3ckup/version.rb
+homepage: https://github.com/tscolari/s3ckup
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Folders to s3 backuping tool.
+test_files: []