s3archive 1.0

data/bin/s3archive

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+load File.expand_path('../../lib/tasks/s3archive.thor', __FILE__)
+S3Archive::Cli.start

data/lib/s3archive/compress_and_upload.rb

@@ -0,0 +1,100 @@
+# The main entry point db:backup_to_s3 is run by /etc/logrotate.d/rails3app
+# daily. changed
+
+require 'thor'
+require 'socket'  # For hostname
+require 'tempfile'
+require_relative 'logging'
+require_relative 'config'
+require_relative 's3_file_synchronizer'
+
+module S3Archive
+  class CompressAndUpload
+    include Logging
+
+    def self.run(path)
+      new(path).run
+    end
+
+    attr_reader :path
+    def initialize(path)
+      @path = path
+    end
+    
+    def run
+      unless File.exists?(path)
+        logger.error("COULD NOT FIND '#{path}'")
+        return
+      end
+
+      logger.info("* Processing #{path}")
+
+      compress! if compress?
+      upload!
+      delete_tempfile! if compress?
+    end
+
+    private
+    def compress?
+      @do_compress ||= begin
+        if path.end_with?('.gz')
+          logger.info("** #{path} already compressed, skipping compression")
+          false
+        else
+          true
+        end
+      end
+    end
+
+    def compress!
+      logger.info("** Compressing #{path} to #{tempfile.path}")
+      system "gzip -n -c < #{path} > #{tempfile.path}"
+    end
+
+    def upload!
+      bucket = S3Archive.config.bucket
+      logger.info("** Uploading #{path_to_upload} to s3://#{bucket}/#{key}")
+      S3FileSynchronizer.run(path_to_upload, bucket, key)
+    end
+
+    def path_to_upload
+      compress? ? tempfile.path : path
+    end
+
+    def delete_tempfile!
+      logger.info("** Deleting #{tempfile.path}")
+      tempfile.unlink
+    end
+
+    def tempfile
+      @tempfile ||= begin
+        tempfile = Tempfile.new([filename, '.gz'])
+        tempfile.close # An external process will write to it
+        tempfile
+      end
+    end
+
+    def key
+      year, month, day = Time.now.strftime("%Y-%m-%d").split('-')
+      [hostname, year, month, day, "#{filename}.gz"].join('/')
+    end
+
+    def filename
+      @filename ||= File.basename(path)
+    end
+
+    def hostname
+      Socket.gethostname
+    end
+  end
+end
+
+
+if $0 == __FILE__
+  path = File.join(File.dirname(__FILE__), 'config.rb')
+  S3Archive::CompressAndUpload.run(path)
+
+#  bucket = 'this.is.my.test.bucket'
+#  key = 'this is a folder/foobar'
+#  S3FileSynchronizer.run(in_path, bucket, key)
+end

data/lib/s3archive/config.rb

@@ -0,0 +1,26 @@
+require 'singleton'
+require 'yaml'
+
+module S3Archive
+  def self.config_path=(config_path)
+    @config_path = config_path
+  end
+
+  def self.config_path
+    @config_path || "/etc/s3archive.yml"
+  end
+
+  def self.config
+    @config ||= Config.new(YAML.load(File.read(config_path)))
+  end
+
+  class Config
+    attr_accessor :bucket, :access_key_id, :secret_access_key
+
+    def initialize(params = {})
+      params.each do |key, val|
+        send("#{key}=", val)
+      end
+    end
+  end
+end

data/lib/s3archive/logging.rb

@@ -0,0 +1,9 @@
+require 'logger'
+
+module S3Archive
+  module Logging
+    def logger
+      @logger ||= Logger.new(STDOUT)
+    end
+  end
+end

data/lib/s3archive/s3_file_synchronizer.rb

@@ -0,0 +1,113 @@
+require 'right_aws'
+require 'digest/md5'
+require_relative 'logging'
+
+module S3Archive
+  class S3FileSynchronizer
+    include Logging
+
+    attr_reader :local_file, :s3_file
+    def initialize(local_file, s3_file)
+      @local_file = local_file
+      @s3_file    = s3_file
+    end
+
+    def run
+      if s3_file.exists?
+        if s3_file.md5_hex == local_file.md5_hex
+          logger.info("'#{s3_file}' already exists and has correct checksum")
+          nil
+        else
+          new_s3_file = S3File.new(s3_file.bucket, "#{s3_file.key}.#{local_file.md5_hex}")
+          logger.error("'#{s3_file}' already exists and has wrong checksum. Uploading to '#{new_s3_file}'.")
+          new_s3_file.put(local_file)
+        end
+      else
+        s3_file.put(local_file)
+      end
+    end
+
+    def self.run(local_path, bucket, key)
+      local_file = LocalFile.new(local_path)
+      s3_file = S3File.new(bucket, key)
+
+      new(local_file, s3_file).run
+    end
+  end
+
+  # Just a wrapper around a path with some md5 functions
+  LocalFile = Struct.new(:path) do
+    include Logging
+
+    def open(*args, &block)
+      File.open(path, *args, &block)
+    end
+
+    def md5_hex
+      md5.hexdigest
+    end
+
+    def md5_base64
+      md5.base64digest
+    end
+
+    def md5
+      @md5 ||= Digest::MD5.file(path)
+    end
+
+    def to_s
+      path
+    end
+  end
+
+  # A wrapper around a s3 path (bucket, key) with some md5 and a put function
+  S3File = Struct.new(:bucket, :key) do
+    include Logging
+
+    def md5_hex
+      exists? && headers.fetch("etag").tr('"', '')
+    end
+
+    def exists?
+      !headers.nil?
+    end
+
+    def put(local_file)
+      local_file.open do |file|
+        logger.info("Putting '#{local_file}' to '#{self}'")
+        s3interface.put(bucket, key, file, 'Content-MD5' => local_file.md5_base64)
+      end
+    end
+
+    def to_s
+      "s3://#{bucket}/#{key}"
+    end
+
+    private
+    def headers
+      @headers ||= without_close_on_error do
+        begin
+          s3interface.head(bucket, key)
+        rescue RightAws::AwsError => e
+          raise unless e.http_code.to_s == '404'
+        end
+      end
+    end
+
+    def without_close_on_error(&block)
+      old_val = RightAws::AWSErrorHandler.close_on_error
+      RightAws::AWSErrorHandler.close_on_error = false
+      block.call
+    ensure
+      RightAws::AWSErrorHandler.close_on_error = old_val
+    end
+
+    def s3interface
+      @s3interface ||= RightAws::S3Interface.new(
+        S3Archive.config.access_key_id,
+        S3Archive.config.secret_access_key,
+        :logger => logger
+      )
+    end
+  end
+end

data/lib/tasks/s3archive.thor

@@ -0,0 +1,22 @@
+require 'thor'
+$:.unshift File.join(File.dirname(__FILE__), '..')
+require 's3archive/compress_and_upload'
+
+module S3Archive
+  class Cli < Thor
+    namespace :s3archive
+
+    class_option "-c",
+      :desc => "Path to config file",
+      :banner => "CONFIG_FILE",
+      :type => :string,
+      :aliases => "--config",
+      :default => "/etc/s3archive.yml"
+
+    desc "upload PATH", "Compresses PATH and uploads to s3://<bucket>/<year>/<month>/<day>/<filename>.gz"
+    def upload (orig_path, options = {})
+      S3Archive.config_path = self.options["c"] if self.options["c"]
+      CompressAndUpload.run(orig_path)
+    end
+  end
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: s3archive
+version: !ruby/object:Gem::Version
+  version: '1.0'
+  prerelease: 
+platform: ruby
+authors:
+- Petter Remen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-05-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.14.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.14.6
+- !ruby/object:Gem::Dependency
+  name: right_aws
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.9.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.9.0
+description: ''
+email:
+- petter@spnab.com
+executables:
+- s3archive
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/s3archive
+- lib/s3archive/compress_and_upload.rb
+- lib/s3archive/config.rb
+- lib/s3archive/logging.rb
+- lib/s3archive/s3_file_synchronizer.rb
+- lib/tasks/s3archive.thor
+homepage: http://github.com/spab/s3archive
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Simple script to safely archive a file to S3
+test_files: []