ssync 0.1.0

data/.gitignore

@@ -0,0 +1,2 @@
+.bundle/*
+pkg/*

data/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,23 @@
+PATH
+  remote: .
+  specs:
+    ssync (0.1.0)
+      aws-s3 (~> 0.6.2)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    aws-s3 (0.6.2)
+      builder
+      mime-types
+      xml-simple
+    builder (2.1.2)
+    mime-types (1.16)
+    xml-simple (1.0.12)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aws-s3 (~> 0.6.2)
+  ssync!

data/MIT-LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2010 Ryan Allen, Envato Pty Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,70 @@
+# Ssync
+
+__Ssync__, an optimised S3 sync tool using the power of *nix!
+
+## Requirements
+
+- Ruby 1.8 or 1.9
+- RubyGems
+- 'aws-s3' rubygem
+- `openssl`
+- `find` and `xargs`
+
+## Installation
+
+    gem install ssync
+
+## Configuration
+
+To configure, run `ssync setup` and follow the prompts, you'll
+need your AWS keys, the local file path you want to back up, the bucket name
+to back up to, and any extra options to pass into find (i.e. for ignoring
+filepaths etc). It'll write the config to `~/.ssync.yml`.
+
+## Synchronisation
+
+To sync, run `ssync sync` and away it goes.
+
+In the case of a corrupted/incomplete synchronisation, run `ssync sync -f`
+or `ssync sync --force` to force a checksum comparison.
+
+## Why?
+
+This library was written because we needed to be able to back up loads of
+data without having to worry about if we had enough disk space on the remote.
+That's where S3 is nice.
+
+We tried [s3sync](http://www.s3sync.net/) but it blew our server load (we do in excess of
+500,000 requests a day (page views, not including hits for images and what not,
+and the server needs to stay responsive). The secret sauce is using the *nix
+`find`, `xargs` and `openssl` commands to generate md5 checksums for comparison.
+Seems to work quite well for us (we have almost 90,000 files to compare).
+
+Initially the plan was to use `find` with `-ctime` but S3 isn't particularly nice about
+returning a full list of objects in a bucket (default is 1000, and I want all
+90,000, and it ignores me when I ask for 1,000,000 objects). Manifest generation
+on a server under load is fast enough and low enough on resources so we're sticking
+with that in the interim.
+
+FYI when you run sync, the output will look something like this:
+
+    [Thu Apr 01 11:50:25 +1100 2010] Starting, performing pre-sync checks ...
+    [Thu Apr 01 11:50:26 +1100 2010] Generating local manifest ...
+    [Thu Apr 01 11:50:26 +1100 2010] Fetching remote manifest ...
+    [Thu Apr 01 11:50:27 +1100 2010] Performing checksum comparison ...
+    [Thu Apr 01 11:50:27 +1100 2010] Pushing /tmp/backups/deep/four ...
+    [Thu Apr 01 11:50:28 +1100 2010] Pushing /tmp/backups/three ...
+    [Thu Apr 01 11:50:29 +1100 2010] Pushing /tmp/backups/two ...
+    [Thu Apr 01 11:50:30 +1100 2010] Pushing local manifest up to remote ...
+    [Thu Apr 01 11:50:31 +1100 2010] Sync complete!
+
+You could pipe sync into a log file, which might be nice.
+
+Have fun!
+
+## Authors
+
+- [Ryan Allen](https://github.com/ryan-allen)
+- [Fred Wu](https://github.com/fredwu)
+
+This project is brought to you by [Envato](http://envato.com/) Pty Ltd.

data/Rakefile

@@ -0,0 +1,5 @@
+begin
+  require 'bundler'
+  Bundler::GemHelper.install_tasks
+rescue Exception => e
+end

data/bin/ssync

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+require "ssync/command"
+
+Ssync::Command.run!(*ARGV)

data/lib/ssync.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+
+require "aws/s3"
+require "yaml"
+require "ssync/helpers"
+require "ssync/setup"
+require "ssync/sync"
+require "ssync/version"

data/lib/ssync/command.rb

@@ -0,0 +1,52 @@
+require "ssync"
+
+module Ssync
+  class Command
+    include Helpers
+
+    def self.action
+      @@action
+    end
+
+    def self.args
+      @@args
+    end
+
+    def self.run!(*args)
+      new(*args).run!
+    end
+
+    def initialize(action = :sync, *args)
+      @@action = action.to_sym
+      @@args   = *args
+    end
+
+    def run!
+      pre_run_check!
+      perform_action!
+    end
+
+    private
+
+    def pre_run_check!
+      if action_eq?(:setup) && config_exists?
+        e! "Cannot run the setup, there is already an Ssync configuration in '#{config_path}'."
+      elsif action_eq?(:sync) && !config_exists?
+        e! "Cannot run the sync, there is no Ssync configuration, try 'ssync setup' to create one first."
+      end
+    end
+
+    def perform_action!
+      case @@action
+      when :setup
+        aquire_lock! { Ssync::Setup.run! }
+      when :sync
+        aquire_lock! { Ssync::Sync.run! }
+      when :help
+        display_help!
+      else
+        e! "Cannot perform action '#{@action}', try 'ssync help' for usage."
+      end
+    end
+  end
+end

data/lib/ssync/helpers.rb

@@ -0,0 +1,76 @@
+module Ssync
+  module Helpers
+    def display_help!
+      display("Not implemented yet.")
+    end
+
+    def display(message)
+      puts("[#{Time.now}] #{message}")
+    end
+
+    def display_error(message)
+      display("Error! " + message)
+    end
+
+    def exit_with_error!(message)
+      display_error(message)
+      exit
+    end
+
+    alias :e :display_error
+    alias :e! :exit_with_error!
+
+    def ask(config_item, question)
+      print(question + " [#{config_item}]: ")
+      a = $stdin.readline.chomp
+      a.empty? ? config_item : a
+    end
+
+    def action_eq?(action)
+      Ssync::Command.action == action.to_sym
+    end
+
+    def config_exists?
+      File.exist?(config_path)
+    end
+
+    def config_path
+      ENV['HOME'] + "/.ssync.yml"
+    end
+
+    def lock_path
+      ENV['HOME'] + "/.ssync.lock"
+    end
+
+    def aquire_lock!
+      # better way is to write out the pid ($$) and read it back in, to make sure it's the same
+      e! "Found a lock at #{lock_path}, is another instance of Ssync running?" if File.exist?(lock_path)
+
+      begin
+        system "touch #{lock_path}"
+        yield
+      ensure
+        system "rm #{lock_path}"
+      end
+    end
+
+    def read_config
+      begin
+        open(config_path, "r") { |f| YAML::load(f) }
+      rescue
+        {}
+      end
+    end
+
+    def write_config!(config)
+      open(config_path, "w") { |f| YAML::dump(config, f) }
+    end
+
+    def options_set?(*options)
+      false
+      [options].flatten.each do |option|
+        return true if Command.args.include?("#{option.to_s}")
+      end
+    end
+  end
+end

data/lib/ssync/setup.rb

@@ -0,0 +1,92 @@
+module Ssync
+  class Setup
+    class << self
+      include Helpers
+
+      def config
+        @config
+      end
+
+      def config=(config)
+        @config = config
+      end
+
+      def run!
+        display "Welcome to Ssync! You will now be asked a few questions, the results will be stored at '#{config_path}'."
+
+        config = read_config
+
+        config[:aws_access_key] = ask config[:aws_access_key], "What is the AWS Access Key ID?"
+        config[:aws_secret_key] = ask config[:aws_secret_key], "What is the AWS Secret Access Key?"
+
+        display "Please wait while Ssync is connecting to AWS ..."
+
+        if aws_credentials_is_valid?(config)
+          display "Successfully connected to AWS."
+
+          config[:aws_dest_bucket] = ask config[:aws_dest_bucket], "Which bucket would you like to put your backups in? Ssync will create the bucket for you if it doesn't exist."
+
+          if bucket_exists?(config)
+            if bucket_empty?(config)
+              display "The bucket exists and is empty, great!"
+            else
+              e! "The bucket exists but is not empty, we cannot sync to a bucket that is not empty!"
+            end
+          else
+            display "The bucket doesn't exist, creating it now ..."
+            create_bucket(config)
+            display "The bucket has been created."
+          end
+        else
+          e! "Ssync wasn't able to connect to AWS, please check the credentials you supplied are correct."
+        end
+
+        require "pathname"
+        config[:local_file_path] = ask config[:local_file_path], "What is the path you would like to backup? (i.e. '/var/www')."
+        config[:local_file_path] = Pathname.new(config[:local_file_path]).realpath.to_s
+
+        if local_file_path_exists?(config)
+          display "The path is set to '#{config[:local_file_path]}'."
+        else
+          e! "The path you specified does not exist!"
+        end
+
+        config[:find_options] = ask config[:find_options], "Do you have any options for 'find'? (e.g. \! -path *.git*)."
+
+        display "Writing the supplied details to '#{config_path}' for future reference ..."
+        write_config!(config)
+        display "All done! You may now use 'ssync sync' to syncronise your files to the S3 bucket."
+      end
+
+      def aws_credentials_is_valid?(config = read_config)
+        AWS::S3::Base.establish_connection!(:access_key_id => config[:aws_access_key], :secret_access_key => config[:aws_secret_key])
+        begin
+          # AWS::S3 don't try to connect at all until you ask it for something.
+          AWS::S3::Service.buckets
+        rescue AWS::S3::InvalidAccessKeyId => e
+          false
+        else
+          true
+        end
+      end
+
+      def bucket_exists?(config = read_config)
+        AWS::S3::Bucket.find(config[:aws_dest_bucket])
+      rescue AWS::S3::NoSuchBucket => e
+        false
+      end
+
+      def bucket_empty?(config = read_config)
+        AWS::S3::Bucket.find(config[:aws_dest_bucket]).empty?
+      end
+
+      def create_bucket(config = read_config)
+        AWS::S3::Bucket.create(config[:aws_dest_bucket])
+      end
+
+      def local_file_path_exists?(config = read_config)
+        File.exist?(config[:local_file_path])
+      end
+    end
+  end
+end

data/lib/ssync/sync.rb

@@ -0,0 +1,126 @@
+module Ssync
+  class Sync
+    class << self
+      include Helpers
+
+      def run!
+        display "Initialising Ssync, performing pre-sync checks ..."
+
+        e! "Couldn't connect to AWS with the credentials specified in '#{config_path}'." unless Setup.aws_credentials_is_valid?
+        e! "Couldn't find the S3 bucket specified in '#{config_path}'." unless Setup.bucket_exists?
+        e! "The local path specified in '#{config_path}' does not exist." unless Setup.local_file_path_exists?
+
+        if options_set?("-f", "--force")
+          display "Clearing previous sync state ..."
+          clear_sync_state
+        end
+        create_tmp_sync_state
+
+        if last_sync_recorded?
+          display "Performing time based comparison ..."
+          files_modified_since_last_sync
+        else
+          display "Performing (potentially expensive) MD5 checksum comparison ..."
+          display "Generating local manifest ..."
+          generate_local_manifest
+          display "Traversing S3 for remote manifest ..."
+          fetch_remote_manifest
+          # note that we do not remove files on s3 that no longer exist on local host.
+          # this behaviour may be desirable (ala rsync --delete) but we currently don't support it.
+          display "Performing checksum comparison ..."
+          files_on_localhost_with_checksums - files_on_s3
+        end.each { |file| push_file(file) }
+
+        finalize_sync_state
+
+        display "Sync complete!"
+      end
+
+      def clear_sync_state
+        `rm -f #{last_sync_started} #{last_sync_completed}`
+      end
+
+      def create_tmp_sync_state
+        `touch #{last_sync_started}`
+      end
+
+      def last_sync_started
+        ENV['HOME'] + "/.ssync.last-sync.started"
+      end
+
+      def last_sync_completed
+        ENV['HOME'] + "/.ssync.last-sync.completed"
+      end
+
+      def last_sync_recorded?
+        File.exist?(last_sync_completed)
+      end
+
+      def finalize_sync_state
+        `cp #{last_sync_started} #{last_sync_completed}`
+      end
+
+      def files_modified_since_last_sync
+        # '! -type d' ignores directories, in local manifest directories are spit out to stderr whereas directories pop up in this query
+        `find #{read_config[:local_file_path]} #{read_config[:find_options]} \! -type d -cnewer #{last_sync_completed}`.split("\n").collect { |path| { :path => path } }
+      end
+
+      def update_config_with_sync_state(sync_start)
+        config = read_config()
+        config[:last_sync_at] = sync_start
+        write_config!(config)
+      end
+
+      def generate_local_manifest
+        `find #{read_config[:local_file_path]} #{read_config[:find_options]} -print0 | xargs -0 openssl md5 2> /dev/null > #{local_manifest_path}`
+      end
+
+      def fetch_remote_manifest
+        @remote_objects_cache = []
+        traverse_s3_for_objects(AWS::S3::Bucket.find(read_config[:aws_dest_bucket]), @remote_objects_cache)
+      end
+
+      def traverse_s3_for_objects(bucket, collection, n = 1000, upto = 0, marker = nil)
+        objects = bucket.objects(:marker => marker, :max_keys => n)
+        if objects.size == 0
+          return
+        else
+          objects.each { |object| collection << { :path => "/#{object.key}", :checksum => object.etag } }
+          traverse_s3_for_objects(bucket, collection, n, upto+n, objects.last.key)
+        end
+      end
+
+      def files_on_localhost_with_checksums
+        parse_manifest(local_manifest_path)
+      end
+
+      def files_on_s3
+        @remote_objects_cache
+      end
+
+      def local_manifest_path
+        "/tmp/ssync.manifest.local"
+      end
+
+      def parse_manifest(location)
+        []
+        if File.exist?(location)
+          open(location, "r") do |file|
+            file.collect do |line|
+              path, checksum = *line.chomp.match(/^MD5\((.*)\)= (.*)$/).captures
+              { :path => path, :checksum => checksum }
+            end
+          end
+        end
+      end
+
+      def push_file(file)
+        # xfer speed, logging, etc can occur in this method
+        display "Pushing '#{file[:path]}' ..."
+        AWS::S3::S3Object.store(file[:path], open(file[:path]), read_config[:aws_dest_bucket])
+      rescue
+        e "Could not push '#{file[:path]}': #{$!.inspect}"
+      end
+    end
+  end
+end

data/lib/ssync/version.rb

@@ -0,0 +1,3 @@
+module Ssync
+  VERSION = "0.1.0"
+end

data/ssync.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+require File.dirname(__FILE__) + "/lib/ssync/version"
+
+Gem::Specification.new do |s|
+  s.name              = "ssync"
+  s.version           = Ssync::VERSION
+  s.date              = Date.today.to_s
+  s.authors           = ["Fred Wu", "Ryan Allen"]
+  s.email             = ["fred@envato.com", "ryan@envato.com"]
+  s.summary           = %q{Ssync, an optimised S3 sync tool using the power of *nix!}
+  s.description       = %q{Ssync, an optimised S3 sync tool using the power of *nix!}
+  s.homepage          = %q{http://github.com/fredwu/ssync}
+  s.extra_rdoc_files  = ["README.md"]
+  s.rdoc_options      = ["--charset=UTF-8"]
+  s.require_paths     = ["lib"]
+  s.rubyforge_project = s.name
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency(%q<aws-s3>, ["~> 0.6.2"])
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification 
+name: ssync
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Fred Wu
+- Ryan Allen
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-11-12 00:00:00 +11:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: aws-s3
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 6
+        - 2
+        version: 0.6.2
+  type: :runtime
+  version_requirements: *id001
+description: Ssync, an optimised S3 sync tool using the power of *nix!
+email: 
+- fred@envato.com
+- ryan@envato.com
+executables: 
+- ssync
+extensions: []
+
+extra_rdoc_files: 
+- README.md
+files: 
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- MIT-LICENSE
+- README.md
+- Rakefile
+- bin/ssync
+- lib/ssync.rb
+- lib/ssync/command.rb
+- lib/ssync/helpers.rb
+- lib/ssync/setup.rb
+- lib/ssync/sync.rb
+- lib/ssync/version.rb
+- ssync.gemspec
+has_rdoc: true
+homepage: http://github.com/fredwu/ssync
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: ssync
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Ssync, an optimised S3 sync tool using the power of *nix!
+test_files: []
+