s3aps 0.0.1

data/lib/s3aps.rb

@@ -0,0 +1,5 @@
+require "s3aps/runner"
+require "s3aps/file_adapter"
+require "s3aps/s3_adapter"
+require "s3aps/sync"
+require "s3aps/version"

data/lib/s3aps/file_adapter.rb

@@ -0,0 +1,58 @@
+module S3aps
+
+  # Reads, writes and lists files on the filesystem.
+  class FileAdapter
+    
+    # Options:
+    #
+    # * :path - where the local files are (default is +tmp+)
+    def initialize(options = {})
+      @path = options[:path] || "tmp"
+    end
+    
+    # List all the files in the path, recursively
+    def list
+      Dir.glob("#{@path}/**/*").select{|path| File.file?(path) }.map do |path|
+        path.sub Regexp.new("^#{@path}\/"), ''
+      end
+    end
+    
+    # Write the file. It will always write the file whether it's there
+    # or not, unless you supply an +md5sum+ in which case it will check
+    # whether it's different first.
+    def write(path, value, md5sum = nil)
+		  just_path = path.sub /\/[^\/]*$/, ''
+		  local_md5 = md5sum(path)
+		  if local_md5.nil? || local_md5 != md5sum
+  		  FileUtils.mkdir_p(File.join(@path, just_path))
+        File.open(File.join(@path, path), 'w') {|outfile| 
+          outfile.print(value)
+        }
+        true
+      else
+        false
+      end
+    end
+    
+    # Read the file.
+    def read(path)
+      File.open(File.join(@path, path),"rb") {|io| io.read}
+    end
+    
+    def md5sum(path) #:nodoc:
+      md5 = nil
+      begin
+        md5 = File.open(File.join(@path, path), 'rb') do |io|
+          d = Digest::MD5.new
+          s = ""
+          d.update(s) while io.read(4096, s)
+          d
+        end
+      rescue
+      end
+      md5 ? md5.to_s : nil
+    end
+    
+  end
+  
+end

data/lib/s3aps/runner.rb

@@ -0,0 +1,83 @@
+require 'optparse'
+require 'fileutils'
+require 's3aps/sync'
+
+module S3aps
+  
+  # Allows you to run S3aps from the command line
+  class Runner
+
+  	attr_accessor :argv
+
+  	def initialize(argv)
+      @options = {}
+      @cmd = :help
+      @option_parser = OptionParser.new do |o|
+        o.banner = <<EOS
+S3aps version #{S3aps::VERSION}
+Usage: s3aps (list|pull|push|version) [OPTIONS]
+
+  list  List files locally and remotely
+  pull  Pull new files from remote
+  push  Push new files to remote (be careful)
+
+EOS
+        o.on("-c", "--config FILE", "YAML file containing S3 credentials") {|name| @options[:config] = name }
+        o.on("-e", "--env ENV", "Section of YAML to use") {|name| @options[:env] = name }
+        o.on("-b", "--bucket NAME", "S3 bucket name") {|name| @options[:bucket] = name }
+        o.on("-k", "--key KEY", "S3 access key") {|name| @options[:key] = name }
+        o.on("-s", "--secret KEY", "S3 secret key") {|name| @options[:secret] = name }
+        o.on("-p", "--path PATH", "Local path") {|name| @options[:path] = name }
+        o.parse!(argv)
+      end
+      if argv.size == 1 && %w(list push pull version).include?(argv.first)
+        @sync = S3aps::Sync.new @options 
+        @cmd = argv.first
+      end
+  	end
+
+  	def run #:nodoc:
+     send(@cmd)
+  	end
+  	
+  	def help #:nodoc:
+  	 puts @option_parser.help
+	  end
+	  
+	  def sync(options) #:nodoc:
+	    S3aps::Sync.new options
+    end
+
+    # Prints out a list of all remote and local files and shows which ones
+    # are remote, local or both.
+    def list
+      array = @sync.list
+      local_count = 0
+      remote_count = 0
+      array.each do |item|
+        line = item.join(' ')
+        local_count +=1 if line =~ /^L\s\s/
+        remote_count +=1 if line =~ /^\s\sR/
+        puts item.join(' ')
+      end
+      puts "Found #{array.size} file(s): #{local_count} local only, #{remote_count} remote only, #{array.size - local_count - remote_count} common."
+    end
+  	
+    # Pull files from S3
+  	def pull
+  	  count, total = @sync.pull 
+      puts "\nPulled #{count}/#{total} file#{'s' unless count == 1} from S3"
+  	end
+
+    # Push files to S3
+  	def push
+      count, total = @sync.push
+      puts "\nPushed #{count} file#{'s' unless count == 1} to S3"
+  	end
+
+  	def version #:nodoc:
+  		puts S3aps::VERSION
+  	end
+
+  end
+end

data/lib/s3aps/s3_adapter.rb

@@ -0,0 +1,40 @@
+require 'rubygems'
+require 'right_aws'
+
+module S3aps
+
+  # Reads, writes and lists files on S3.
+  class S3Adapter
+    
+    def initialize(options)
+      @s3 = RightAws::S3Interface.new(options[:key], options[:secret], :protocol => 'http', :port => 80)
+      @bucket_name = options[:bucket]
+    end
+    
+    # List the keys of all files in the bucket
+    def list
+      keys = []
+      @s3.incrementally_list_bucket(@bucket_name) {|key_set|
+        keys << key_set[:contents].map {|key|
+          key
+        }
+      }
+      keys.flatten
+    end
+    
+    # Write a value, potentially overwriting what is already there.
+    def write(s3_name, value)
+      @s3.put(@bucket_name, s3_name, value)
+    end
+    
+    def read(key)
+      begin
+        @s3.get_object(@bucket_name, key)
+      rescue 
+        raise exc
+      end
+    end
+
+  end
+
+end

data/lib/s3aps/sync.rb

@@ -0,0 +1,110 @@
+require 's3aps/file_adapter'
+require 's3aps/s3_adapter'
+require 'yaml'
+
+module S3aps
+  
+  def self.symbolize_keys(hash) #:nodoc:
+    new_hash = {}
+    hash.each do |key, value|
+      if Hash === value
+        new_hash[key.to_sym] = symbolize_keys value
+      else
+        new_hash[key.to_sym] = value
+      end
+    end
+    new_hash
+  end
+
+  # The controller for everything else. 
+  class Sync
+    
+    def initialize(options = {})
+      options = S3aps::symbolize_keys(options)
+      s3_file = [options[:config], "s3.yml", "config/s3.yml"].detect {|f| f && File.file?(f) }
+      if s3_file
+        puts "Configuring S3 from #{s3_file}"
+        yaml_options = S3aps::symbolize_keys YAML.load_file(s3_file)
+        env = options[:env]
+        if env && yaml_options[env.to_sym]
+          yaml_options = S3aps::symbolize_keys yaml_options[env.to_sym]
+          puts "Using #{env} environment"
+        end
+        # Support for alias: access_key_id = key 
+        if yaml_options[:access_key_id]
+          yaml_options[:key] ||= yaml_options[:access_key_id]
+          yaml_options.delete(:access_key_id)
+        end
+        # Support for alias: secret_access_key = secret 
+        if yaml_options[:secret_access_key]
+          yaml_options[:secret] ||= yaml_options[:secret_access_key]
+          yaml_options.delete(:secret_access_key)
+        end
+        options.merge!(yaml_options) {|k,o,n| o }
+      end
+      @s3_adapter = S3aps::S3Adapter.new options
+      @file_adapter = S3aps::FileAdapter.new options
+    end
+    
+    # Merge the local and remote lists into one and indicate whether
+    # each item is local, remote or both
+    def list
+      local = @file_adapter.list 
+      remote = @s3_adapter.list.map{|i| i[:key]}
+      all = (local + remote).uniq.sort
+      all.map do |path|
+        [
+          local.include?(path) ? "L" : " ",
+          remote.include?(path) ? "R" : " ",
+          path
+        ]
+      end
+    end
+    
+    # Get all the remote files and write them out locally, if they don't already
+    # exist. For simplicity, we assume that if a file of the same name already
+    # exists then it is the same as the remote one. 
+    def pull
+      check_list = @file_adapter.list
+      count = 0
+      total = 0
+  		@s3_adapter.list.each do |o|
+        if !check_list.include?(o[:key])
+          $stdout.print "*"
+  		    if @file_adapter.write(o[:key], @s3_adapter.read(o[:key]), o[:etag])
+  		      count += 1
+		      end
+		    else
+          $stdout.print "."
+		    end
+        $stdout.flush
+        total += 1
+		  end
+    [count, total]
+    end
+    
+    # Get all the local files, recursively, and write them to S3, if they don't already
+    # exist. For simplicity, we assume that if a file of the same name already
+    # exists then it is the same as the local one. 
+    def push
+      check_list = @s3_adapter.list.map{|i| i[:key]}
+      count = 0
+      total = 0
+      @file_adapter.list.each do |path|
+        if !check_list.include?(path)
+          $stdout.print "*"
+          if @s3_adapter.write(path, @file_adapter.read(path))
+            count += 1
+          end
+		    else
+          $stdout.print "."
+        end
+        $stdout.flush
+        total += 1
+      end
+      [count, total]
+    end 
+
+  end
+  
+end

data/lib/s3aps/version.rb

@@ -0,0 +1,3 @@
+module S3aps #:nodoc:
+  VERSION = "0.0.1" #:nodoc:
+end

data/s3aps.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "s3aps/version"
+
+Gem::Specification.new do |s|
+  s.name        = "s3aps"
+  s.version     = S3aps::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Bill Horsman"]
+  s.email       = ["bill@logicalcobwebs.com"]
+  s.homepage    = "http://github.com/billhorsman/s3aps"
+  s.summary     = "s3aps-#{S3aps::VERSION}"
+  s.description = "Synchronize between S3 bucket and filesystem"
+
+  s.required_rubygems_version = ">= 1.3.6"
+
+  s.add_development_dependency "bundler", ">= 1.0.0"
+  s.add_runtime_dependency "right_aws"
+
+  s.files        = `git ls-files`.split("\n")
+  s.executables  = `git ls-files`.split("\n").map{|f| f =~ /^bin\/(.*)/ ? $1 : nil}.compact
+  s.require_path = 'lib'
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification 
+name: s3aps
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Bill Horsman
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-05-17 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: bundler
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.0
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: right_aws
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+description: Synchronize between S3 bucket and filesystem
+email: 
+- bill@logicalcobwebs.com
+executables: 
+- s3aps
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- README.rdoc
+- Rakefile
+- bin/s3aps
+- doc/S3aps.html
+- doc/S3aps/FileAdapter.html
+- doc/S3aps/Runner.html
+- doc/S3aps/S3Adapter.html
+- doc/S3aps/Sync.html
+- doc/created.rid
+- doc/index.html
+- doc/lib/s3aps/file_adapter_rb.html
+- doc/lib/s3aps/runner_rb.html
+- doc/lib/s3aps/s3_adapter_rb.html
+- doc/lib/s3aps/sync_rb.html
+- doc/lib/s3aps/version_rb.html
+- doc/lib/s3aps_rb.html
+- doc/rdoc.css
+- lib/s3aps.rb
+- lib/s3aps/file_adapter.rb
+- lib/s3aps/runner.rb
+- lib/s3aps/s3_adapter.rb
+- lib/s3aps/sync.rb
+- lib/s3aps/version.rb
+- s3aps.gemspec
+has_rdoc: true
+homepage: http://github.com/billhorsman/s3aps
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: 1.3.6
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: s3aps-0.0.1
+test_files: []
+