s3_meta_sync 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: dc3bcfa388ab131fa0a8cd4117dac7c2e21460e7
+  data.tar.gz: 311333996d21e3ee022dd1beb8727b5298b2b5d6
+SHA512:
+  metadata.gz: 1d6d33a5fbfac5e447a0aa76470d35ab31e691bca5a5464af9d767d2a2aac569d61669a75d14e32d177f18cb2bf8ec68dabd1b0adae3c755e910c1f5d02fd5f1
+  data.tar.gz: 3b2adea548763266b7c439148d629c6ac521567aea458158c15be01bf7c81cf481de867790998dd2c7695b4a603d98e9344e4e79a3a6cd09c3e603c0ba2ca65b

checksums.yaml.gz.sig

@@ -0,0 +1,2 @@
+��gm����
+�b��ې�m0�?�|�jC�$�"���]���{ɤ'ٍර#sJE�F����8`F�,jel5()Z�G�

data/bin/s3-meta-sync

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$LOAD_PATH << File.expand_path("../../lib", __FILE__)
+require "rubygems" if RUBY_VERSION < "1.9"
+require "s3_meta_sync"
+exit S3MetaSync.run(ARGV)

data/lib/s3_meta_sync/version.rb

@@ -0,0 +1,3 @@
+module S3MetaSync
+  VERSION = "0.1.0"
+end

data/lib/s3_meta_sync.rb

@@ -0,0 +1,162 @@
+require "s3_meta_sync/version"
+require "open-uri"
+require "yaml"
+require "digest/md5"
+require "aws/s3"
+require "optparse"
+
+module S3MetaSync
+  RemoteWithoutMeta = Class.new(Exception)
+  META_FILE = ".s3-meta-sync"
+
+  class Syncer
+    def initialize(config)
+      @config = config
+    end
+
+    def sync(source, destination)
+      raise if source.end_with?("/") or destination.end_with?("/")
+
+      if destination.include?(":")
+        @bucket, destination = destination.split(":")
+        upload(source, destination)
+      else
+        @bucket, source = source.split(":")
+        download(source, destination)
+      end
+    end
+
+    private
+
+    def upload(source, destination)
+      remote_info = begin
+        download_meta(destination)
+      rescue RemoteWithoutMeta
+        {}
+      end
+      generate_meta(source)
+      local_info = read_meta(source)
+
+      local_info.each do |path, md5|
+        next if remote_info[path] == md5
+        upload_file(source, path, destination)
+      end
+
+      (remote_info.keys - local_info.keys).each do |path|
+        delete_remote_file(destination, path)
+      end
+
+      upload_file(source, META_FILE, destination)
+    end
+
+    def upload_file(source, path, destination)
+      s3.objects["#{destination}/#{path}"].write File.read("#{source}/#{path}"), :acl => :public_read
+    end
+
+    def delete_remote_file(remote, path)
+      s3.objects["#{remote}/#{path}"].delete
+    end
+
+    def delete_local_file(local, path)
+      File.delete("#{local}/#{path}")
+    end
+
+    def s3
+      @s3 ||= ::AWS::S3.new(:access_key_id => @config[:key], :secret_access_key => @config[:secret]).buckets[@bucket]
+    end
+
+    def generate_meta(source)
+      meta = Hash[Dir["#{source}/**/*"].select { |f| File.file?(f) }.map do |file|
+        [file.sub("#{source}/", ""), Digest::MD5.file(file).to_s]
+      end]
+      file = "#{source}/#{META_FILE}"
+      FileUtils.mkdir_p(File.dirname(file))
+      File.write(file, meta.to_yaml)
+    end
+
+    def read_meta(source)
+      file = "#{source}/#{META_FILE}"
+      File.exist?(file) ? YAML.load(File.read(file)) : {}
+    end
+
+    def download_meta(destination)
+      content = download_content("#{destination}/#{META_FILE}")
+      YAML.load(content)
+    rescue
+      raise RemoteWithoutMeta
+    end
+
+    def download_file(source, path, destination)
+      content = download_content("#{source}/#{path}")
+      file = "#{destination}/#{path}"
+      FileUtils.mkdir_p(File.dirname(file))
+      File.write(file, content)
+    end
+
+    def download_content(path)
+      url = "https://s3#{"-#{region}" if region}.amazonaws.com/#{@bucket}/#{path}"
+      open(url).read
+    rescue OpenURI::HTTPError
+      raise "Unable to download #{url} -- #{$!}"
+    end
+
+    def download(source, destination)
+      remote_info = download_meta(source)
+      generate_meta(destination)
+      local_info = read_meta(destination) # TODO maybe generate !?
+
+      remote_info.each do |path, md5|
+        next if local_info[path] == md5
+        download_file(source, path, destination)
+      end
+
+      (local_info.keys - remote_info.keys).each do |path|
+        delete_local_file(destination, path)
+      end
+
+      download_file(source, META_FILE, destination)
+
+      `find #{destination} -depth -empty -delete`
+    end
+
+    def region
+      @config[:region] unless @config[:region].to_s.empty?
+    end
+  end
+
+  class << self
+    def run(argv)
+      source, dest, options = parse_options(argv)
+      Syncer.new(options).sync(source, dest)
+      0
+    end
+
+    def parse_options(argv)
+      options = {}
+      OptionParser.new do |opts|
+        opts.banner = <<-BANNER.gsub(/^ {10}/, "")
+          Sync folders with s3 using a metadata file with md5 sums.
+
+          # upload local files and remove everything that is not local
+          s3-meta-sync <local> <bucket:folder> --key <aws-access-key> --secret <aws-secret-key>
+
+          # download files and remove everything that is not remote
+          s3-meta-sync <bucket:folder> <local> # no credentials required
+
+
+          Options:
+        BANNER
+        opts.on("-k", "--key KEY", "AWS access key") { |c| options[:key] = c }
+        opts.on("-s", "--secret SECRET", "AWS secret key") { |c| options[:secret] = c }
+        opts.on("-r", "--region REGION", "AWS region if not us-standard") { |c| options[:region] = c }
+        opts.on("-h", "--help", "Show this.") { puts opts; exit }
+        opts.on("-v", "--version", "Show Version"){ puts VERSION; exit}
+      end.parse!(argv)
+
+      raise "need source and destination" unless argv.size == 2
+      raise "need credentials --key + --secret" if argv.last.include?(":") and not options[:key] or not options[:secret]
+
+      [*argv, options]
+    end
+  end
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification
+name: s3_meta_sync
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Michael Grosser
+autorequire: 
+bindir: bin
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIDMjCCAhqgAwIBAgIBADANBgkqhkiG9w0BAQUFADA/MRAwDgYDVQQDDAdtaWNo
+  YWVsMRcwFQYKCZImiZPyLGQBGRYHZ3Jvc3NlcjESMBAGCgmSJomT8ixkARkWAml0
+  MB4XDTEzMDIwMzE4MTMxMVoXDTE0MDIwMzE4MTMxMVowPzEQMA4GA1UEAwwHbWlj
+  aGFlbDEXMBUGCgmSJomT8ixkARkWB2dyb3NzZXIxEjAQBgoJkiaJk/IsZAEZFgJp
+  dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMorXo/hgbUq97+kII9H
+  MsQcLdC/7wQ1ZP2OshVHPkeP0qH8MBHGg6eYisOX2ubNagF9YTCZWnhrdKrwpLOO
+  cPLaZbjUjljJ3cQR3B8Yn1veV5IhG86QseTBjymzJWsLpqJ1UZGpfB9tXcsFtuxO
+  6vHvcIHdzvc/OUkICttLbH+1qb6rsHUceqh+JrH4GrsJ5H4hAfIdyS2XMK7YRKbh
+  h+IBu6dFWJJByzFsYmV1PDXln3UBmgAt65cmCu4qPfThioCGDzbSJrGDGLmw/pFX
+  FPpVCm1zgYSb1v6Qnf3cgXa2f2wYGm17+zAVyIDpwryFru9yF/jJxE38z/DRsd9R
+  /88CAwEAAaM5MDcwCQYDVR0TBAIwADAdBgNVHQ4EFgQUsiNnXHtKeMYYcr4yJVmQ
+  WONL+IwwCwYDVR0PBAQDAgSwMA0GCSqGSIb3DQEBBQUAA4IBAQAlyN7kKo/NQCQ0
+  AOzZLZ3WAePvStkCFIJ53tsv5Kyo4pMAllv+BgPzzBt7qi605mFSL6zBd9uLou+W
+  Co3s48p1dy7CjjAfVQdmVNHF3MwXtfC2OEyvSQPi4xKR8iba8wa3xp9LVo1PuLpw
+  /6DsrChWw74HfsJN6qJOK684hJeT8lBYAUfiC3wD0owoPSg+XtyAAddisR+KV5Y1
+  NmVHuLtQcNTZy+gRht3ahJRMuC6QyLmkTsf+6MaenwAMkAgHdswGsJztOnNnBa3F
+  y0kCSWmK6D+x/SbfS6r7Ke07MRqziJdB9GuE1+0cIRuFh8EQ+LN6HXCKM5pon/GU
+  ycwMXfl0
+  -----END CERTIFICATE-----
+date: 2013-09-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: michael@grosser.it
+executables:
+- s3-meta-sync
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/s3-meta-sync
+- lib/s3_meta_sync.rb
+- lib/s3_meta_sync/version.rb
+homepage: http://github.com/grosser/s3_meta_sync
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.6
+signing_key: 
+specification_version: 4
+summary: Sync folders with s3 using a metadata file and md5 diffs
+test_files: []

metadata.gz.sig

@@ -0,0 +1 @@
+<]k1��le-*�Cs;�?㣻K�c\