s3ftp 0.0.1

data/Gemfile

@@ -0,0 +1,3 @@
+source :gemcutter
+
+gemspec

data/MIT-LICENSE

@@ -0,0 +1,21 @@
+Copyright (c) 2009 Peter Jones
+Copyright (c) 2009 James Healy
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,36 @@
+# s3ftp
+
+A mini-FTP server that persists all data to Amazon S3.
+
+## Installation
+
+    gem install s3ftp
+
+## Configuration
+
+1. Upload a passwd file to your S3 bucket. It should contain a single line
+   per user and be a CSV. It should look something like this
+
+   user1,password,y
+   user2,password,n
+
+   the third column indicates the users administrator status. Administrators can
+   see all files. Regular users are sandboxed to their own directory.
+
+2. Create a config.rb file that looks something like this
+
+    require 's3ftp'
+
+    AWS_KEY    = 'foo'
+    AWS_SECRET = 'bar'
+    AWS_BUCKET = 'my-ftp-bucket'
+
+    driver      S3FTP::Driver
+    driver_args AWS_KEY, AWS_SECRET, AWS_BUCKET
+
+3. As root, run 'em-ftpd config.rb'
+
+## License
+
+This library is distributed under the terms of the MIT License. See the included file for
+more detail.

data/lib/happening/s3/bucket.rb

@@ -0,0 +1,116 @@
+require 'uri'
+require 'cgi'
+
+module Happening
+  module S3
+    class Bucket
+      include Utils
+    
+      REQUIRED_FIELDS = [:server]
+      VALID_HEADERS = ['Cache-Control', 'Content-Disposition', 'Content-Encoding', 'Content-Length', 'Content-MD5', 'Content-Type', 'Expect', 'Expires']
+    
+      attr_accessor :bucket, :options
+
+      def initialize(bucket, options = {})
+        @options = {
+          :timeout => 10,
+          :server => 's3.amazonaws.com',
+          :protocol => 'https',
+          :aws_access_key_id => nil,
+          :aws_secret_access_key => nil,
+          :retry_count => 4,
+          :permissions => 'private',
+          :ssl         => Happening::S3.ssl_options,
+          :prefix      => nil,
+          :delimiter   => nil
+        }.update(symbolize_keys(options))
+        assert_valid_keys(options, :timeout, :server, :protocol, :aws_access_key_id, :aws_secret_access_key, :retry_count, :permissions, :ssl, :prefix, :delimiter)
+        @bucket = bucket.to_s
+      
+        validate
+      end
+    
+      def get(request_options = {}, &blk)
+        headers = needs_to_sign? ? aws.sign("GET", path_with_query) : {}
+        request_options[:on_success] = blk if blk
+        request_options.update(:headers => headers)
+        Happening::S3::Request.new(:get, url, {:ssl => options[:ssl]}.update(request_options)).execute
+      end
+    
+      def url
+        URI::Generic.new(options[:protocol], nil, server, port, nil, path(!dns_bucket?), nil, query_string, nil).to_s
+      end
+      
+      def server
+        dns_bucket? ? "#{bucket}.#{options[:server]}" : options[:server]
+      end
+      
+      def path(with_bucket=true)
+        with_bucket ? "/#{bucket}/" : "/"
+      end
+
+      def path_with_query(with_bucket=true)
+        base  = path(with_bucket)
+        query = query_string
+        query ? "#{base}?#{query}" : base
+      end
+    
+    protected
+
+      def query_string
+        if @options[:prefix] || @options[:delimiter]
+          str  = ""
+          str += "prefix=#{CGI::escape(@options[:prefix])}&"       if @options[:prefix]
+          str += "delimiter=#{CGI::escape(@options[:delimiter])}&" if @options[:delimiter]
+          str.gsub(/&\Z/,"")
+        else
+          nil
+        end
+      end
+        
+      def needs_to_sign?
+        present?(options[:aws_access_key_id])
+      end
+    
+      def dns_bucket?
+        # http://docs.amazonwebservices.com/AmazonS3/2006-03-01/index.html?BucketRestrictions.html
+        return false unless (3..63) === bucket.size
+        bucket.split('.').each do |component|
+          return false unless component[/^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/]
+        end
+        true
+      end
+    
+      def port
+        (options[:protocol].to_s == 'https') ? 443 : 80
+      end
+    
+      def validate
+        raise ArgumentError, "need a bucket name" unless present?(bucket)
+      
+        REQUIRED_FIELDS.each do |field|
+          raise ArgumentError, "need field #{field}" unless present?(options[field])
+        end
+      
+        raise ArgumentError, "unknown protocol #{options[:protocol]}" unless ['http', 'https'].include?(options[:protocol])
+      end
+      
+      def aws
+        @aws ||= Happening::AWS.new(options[:aws_access_key_id], options[:aws_secret_access_key])
+      end
+      
+      def construct_aws_headers(http_method, headers = {})
+        unless headers.keys.all?{|header| VALID_HEADERS.include?(header) || header.to_s.match(/\Ax-amz-/) }
+          raise ArgumentError, "invalid headers. All headers must either one of #{VALID_HEADERS} or start with 'x-amz-'" 
+        end
+        
+        permissions = options[:permissions] != 'private' ? {'x-amz-acl' => options[:permissions] } : {}
+        headers.update(permissions)
+        headers.update({'url' => path})
+        
+        headers = needs_to_sign? ? aws.sign(http_method, path, headers) : headers
+      end
+    
+    end
+  end
+end

data/lib/s3ftp.rb

@@ -0,0 +1,11 @@
+require 'em-ftpd'
+require 'happening'
+require 'csv'
+require 'nokogiri'
+
+# load our happening extension for interacting with buckets. Hopefully
+# it will be accepted upstream soon and can be dropped
+require 'happening/s3/bucket'
+
+# and now the secret sauce
+require 's3ftp/driver'

data/lib/s3ftp/driver.rb

@@ -0,0 +1,245 @@
+# coding: utf-8
+
+require 'tempfile'
+
+module S3FTP
+  class Driver
+
+    USER  = 0
+    PASS  = 1
+    ADMIN = 2
+
+    def initialize(key, secret, bucket)
+      @aws_key, @aws_secret, @aws_bucket = key, secret, bucket
+    end
+
+    def change_dir(path, &block)
+      prefix = scoped_path(path)
+
+      item = Happening::S3::Bucket.new(@aws_bucket, :aws_access_key_id => @aws_key, :aws_secret_access_key => @aws_secret, :prefix => prefix, :delimiter => "/")
+      item.get do |response|
+        yield contains_directory?(response.response, prefix)
+      end
+    end
+
+    def dir_contents(path, &block)
+      prefix = scoped_path_with_trailing_slash(path)
+
+      on_error   = Proc.new {|response| yield false }
+      on_success = Proc.new {|response| yield response.response_header["CONTENT_LENGTH"].to_i }
+
+      item = Happening::S3::Bucket.new(@aws_bucket, :aws_access_key_id => @aws_key, :aws_secret_access_key => @aws_secret, :prefix => prefix, :delimiter => "/")
+      item.get do |response|
+        yield parse_bucket_list(response.response)
+      end
+    end
+
+    def authenticate(user, pass, &block)
+      download_passwd_file do |passwd|
+        @users = extract_users(passwd)
+
+        if @users[user] && @users[user][:pass] == pass
+          @user = user
+          yield true
+        else
+          yield false
+        end
+      end
+    end
+
+    def bytes(path, &block)
+      key = scoped_path(path)
+
+      on_error   = Proc.new {|response| yield false }
+      on_success = Proc.new {|response| yield response.response_header["CONTENT_LENGTH"].to_i }
+
+      item = Happening::S3::Item.new(@aws_bucket, key, :aws_access_key_id => @aws_key, :aws_secret_access_key => @aws_secret)
+      item.head(:retry_count => 0, :on_success => on_success, :on_error => on_error)
+    end
+
+    def get_file(path, &block)
+      key = scoped_path(path)
+
+      # open a tempfile to store the file as it's downloaded from S3.
+      # em-ftpd will close it for us
+      tmpfile = Tempfile.new("s3ftp")
+
+      on_error   = Proc.new {|response| yield false }
+      on_success = Proc.new {|response|
+        tmpfile.flush
+        tmpfile.seek(0)
+        yield tmpfile
+      }
+
+      item = Happening::S3::Item.new(@aws_bucket, key, :aws_access_key_id => @aws_key, :aws_secret_access_key => @aws_secret)
+      item.get(:retry_count => 1, :on_success => on_success, :on_error => on_error).stream do |chunk|
+        tmpfile.write chunk
+      end
+    end
+
+    def put_file(path, tmp_path, &block)
+      key = scoped_path(path)
+
+      bytes      = File.size(tmp_path)
+      on_error   = Proc.new {|response| yield false }
+      on_success = Proc.new {|response| yield bytes  }
+
+      item = Happening::S3::Item.new(@aws_bucket, key, :aws_access_key_id => @aws_key, :aws_secret_access_key => @aws_secret)
+      item.put(File.binread(tmp_path), :retry_count => 0, :on_success => on_success, :on_error => on_error)
+    end
+
+    def delete_file(path, &block)
+      key = scoped_path(path)
+
+      on_error   = Proc.new {|response| yield false }
+      on_success = Proc.new {|response| yield true  }
+
+      item = Happening::S3::Item.new(@aws_bucket, key, :aws_access_key_id => @aws_key, :aws_secret_access_key => @aws_secret)
+      item.delete(:retry_count => 1, :on_success => on_success, :on_error => on_error)
+    end
+
+    def delete_dir(path, &block)
+      prefix = scoped_path(path)
+
+      on_error   = Proc.new {|response| yield false }
+
+      item = Happening::S3::Bucket.new(@aws_bucket, :aws_access_key_id => @aws_key, :aws_secret_access_key => @aws_secret, :prefix => prefix)
+      item.get(:on_error => on_error) do |response|
+        keys = bucket_list_to_full_keys(response.response)
+        delete_object = Proc.new { |key, iter|
+          item = Happening::S3::Item.new(@aws_bucket, key, :aws_access_key_id => @aws_key, :aws_secret_access_key => @aws_secret)
+          item.delete(:retry_count => 1, :on_error => on_error) do |response|
+            iter.next
+          end
+        }
+        on_complete = Proc.new { yield true }
+
+        EM::Iterator.new(keys, 5).each(delete_object, on_complete)
+      end
+    end
+
+    def rename(from, to, &block)
+      source_key = scoped_path(from)
+      source_obj = @aws_bucket + "/" + source_key
+      dest_key   = scoped_path(to)
+
+      on_error   = Proc.new {|response| yield false }
+      on_success = Proc.new {|response| yield true  }
+
+      item = Happening::S3::Item.new(@aws_bucket, dest_key, :aws_access_key_id => @aws_key, :aws_secret_access_key => @aws_secret)
+      item.put(nil, :retry_count => 1, :on_error => on_error, :headers => {"x-amz-copy-source" => source_obj}) do |response|
+        item = Happening::S3::Item.new(@aws_bucket, source_key, :aws_access_key_id => @aws_key, :aws_secret_access_key => @aws_secret)
+        item.delete(:retry_count => 1, :on_success => on_success, :on_error => on_error)
+      end
+    end
+
+    def make_dir(path, &block)
+      key = scoped_path(path) + "/.dir"
+
+      on_error   = Proc.new {|response| yield false }
+      on_success = Proc.new {|response| yield true  }
+
+      item = Happening::S3::Item.new(@aws_bucket, key, :aws_access_key_id => @aws_key, :aws_secret_access_key => @aws_secret)
+      item.put("", :retry_count => 0, :on_success => on_success, :on_error => on_error)
+    end
+
+    private
+
+    def extract_users(passwd)
+      users  = {}
+      CSV.parse(passwd).each { |row|
+        users[row[USER]] = {
+          :pass  => row[PASS],
+          :admin => row[ADMIN].to_s.upcase == "Y"
+        }
+      }
+      users
+    end
+
+    def download_passwd_file(&block)
+      on_error = Proc.new { |response|
+        yield false
+      }
+      on_success = Proc.new { |response|
+        yield response.response
+      }
+      item = Happening::S3::Item.new(@aws_bucket, 'passwd', :aws_access_key_id => @aws_key, :aws_secret_access_key => @aws_secret)
+      item.get(:on_success => on_success, :on_error => on_error)
+    end
+
+    def admin?
+      @users[@user] && @users[@user][:admin]
+    end
+
+    def scoped_path_with_trailing_slash(path)
+      path  = scoped_path(path)
+      path += "/" if path[-1,1] != "/"
+      path == "/" ? nil : path
+    end
+
+    def scoped_path(path)
+      path = "" if path == "/"
+
+      if admin?
+        File.join("/", path)[1,1024]
+      else
+        File.join("/", @user, path)[1,1024]
+      end
+    end
+
+    def bucket_list_to_full_keys(xml)
+      doc = Nokogiri::XML(xml)
+      doc.remove_namespaces!
+      doc.xpath('//Contents').map { |node|
+        node.xpath('./Key').first.content
+      }
+    end
+
+    def contains_directory?(xml, path)
+      doc = Nokogiri::XML(xml)
+      doc.remove_namespaces!
+      prefix = doc.xpath('/ListBucketResult/Prefix').first.content
+
+      doc.xpath('//CommonPrefixes').any? { |node|
+        name  = node.xpath('./Prefix').first.content
+
+        name.to_s.start_with?(prefix)
+      }
+    end
+
+    def parse_bucket_list(xml)
+      doc = Nokogiri::XML(xml)
+      doc.remove_namespaces!
+      prefix = doc.xpath('/ListBucketResult/Prefix').first.content
+      files = doc.xpath('//Contents').select { |node|
+        name  = node.xpath('./Key').first.content
+        bytes = node.xpath('./Size').first.content.to_i
+        name != prefix && bytes > 0
+      }.map { |node|
+        name  = node.xpath('./Key').first.content
+        bytes = node.xpath('./Size').first.content
+        file_item(name[prefix.size, 1024], bytes)
+      }
+      dirs = doc.xpath('//CommonPrefixes').select { |node|
+        node.xpath('./Prefix').first.content != prefix + "/"
+      }.map { |node|
+        name  = node.xpath('./Prefix').first.content
+        dir_item(name[prefix.size, 1024].tr("/",""))
+      }
+      default_dirs + dirs + files
+    end
+
+    def default_dirs
+      [dir_item("."), dir_item("..")]
+    end
+
+    def dir_item(name)
+      EM::FTPD::DirectoryItem.new(:name => name, :directory => true, :size => 0)
+    end
+
+    def file_item(name, bytes)
+      EM::FTPD::DirectoryItem.new(:name => name, :directory => false, :size => bytes)
+    end
+
+  end
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: s3ftp
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- James Healy
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-11-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &21376320 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *21376320
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &21375560 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :development
+  prerelease: false
+  version_requirements: *21375560
+- !ruby/object:Gem::Dependency
+  name: em-ftpd
+  requirement: &21374880 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *21374880
+- !ruby/object:Gem::Dependency
+  name: happening
+  requirement: &21374100 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *21374100
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &21373140 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *21373140
+description: Run an FTP server that persists all data to an Amazon S3 bucket
+email:
+- jimmy@deefa.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.markdown
+- MIT-LICENSE
+files:
+- lib/happening/s3/bucket.rb
+- lib/s3ftp.rb
+- lib/s3ftp/driver.rb
+- Gemfile
+- README.markdown
+- MIT-LICENSE
+homepage: http://github.com/yob/s3ftp
+licenses: []
+post_install_message: 
+rdoc_options:
+- --title
+- S3-FTP Documentation
+- --main
+- README.markdown
+- -q
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: An FTP proxy in front of an Amazon S3 bucket
+test_files: []