s3fsr 1.4

data/lib/aws-matt/s3/parsing.rb

@@ -0,0 +1,99 @@
+#:stopdoc:
+module AWS
+  module S3
+    module Parsing
+      class << self
+        def parser=(parsing_library)
+          XmlParser.parsing_library = parsing_library
+        end
+        
+        def parser
+          XmlParser.parsing_library
+        end
+      end
+      
+      module Typecasting
+        def typecast(object)
+          case object
+          when Hash
+            typecast_hash(object)
+          when Array
+            object.map {|element| typecast(element)}
+          when String
+            CoercibleString.coerce(object)
+          else
+            object
+          end
+        end
+        
+        def typecast_hash(hash)
+          if content = hash['__content__']  
+            typecast(content)
+          else
+            keys   = hash.keys.map {|key| key.underscore}
+            values = hash.values.map {|value| typecast(value)}
+            keys.inject({}) do |new_hash, key|
+              new_hash[key] = values.slice!(0)
+              new_hash
+            end
+          end
+        end
+      end
+      
+      class XmlParser < Hash
+        include Typecasting
+        
+        class << self
+          attr_accessor :parsing_library
+        end
+        
+        attr_reader :body, :xml_in, :root
+        
+        def initialize(body)
+          @body = body
+          unless body.strip.empty?
+            parse
+            set_root
+            typecast_xml_in
+          end
+        end
+      
+        private
+      
+          def parse
+            @xml_in = self.class.parsing_library.xml_in(body, parsing_options)
+          end
+          
+          def parsing_options
+            {
+              # Includes the enclosing tag as the top level key
+              'keeproot'      => true, 
+              # Makes tag value available via the '__content__' key
+              'contentkey'    => '__content__', 
+              # Always parse tags into a hash, even when there are no attributes 
+              # (unless there is also no value, in which case it is nil)
+              'forcecontent'  => true, 
+              # If a tag is empty, makes its content nil
+              'suppressempty' => nil,
+              # Force nested elements to be put into an array, even if there is only one of them
+              'forcearray'    => ['Contents', 'Bucket', 'Grant', 'CommonPrefixes']
+            }
+          end
+          
+          def set_root
+            @root = @xml_in.keys.first.underscore
+          end
+          
+          def typecast_xml_in
+            typecast_xml = {}
+            @xml_in.dup.each do |key, value| # Some typecasting is destructive so we dup
+              typecast_xml[key.underscore] = typecast(value)
+            end
+            # An empty body will try to update with a string so only update if the result is a hash
+            update(typecast_xml[root]) if typecast_xml[root].is_a?(Hash)
+          end
+      end
+    end
+  end
+end
+#:startdoc:

data/lib/aws-matt/s3/response.rb

@@ -0,0 +1,180 @@
+#:stopdoc:
+module AWS
+  module S3
+    class Base
+      class Response < String  
+        attr_reader :response, :body, :parsed
+        def initialize(response)
+          @response = response
+          @body     = response.body.to_s
+          super(body)
+        end
+    
+        def headers
+          headers = {}
+          response.each do |header, value|
+            headers[header] = value
+          end
+          headers
+        end
+        memoized :headers
+    
+        def [](header)
+          headers[header]
+        end
+    
+        def each(&block)
+          headers.each(&block)
+        end
+    
+        def code
+          response.code.to_i
+        end
+    
+        {:success      => 200..299, :redirect     => 300..399,
+         :client_error => 400..499, :server_error => 500..599}.each do |result, code_range|
+          class_eval(<<-EVAL, __FILE__, __LINE__)
+            def #{result}? 
+              return false unless response
+              (#{code_range}).include? code
+            end
+          EVAL
+        end
+    
+        def error?
+          !success? && response['content-type'] == 'application/xml' && parsed.root == 'error'
+        end
+    
+        def error
+          Error.new(parsed, self)
+        end
+        memoized :error
+    
+        def parsed()
+          # XmlSimple is picky about what kind of object it parses, so we pass in body rather than self
+          Parsing::XmlParser.new(body)
+        end
+        memoized :parsed
+    
+        def inspect
+          "#<%s:0x%s %s %s>" % [self.class, object_id, response.code, response.message]
+        end
+      end
+    end
+    
+    class Bucket  
+      class Response < Base::Response
+        def bucket
+          parsed
+        end
+      end
+    end
+  
+    class S3Object
+      class Response < Base::Response
+        def etag
+          headers['etag'][1...-1]
+        end
+      end
+    end
+
+    class Service
+      class Response < Base::Response
+        def empty?
+          parsed['buckets'].nil?
+        end
+  
+        def buckets
+          parsed['buckets']['bucket'] || []
+        end
+      end
+    end
+    
+    module ACL
+      class Policy
+        class Response < Base::Response
+          alias_method :policy, :parsed
+        end
+      end
+    end
+    
+    # Requests whose response code is between 300 and 599 and contain an <Error></Error> in their body
+    # are wrapped in an Error::Response. This Error::Response contains an Error object which raises an exception
+    # that corresponds to the error in the response body. The exception object contains the ErrorResponse, so
+    # in all cases where a request happens, you can rescue ResponseError and have access to the ErrorResponse and
+    # its Error object which contains information about the ResponseError.
+    #
+    #   begin
+    #     Bucket.create(..)
+    #   rescue ResponseError => exception
+    #    exception.response
+    #    # => <Error::Response>
+    #    exception.response.error
+    #    # => <Error>
+    #   end
+    class Error
+      class Response < Base::Response
+        def error? 
+          true
+        end
+      
+        def inspect
+          "#<%s:0x%s %s %s: '%s'>" % [self.class.name, object_id, response.code, error.code, error.message]
+        end
+      end
+    end
+
+    # Guess response class name from current class name. If the guessed response class doesn't exist
+    # do the same thing to the current class's parent class, up the inheritance heirarchy until either
+    # a response class is found or until we get to the top of the heirarchy in which case we just use 
+    # the the Base response class.
+    #
+    # Important: This implemantation assumes that the Base class has a corresponding Base::Response.
+    class FindResponseClass #:nodoc:
+      class << self
+        def for(start)
+          new(start).find
+        end
+      end
+  
+      def initialize(start)
+        @container     = AWS::S3
+        @current_class = start
+      end
+  
+      def find
+        self.current_class = current_class.superclass until response_class_found?            
+        target.const_get(class_to_find)
+      end
+  
+      private
+        attr_reader :container
+        attr_accessor :current_class
+        
+        def target
+          container.const_get(current_name)
+        end
+        
+        def target?
+          container.const_defined?(current_name)
+        end
+        
+        def response_class_found?
+          target? && target.const_defined?(class_to_find)
+        end
+        
+        def class_to_find
+          :Response
+        end
+        
+        def current_name
+          truncate(current_class)
+        end
+        
+        def truncate(klass)
+          klass.name[/[^:]+$/]
+        end
+      end
+  end
+end
+#:startdoc:

data/lib/aws-matt/s3/service.rb

@@ -0,0 +1,51 @@
+module AWS
+  module S3
+    # The service lets you find out general information about your account, like what buckets you have. 
+    # 
+    #   Service.buckets
+    #   # => []
+    class Service < Base
+      @@response = nil #:nodoc:
+      
+      class << self
+        # List all your buckets.
+        #
+        #   Service.buckets
+        #   # => []
+        #
+        # For performance reasons, the bucket list will be cached. If you want avoid all caching, pass the <tt>:reload</tt> 
+        # as an argument:
+        #
+        #   Service.buckets(:reload)
+        def buckets
+          response = get('/')
+          if response.empty?
+            []
+          else
+            response.buckets.map {|attributes| Bucket.new(attributes)}
+          end
+        end
+        memoized :buckets
+        
+        # Sometimes methods that make requests to the S3 servers return some object, like a Bucket or an S3Object. 
+        # Othertimes they return just <tt>true</tt>. Other times they raise an exception that you may want to rescue. Despite all these 
+        # possible outcomes, every method that makes a request stores its response object for you in Service.response. You can always 
+        # get to the last request's response via Service.response.
+        # 
+        #   objects = Bucket.objects('jukebox')
+        #   Service.response.success?
+        #   # => true
+        #
+        # This is also useful when an error exception is raised in the console which you weren't expecting. You can 
+        # root around in the response to get more details of what might have gone wrong.
+        def response
+          @@response
+        end
+        
+        def response=(response) #:nodoc:
+          @@response = response
+        end
+      end
+    end
+  end
+end

data/lib/aws-matt/s3/version.rb

@@ -0,0 +1,12 @@
+module AWS
+  module S3
+    module VERSION #:nodoc:
+      MAJOR    = '0'
+      MINOR    = '5'
+      TINY     = '1' 
+      BETA     = Time.now.to_i.to_s
+    end
+    
+    Version = [VERSION::MAJOR, VERSION::MINOR, VERSION::TINY, VERSION::BETA].compact * '.'
+  end
+end

data/lib/aws-matt/s3.rb

@@ -0,0 +1,61 @@
+require 'base64'
+require 'cgi'
+require 'uri'
+require 'openssl'
+require 'digest/sha1'
+require 'net/https'
+require 'time'
+require 'date'
+require 'open-uri'
+
+$:.unshift(File.dirname(__FILE__))
+require 's3/extensions'
+require_library_or_gem 'builder' unless defined? Builder
+require_library_or_gem 'mime/types' unless defined? MIME::Types
+
+require 's3/base'
+require 's3/version'
+require 's3/parsing'
+require 's3/acl'
+require 's3/logging'
+require 's3/bittorrent'
+require 's3/service'
+require 's3/owner'
+require 's3/bucket'
+require 's3/object'
+require 's3/error'
+require 's3/exceptions'
+require 's3/connection'
+require 's3/authentication'
+require 's3/response'
+
+AWS::S3::Base.class_eval do
+  include AWS::S3::Connection::Management
+end
+
+AWS::S3::Bucket.class_eval do
+  include AWS::S3::Logging::Management
+  include AWS::S3::ACL::Bucket
+end
+
+AWS::S3::S3Object.class_eval do
+  include AWS::S3::ACL::S3Object
+  include AWS::S3::BitTorrent
+end
+
+require_library_or_gem 'xmlsimple' unless defined? XmlSimple
+# If libxml is installed, we use the FasterXmlSimple library, that provides most of the functionality of XmlSimple
+# except it uses the xml/libxml library for xml parsing (rather than REXML). If libxml isn't installed, we just fall back on
+# XmlSimple.
+AWS::S3::Parsing.parser =
+  begin
+    require_library_or_gem 'xml/libxml'
+    # Older version of libxml aren't stable (bus error when requesting attributes that don't exist) so we
+    # have to use a version greater than '0.3.8.2'.
+    raise LoadError unless XML::Parser::VERSION > '0.3.8.2'
+    $:.push(File.join(File.dirname(__FILE__), '..', '..', 'support', 'faster-xml-simple', 'lib'))
+    require_library_or_gem 'faster_xml_simple' 
+    FasterXmlSimple
+  rescue LoadError
+    XmlSimple
+  end

data/lib/s3fsr.rb

@@ -0,0 +1,296 @@
+#!/usr/bin/env ruby
+
+require 'fusefs'
+require File.dirname(__FILE__) + '/aws-matt/s3'
+
+S3ORGANIZER_DIR_SUFFIX = '_$folder$'
+S3SYNC_DIR_CONTENTS = '{E40327BF-517A-46e8-A6C3-AF51BC263F59}'
+S3SYNC_DIR_ETAG = 'd66759af42f282e1ba19144df2d405d0'
+S3SYNC_DIR_LENGTH = 38
+AWS::S3::Base.establish_connection!(:access_key_id => ENV['AWS_ACCESS_KEY_ID'], :secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'])
+
+class SFile
+  def initialize(parent, s3obj)
+    @parent = parent
+    @s3obj = s3obj
+  end
+  def name
+    @s3obj.key.split('/')[-1]
+  end
+  def is_directory?
+    false
+  end
+  def is_file?
+    true
+  end
+  def value
+    @s3obj.value(:reload)
+  end
+  def write data
+    AWS::S3::S3Object.store @s3obj.key, data, @s3obj.bucket.name, @s3obj.about.to_headers
+  end
+  def delete
+    AWS::S3::S3Object.delete @s3obj.key, @s3obj.bucket.name
+    @parent.content_deleted name
+  end
+  def size
+    @s3obj.content_length
+  end
+  def touch
+  end
+end
+
+class SBaseDir
+  def is_directory?
+    true
+  end
+  def is_file?
+    false
+  end
+  def can_write_files?
+    true
+  end
+  def content_deleted name
+    get_contents.delete_if { |i| i.name == name }
+  end
+  def create_file child_key, content
+    AWS::S3::S3Object.store(child_key, content, bucket)
+    get_contents << SFile.new(self, AWS::S3::S3Object.find(child_key, bucket))
+  end
+  def create_dir child_key
+    AWS::S3::S3Object.store(child_key, S3SYNC_DIR_CONTENTS, bucket)
+    get_contents << SFakeDir.new(self, child_key)
+  end
+  def delete
+    AWS::S3::S3Object.delete @key, bucket
+    @parent.content_deleted name
+  end
+  def contents
+    get_contents.collect { |i| i.name }
+  end
+  def get(name)
+    get_contents.find { |i| i.name == name }
+  end
+  def size
+    0
+  end
+  def touch
+    @data = nil
+  end
+  private
+    def get_contents
+      return @data if @data != nil
+      puts "Loading '#{name}' from #{bucket}..."
+      @data = []
+      marker = ''
+      loop do
+        s3_bucket = AWS::S3::Bucket.find(bucket, :prefix => prefix, :delimiter => '/', :marker => marker, :max_keys => 1000)
+        s3_bucket.object_cache.each do |s3_obj|
+          # Technically we should use S3SYNC_DIR_LENGTH but aws-s3 decides it
+          # needs to issue an HEAD request for every dir for that.
+          if s3_obj.etag == S3SYNC_DIR_ETAG or s3_obj.key.end_with? S3ORGANIZER_DIR_SUFFIX
+            @data << SFakeDir.new(self, s3_obj.key)
+          else
+            @data << SFile.new(self, s3_obj)
+          end
+        end
+        s3_bucket.common_prefix_cache.reject { |p| p == '/' }.each do |prefix|
+          hidden = SFakeDir.new(self, prefix[0..-2])
+          @data << hidden unless @data.find { |i| i.name == hidden.name }
+        end
+        break unless s3_bucket.object_cache.length > 0 && s3_bucket.object_cache.length % 1000 == 0
+        marker = s3_bucket.object_cache.last.key
+      end
+      puts "done"
+      @data
+    end
+end
+
+class SFakeDir < SBaseDir
+  def initialize(parent, key)
+    @parent = parent
+    @key = key
+    @data = nil
+  end
+  def name
+    strip_dir_suffix @key.split('/')[-1]
+  end
+  def bucket
+    @parent.bucket
+  end
+  private
+    def prefix
+      strip_dir_suffix(@key) + '/'
+    end
+    def strip_dir_suffix str
+      str.end_with?(S3ORGANIZER_DIR_SUFFIX) ? str[0..-10] : str
+    end
+end
+
+class SBucketDir < SBaseDir
+  def initialize(parent, bucket_name)
+    @parent = parent # either SBucketsDir or nil
+    @bucket_name = bucket_name
+  end
+  def delete
+    raise "cannot delete bucket dir #{name}" if @parent == nil
+    AWS::S3::Bucket.delete name
+    @parent.content_deleted name
+  end
+  def bucket
+    @bucket_name
+  end
+  def name
+    @bucket_name
+  end
+  def path_to_key child_key
+    child_key
+  end
+  private
+    def prefix
+      ""
+    end
+end
+
+class SBucketsDir < SBaseDir
+  def initialize
+    @data = nil
+  end
+  def can_write_files?
+    false
+  end
+  def create_file child_key, content
+    raise 'cannot create files outside of a bucket'
+  end
+  def create_dir child_key
+    AWS::S3::Bucket.create(child_key)
+    get_contents << SBucketDir.new(self, child_key)
+  end
+  def delete
+    raise 'cannot delete the buckets dir'
+  end
+  def name
+    return 'buckets'
+  end
+  def path_to_key child_key
+    # if i == -1, this is for creating a bucket
+    i = child_key.index('/')
+    i != nil ? child_key[i+1..-1] : child_key
+  end
+  private
+    def get_contents
+      return @data if @data != nil
+      puts "Loading buckets..."
+      @data = []
+      AWS::S3::Bucket.list(true).each do |s3_bucket|
+        @data << SBucketDir.new(self, s3_bucket.name)
+      end
+      puts "done"
+      @data
+    end
+end
+
+class S3fsr
+  def initialize(root)
+    @root = root
+  end
+  def contents(path)
+    o = get_object(path)
+    o == nil ? "" : o.contents
+  end
+  def directory?(path)
+    o = get_object(path) ; o == nil ? false : o.is_directory?
+  end
+  def file?(path)
+    o = get_object(path) ; o == nil ? false : o.is_file?
+  end
+  def executable?(path)
+    false
+  end
+  def size(path)
+    get_object(path).size
+  end
+  def read_file(path)
+    get_object(path).value
+  end
+  def can_write?(path)
+    o = get_object(path)
+    if o != nil
+      o.is_file?
+    else
+      d = get_parent_object(path) ; d == nil ? false : d.can_write_files?
+    end
+  end
+  def write_to(path, data)
+    o = get_object(path)
+    if o != nil
+      o.write data
+    else
+      d = get_parent_object(path)
+      if d != nil
+        child_key = @root.path_to_key path[1..-1]
+        d.create_file(child_key, data)
+      end
+    end
+  end
+  def can_delete?(path)
+    o = get_object(path) ; o == nil ? false : o.is_file?
+  end
+  def delete(path)
+    get_object(path).delete
+  end
+  def can_mkdir?(path)
+    return false if get_object(path) != nil
+    get_parent_object(path).is_directory?
+  end
+  def mkdir(path)
+    child_key = @root.path_to_key path[1..-1]
+    get_parent_object(path).create_dir(child_key)
+  end
+  def can_rmdir?(path)
+    return false if path == '/'
+    return false unless get_object(path).is_directory?
+    get_object(path).contents.length == 0
+  end
+  def rmdir(path)
+    get_object(path).delete
+  end
+  def touch(path)
+    get_object(path).touch
+  end
+  private
+    def get_parent_object(path)
+      # path[1..-1] because '/'.split('/') -> ['']
+      get_object('/' + (path[1..-1].split('/')[0..-2].join('/')))
+    end
+    def get_object(path)
+      curr = @root
+      # path[1..-1] because '/'.split('/') -> ['']
+      path[1..-1].split('/').each do |part|
+        curr = curr.get(part)
+      end
+      curr
+    end
+end
+
+class MethodLogger
+  instance_methods.each { |m| undef_method m unless m =~ /^__/ }
+  def initialize(obj)
+    @obj = obj
+  end
+  def method_missing(sym, *args, &block)
+    begin
+      puts "#{sym}(#{args})" unless sym == :respond_to? or sym == :write_to
+      puts "#{sym}(#{args[0].length})" if sym == :write_to
+      result = @obj.__send__(sym, *args, &block)
+      puts "    #{result}" unless sym == :respond_to? or sym == :read_file
+      puts "    #{result.length}" if sym == :read_file
+      result
+    rescue => e
+      puts "    #{e.inspect}"
+      puts "    #{e.backtrace}"
+      raise $?
+    end
+  end
+end
+