s3fsr 1.7 → 1.8

data/bin/s3fsr

@@ -16,7 +16,8 @@ if ENV['AWS_ACCESS_KEY_ID'] == nil or ENV['AWS_SECRET_ACCESS_KEY'] == nil
 end
 
 root = BUCKET != nil ? SBucketDir.new(nil, BUCKET) : SBucketsDir.new
-s3fsr = MethodLogger.new(S3fsr.new(root))
+s3fsr = S3fsr.new(root)
+# s3fsr = MethodLogger.new(s3fsr)
 FuseFS.set_root s3fsr
 FuseFS.mount_under MOUNT #, "allow_other"
 begin

data/lib/aws-matt/s3/bucket.rb

@@ -303,6 +303,10 @@ module AWS
         when :deleted then object_cache.delete(object)
         end
       end
+
+      def truncated?
+        attributes['is_truncated']
+      end
       
       private        
         def build_contents!

data/lib/s3fsr.rb

@@ -9,13 +9,25 @@ S3SYNC_DIR_ETAG = 'd66759af42f282e1ba19144df2d405d0'
 S3SYNC_DIR_LENGTH = 38
 AWS::S3::Base.establish_connection!(:access_key_id => ENV['AWS_ACCESS_KEY_ID'], :secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'])
 
+# old key split name perf sucked, so optimize with a cached rindex/slice
+def last_part(key)
+  last_slash = key.rindex('/', -2)
+  drop = key.end_with?('/') ? 1 : 0
+  if last_slash then
+    key.slice(last_slash + 1, key.length - last_slash - 1 - drop)
+  else
+    key.slice(0, key.length - drop)
+  end
+end
+
 class SFile
   def initialize(parent, s3obj)
     @parent = parent
     @s3obj = s3obj
+    @name = last_part(@s3obj.key)
   end
   def name
-    @s3obj.key.split('/')[-1]
+    @name
   end
   def is_directory?
     false
@@ -51,25 +63,27 @@ class SBaseDir
     true
   end
   def content_deleted name
-    get_contents.delete_if { |i| i.name == name }
+    get_contents.delete name
   end
   def create_file child_key, content
     AWS::S3::S3Object.store(child_key, content, bucket)
-    get_contents << SFile.new(self, AWS::S3::S3Object.find(child_key, bucket))
+    f = SFile.new(self, AWS::S3::S3Object.find(child_key, bucket))
+    get_contents[f.name] = f
   end
   def create_dir child_key
     AWS::S3::S3Object.store(child_key + '/', '', bucket)
-    get_contents << SPrefixDir.new(self, child_key + '/')
+    d = SPrefixDir.new(self, child_key + '/')
+    get_contents[d.name] = d
   end
   def delete
     AWS::S3::S3Object.delete @key, bucket
     @parent.content_deleted name
   end
   def contents
-    get_contents.collect { |i| i.name }
+    get_contents.keys
   end
   def get(name)
-    get_contents.find { |i| i.name == name }
+    get_contents[name]
   end
   def size
     0
@@ -81,29 +95,41 @@ class SBaseDir
     def get_contents
       return @data if @data != nil
       puts "Loading '#{name}' from #{bucket}..."
-      @data = []
+      @data = {}
       marker = ''
       loop do
         s3_bucket = AWS::S3::Bucket.find(bucket, :prefix => prefix, :delimiter => '/', :marker => marker, :max_keys => 1000)
-        s3_bucket.object_cache.each do |s3_obj|
+
+        oc = s3_bucket.object_cache
+        cpc = s3_bucket.common_prefix_cache
+
+        cpc.reject { |p| p == '/' }.each do |prefix|
+          d = SPrefixDir.new(self, prefix)
+          @data[d.name] = d
+        end
+
+        oc.each do |s3_obj|
           # Technically we should use S3SYNC_DIR_LENGTH but aws-s3 decides it
           # needs to issue an HEAD request for every dir for that.
           if s3_obj.etag == S3SYNC_DIR_ETAG or s3_obj.key.end_with? S3ORGANIZER_DIR_SUFFIX
-            @data << SFakeDir.new(self, s3_obj.key)
+            d = SFakeDir.new(self, s3_obj.key)
+            @data[d.name] = d
           elsif s3_obj.key.end_with? '/'
             # We passed 'prefix/', delimiter='/', if we get any keys that
             # end with /, it's just a marker object for our current directory
             # so ignore it.
           else
-            @data << SFile.new(self, s3_obj)
+            f = SFile.new(self, s3_obj)
+            @data[f.name] = f
           end
         end
-        s3_bucket.common_prefix_cache.reject { |p| p == '/' }.each do |prefix|
-          hidden = SPrefixDir.new(self, prefix)
-          @data << hidden unless @data.find { |i| i.name == hidden.name }
-        end
-        break unless s3_bucket.object_cache.length > 0 && s3_bucket.object_cache.length % 1000 == 0
-        marker = s3_bucket.object_cache.last.key
+
+        break unless s3_bucket.truncated?
+
+        # find which of the object/prefix keys is alphabetically last
+        last_object_key = oc.size == 0 ? '' : oc.last.key
+        last_prefix_key = cpc.size == 0 ? '' : cpc.last
+        marker = last_object_key < last_prefix_key ? last_prefix_key : last_object_key
       end
       puts "done"
       @data
@@ -116,9 +142,10 @@ class SPrefixDir < SBaseDir
     @parent = parent
     @key = key
     @data = nil
+    @name = last_part(@key)
   end
   def name
-    @key.split('/')[-1]
+    @name
   end
   def bucket
     @parent.bucket
@@ -135,9 +162,10 @@ class SFakeDir < SBaseDir
     @parent = parent
     @key = key
     @data = nil
+    @name = strip_dir_suffix(last_part(@key))
   end
   def name
-    strip_dir_suffix @key.split('/')[-1]
+    @name
   end
   def bucket
     @parent.bucket
@@ -188,7 +216,8 @@ class SBucketsDir < SBaseDir
   end
   def create_dir child_key
     AWS::S3::Bucket.create(child_key)
-    get_contents << SBucketDir.new(self, child_key)
+    d = SBucketDir.new(self, child_key)
+    get_contents[d.name] = d
   end
   def delete
     raise 'cannot delete the buckets dir'
@@ -205,9 +234,10 @@ class SBucketsDir < SBaseDir
     def get_contents
       return @data if @data != nil
       puts "Loading buckets..."
-      @data = []
+      @data = {}
       AWS::S3::Bucket.list(true).each do |s3_bucket|
-        @data << SBucketDir.new(self, s3_bucket.name)
+        d = SBucketDir.new(self, s3_bucket.name)
+        @data[d.name] = d
       end
       puts "done"
       @data

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: s3fsr
 version: !ruby/object:Gem::Version 
-  hash: 1
+  hash: 31
   prerelease: false
   segments: 
   - 1
-  - 7
-  version: "1.7"
+  - 8
+  version: "1.8"
 platform: ruby
 authors: 
 - Stephen Haberman
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-13 00:00:00 -06:00
+date: 2011-01-25 00:00:00 -06:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency