sublimate 0.1.0

data/lib/sublimate/chunked_file.rb

@@ -0,0 +1,22 @@
+module Sublimate
+  #todo: This seems like it could just be IO objects that map to portions of a file
+  class ChunkedFile
+    def initialize(path, opts={})
+      @path = path
+      @chunk_size = opts[:chunk_size] || (1024 * 1024 * 6)
+    end
+
+    def each_chunk
+      counter = 0
+      offset = 0
+      file_size = File.size(@path)
+      while offset < file_size
+        data = IO.read(@path, @chunk_size, offset)
+        details = {:counter => counter }
+        yield(data, details)
+        offset += @chunk_size
+        counter += 1
+      end
+    end
+  end
+end

data/lib/sublimate/uploader.rb

@@ -0,0 +1,63 @@
+module Sublimate
+  class Uploader
+    def initialize(attrs)
+      @attrs = attrs
+
+      fog_attrs = attrs.clone
+      fog_attrs.delete(:auto_create_bucket)
+      fog_attrs.delete(:multipart_chunk_size)
+      fog_attrs.delete(:bucket)
+
+      @fog_storage = Fog::Storage.new(fog_attrs)
+    end
+
+    def storage
+      @fog_storage
+    end
+
+    def store_file(path, opts = {})
+      opts = {:key => path}.merge(opts)
+      size = File.size(path)
+
+      bucket = @fog_storage.directories.get(opts[:bucket])
+      if bucket.nil? && @attrs[:auto_create_bucket]
+        bucket = @fog_storage.directories.create(:key => opts[:bucket])
+      elsif bucket.nil?
+        raise "bucket not found"
+      end
+
+      m = "do_multi_#{@attrs[:provider].downcase}"
+      if respond_to?(m)
+        chunked = ChunkedFile.new(path, :chunk_size => @attrs[:multipart_chunk_size])
+        send(m, bucket, chunked, opts)
+      else
+        opts[:body] = File.open(path)
+        bucket.files.create(opts)
+      end
+    end
+
+    def do_multi_aws(bucket, chunked, opts)
+      multi = @fog_storage.initiate_multipart_upload(opts[:bucket], opts[:key])
+      upload_id = multi.body["UploadId"]
+      results = []
+      chunked.each_chunk do |data, details|
+        part = @fog_storage.upload_part(opts[:bucket], opts[:key], upload_id, details[:counter] + 1, data)
+        etag = part.headers['ETag']
+        results << etag
+      end
+
+      completed_upload = @fog_storage.complete_multipart_upload(opts[:bucket], opts[:key], upload_id, results)
+    end
+
+    def do_multi_rackspace(bucket, chunked, opts)
+      count = 1
+      chunked.each_chunk do |data, details|
+        key = opts[:key] + "/#{count}"
+        bucket.files.create(:key => key, :body => data)
+        count += 1
+      end
+      result = @fog_storage.put_object_manifest(bucket.identity, opts[:key])
+    end
+
+  end
+end

data/lib/sublimate.rb

@@ -0,0 +1,8 @@
+require 'fog'
+# rackspace cloud: http://www.rackspace.com/knowledge_center/index.php/Does_Cloud_Files_support_large_file_transfer
+# s3: https://gist.github.com/908875
+# Google Storage apparently allows single PUTs up to 50GB
+module Sublimate
+end
+require 'sublimate/chunked_file'
+require 'sublimate/uploader'

metadata

@@ -0,0 +1,47 @@
+--- !ruby/object:Gem::Specification
+name: sublimate
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Kurt Mackey
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-12-26 00:00:00.000000000Z
+dependencies: []
+description: Store ginormous files on S3 or the Rackspace Cloud using this gem.
+email: mrkurt@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/sublimate.rb
+- lib/sublimate/chunked_file.rb
+- lib/sublimate/uploader.rb
+homepage: https://github.com/mrkurt/sublimate
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Fog with large files
+test_files: []