mir 0.1.2 → 0.1.3

data/LICENSE

@@ -0,0 +1,5 @@
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,57 @@
+# Mir
+
+Mir is a synchronization tool to help clone a directory to a remote storage
+provider. Currently only S3 is provided. A couple of the features that differentiate
+Mir from other Ruby S3 synchronization tools currently available are:
+
+* Solves S3 connection problems associated with storage and retrieval of large files (>2GB) by transparently splitting and recombining files
+* Only updates and sends resources that have fallen out of synchronization
+* Creates checksum comparisons on reads and writes to ensure complete end-to-end transmission
+* Provides a simple command line interface for pushing and pulling files
+
+The inspiration for this tool is to provide similar functionality to the classic Rsync command, but to utilize cloud-based storage providers.
+
+## Configuration 
+
+Mir uses a YAML file for configuration settings. Unless you specify otherwise, Mir will look for the file  'mir_settings.yml' in the HOME and /etc/mir directories.
+
+    settings:
+      max_upload_retries: 5
+      max_threads: 5
+      cloud_provider:
+        type: s3
+        bucket_name: gotham_backup
+        access_key_id: YOUR_ACCESS_KEY
+        secret_access_key: YOUR_SECRET_ACCESS_KEY
+        chunk_size: 5242880
+      database:
+        adapter: sqlite3
+        database: foobar.db
+
+Configuration keys:
+
+* *max_upload_retries*: This is the maximum number of attempts that Mir will try to upload your file
+* *max_threads*: The maximum number of threads that will run at once
+* *cloud_provider*: Currently only S3 is provided
+* *chunk_size*: This is the maximum number of bytes that will be written to S3 per PUT request. This is useful for sending large files to S3 and avoiding connection errors.
+* *database*: Connection information for your local database. This is delegated to ActiveRecord. See [See ActiveRecord#Base::establish_connection](http://api.rubyonrails.org/classes/ActiveRecord/Base.html#method-c-establish_connection) for more details.
+
+## Usage
+
+Install the gem:
+
+    gem install mir
+  
+Create mir_settings.yml in the HOME or /etc/mir directories. Adjust to taste
+
+Push your local directory to S3
+  
+    mir ~/mydirectory
+  
+To retrieve your remote directory
+  
+    mir -c ~/mydirectory
+
+## Notes
+
+This project is considered in an alpha state and is not ready for use in any sort of production environment. Additionally, this has an embarrassingly small number of specs which should encourage you not to use this for your critical storage needs. 

data/db/migrate/002_create_resources.rb

@@ -7,6 +7,7 @@ class CreateResources < ActiveRecord::Migration
       t.datetime :last_modified
       t.datetime :add_date, :default => DateTime.now
       t.datetime :last_synchronized
+      t.datetime :last_indexed_at
       t.boolean :is_directory, :default => false, :null => false
       t.boolean :in_progress, :default => false
       t.boolean :queued, :default => false
@@ -15,6 +16,7 @@ class CreateResources < ActiveRecord::Migration
     
     add_index :resources, :filename, :unique => true
     add_index :resources, :in_progress
+    add_index :resources, :last_indexed_at
     add_index :resources, :queued
   end
 

data/lib/mir/application.rb

@@ -19,8 +19,16 @@ module Mir
     def initialize
       @options = Mir::Options.parse(ARGV)
       Mir.logger = Logger.new(options.log_destination)
+      Mir.logger.level = if options.debug
+        Logger::DEBUG
+      else
+        Logger::ERROR
+      end
     end
     
+    ##
+    # Begins the synchronization operation after initializing the file index and remote storage
+    # container
     def start
       if options.copy && options.flush
         Mir.logger.error "Conflicting options: Cannot copy from remote source with an empty file index"
@@ -41,13 +49,19 @@ module Mir
       
       options.copy ? pull(param_path) : push(param_path)
     end
-  
+    
+    ##
+    # Returns a global configuration instance
+    # 
+    # @return [Mir::Config]
     def self.config
       @@config
     end
     
+    ##
+    # Alias for +config
     def config
-      @@config
+      self.class.config
     end
     
     private
@@ -67,7 +81,10 @@ module Mir
         nil
       end
       
+      ##
       # Synchronize the local files to the remote disk
+      #
+      # @param [String] the absolute path of the folder that will be synchronized remotely
       def push(target)
         Mir.logger.info "Starting push operation"
         
@@ -87,26 +104,36 @@ module Mir
             end
           end
         end
+        
+        # If any assets have been deleted locally, also remove them from remote disk
+        index.orphans.each { |orphan| disk.delete(orphan.abs_path) }
+        index.clean! # Remove orphans from index
+        puts "Completed push operation #{time}"
         Mir.logger.info time
       end
       
-      
+      ##
       # Uploads a collection of resouces. Blocks until all items in queue have been processed
+      #
       # @param [WorkQueue] a submission queue to manage resource uploads
       # @param [Array] an array of Models::Resource objects that need to be uploaded
       def push_group(work_queue, resources)
         resources.each do |resource|
+          Mir.logger.debug "Enqueueing #{resource.filename}"
           work_queue.enqueue_b do
             resource.start_progress
             disk.write resource.abs_path
             resource.update_success
+            puts "Pushed #{resource.abs_path}"
           end
         end
         work_queue.join
       end
       
+      #
       # Scans a collection of resources for jobs that did no complete successfully and flags them
       # for resubmission
+      #
       # @param [Array] an array of Models::Resources
       def handle_push_failures(resources)
         resources.each do |resource|
@@ -130,17 +157,26 @@ module Mir
           # otherwise download the file
           Models::Resource.ordered_groups(DEFAULT_BATCH_SIZE) do |resources|
             resources.each do |resource|
-              dest = File.join(write_dir.path, resource.filename)          
+              dest = File.join(write_dir.path, resource.filename)
               if resource.is_directory?  
                 Utils.try_create_dir(dest)
               elsif !resource.synchronized?(dest)
-                queue.enqueue_b { @disk.copy(resource.abs_path, dest) }
+                queue.enqueue_b do 
+                  disk.copy(resource.abs_path, dest)
+                  if resource.synchronized?(dest)
+                    Mir.logger.info "Successful download #{dest}"
+                    puts "Pulled #{dest}"
+                  else
+                    Mir.logger.error "Incomplete download #{dest}"
+                  end
+                end
               end
             end
             queue.join
           end
         end
         Mir.logger.info time
+        puts "Completed pull operation #{time}"
       end
           
   end

data/lib/mir/disk/amazon.rb

@@ -1,12 +1,24 @@
 require "right_aws"
+require "tempfile"
+require "digest/md5"
 
 module Mir
   module Disk
     class Amazon
       
-      attr_reader :bucket_name
+      # This is the default size in bytes at which files will be split and stored
+      # on S3. From trial and error, 5MB seems to be a good default size for chunking
+      # large files.
+      DEFAULT_CHUNK_SIZE = 5*(2**20) 
       
-      def self.key_name(path)
+      attr_reader :bucket_name, :connection
+      
+      #
+      # Converts a path name to a key that can be stored on s3
+      # 
+      # @param [String] the path to the file
+      # @return [String] an S3-safe key with leading slashes removed
+      def self.s3_key(path)
         if path[0] == File::SEPARATOR
           path[1..-1] 
         else
@@ -18,6 +30,7 @@ module Mir
         @bucket_name = settings[:bucket_name]
         @access_key_id = settings[:access_key_id]
         @secret_access_key = settings[:secret_access_key]
+        @chunk_size = settings[:chunk_size] || DEFAULT_CHUNK_SIZE
         @connection = try_connect
       end
       
@@ -26,14 +39,43 @@ module Mir
         @connection.list_bucket.select(:key)
       end
       
+      def chunk_size=(n)
+        raise ArgumentError unless n > 0
+        @chunk_size = n
+      end
+      
+      def chunk_size
+        @chunk_size
+      end
+      
+      # Whether the key exists in S3
+      # 
+      # @param [String] the S3 key name
+      # @return [Boolean]
+      def key_exists?(key)
+        begin
+          connection.head(bucket_name, key)
+        rescue RightAws::AwsError => e
+          return false
+        end
+
+        true
+      end
+      
       # Copies the remote resource to the local filesystem
       # @param [String] the remote name of the resource to copy
       # @param [String] the local name of the destination
-      def copy(from, to)
-        open(to, 'w') do |file|
-          @connection.get(bucket_name, self.class.key_name(from)) { |chunk| file.write(chunk) }
+      def copy(from, dest)
+        open(dest, 'w') do |file|
+          key = self.class.s3_key(from)
+          remote_file = MultiPartFile.new(self, key)
+          remote_file.get(dest)
         end
-        Mir.logger.info "Completed download '#{to}'"
+      end
+      
+      # Retrieves the complete object from S3 without streaming
+      def read(key)
+        connection.get_object(bucket_name, key)
       end
       
       def connected?
@@ -41,29 +83,175 @@ module Mir
       end
       
       def volume
-        @connection.bucket(bucket_name, true)
+        connection.bucket(bucket_name, true)
       end
       
+      # Deletes the remote version of the file
+      # @return [Boolean] true if operation succeeded
+      def delete(file_path)
+        key = self.class.s3_key(file_path)
+        Mir.logger.info "Deleting remote object #{file_path}"
+
+        begin
+          remote_file = MultiPartFile.new(self, key)
+        rescue Disk::RemoteFileNotFound => e
+          Mir.logger.warn "Could not find remote resource '#{key}'"
+          return false
+        end
+        
+        if remote_file.multipart?
+          delete_parts(key)
+        else
+          connection.delete(bucket_name, key)
+        end
+      end
+      
+      # Writes a file to Amazon S3. If the file size exceeds the chunk size, the file will
+      # be written in chunks
+      # 
+      # @param [String] the absolute path of the file to be written
+      # @raise [Disk::IncompleteTransmission] raised when remote resource is different from local file
       def write(file_path)
-        @connection.put(bucket_name, self.class.key_name(file_path), File.open(file_path))
+        key = self.class.s3_key(file_path)
+        
+        if File.size(file_path) <= chunk_size
+          connection.put(bucket_name, key, open(file_path))
+          raise Disk::IncompleteTransmission unless equals?(file_path, key)
+        else          
+          delete_parts(file_path) # clean up remaining part files if any exist
+                    
+          open(file_path, "rb") do |source|
+            part_id = 1
+            while part = source.read(chunk_size) do
+              part_name = Mir::Utils.filename_with_sequence(key, part_id)
+              Mir.logger.debug "Writing part #{part_name}"
+              
+              temp_file(part_name) do |tmp|
+                tmp.binmode                
+                tmp.write(part)
+                tmp.rewind
+                connection.put(bucket_name, part_name, open(tmp.path))
+                raise Disk::IncompleteTransmission unless equals?(tmp.path, part_name)
+              end
+
+              part_id += 1
+            end
+          end
+        end
         Mir.logger.info "Completed upload #{file_path}"
       end
       
       private
-        def try_connect
-          begin
-            conn = RightAws::S3Interface.new(@access_key_id, @secret_access_key, {
-              :multi_thread => true,
-              :logger => Mir.logger
-            })
-            @connection_success = true
-            return conn
-          rescue Exception => e
-            @connection_success = false
-            Mir.logger.error "Could not establish connection with S3: '#{e.message}'"
+      
+      # Determines whether a local file matches the remote file
+      #
+      # @param [String] the complete path name to the file
+      # @param [String] the S3 key name for the object
+      # @return [Boolean] whether the MD5 hash of the local file matches the remote value
+      def equals?(filename, key)
+        meta_ob = connection.retrieve_object(:bucket => bucket_name, :key => key)
+        remote_md5 = meta_ob[:headers]["etag"].slice(4..-5)
+        Digest::MD5.file(filename).to_s == remote_md5
+      end
+      
+      def try_connect
+        begin
+          conn = RightAws::S3Interface.new(@access_key_id, @secret_access_key, {
+            :multi_thread => true,
+            :logger => Mir.logger
+          })
+          @connection_success = true
+          return conn
+        rescue Exception => e
+          @connection_success = false
+          Mir.logger.error "Could not establish connection with S3: '#{e.message}'"
+        end
+      end
+      
+      # Yields a temp file object that is immediately discarded after use
+      #
+      # @param [String] the filename
+      # @yields [Tempfile]
+      def temp_file(name, &block)          
+        file = Tempfile.new(File.basename(name))
+        begin
+          yield file
+        ensure
+          file.close
+          file.unlink
+        end
+      end
+      
+      # Used to delete a file that has been broken into chunks
+      #
+      # @return [Boolean] true if succeeded
+      def delete_parts(file_path)
+        flag = true
+        connection.incrementally_list_bucket(bucket_name, 
+                                            { :prefix => self.class.s3_key(file_path), 
+                                              :max_keys => 100 }) do |group|
+          
+          group[:contents].each do |item|
+            if connection.delete(bucket_name, item[:key])
+              Mir.logger.debug("Deleted '#{item[:key]}'")
+            else
+              flag = false 
+            end              
           end
         end
+        flag
+      end
+    end
+    
+    # Used to hide the inner details of multipart file uploads and downloads. It is important
+    # that this class does not throw any exceptions as these exceptions may be swallowed further
+    # up the stack by worker threads
+    class MultiPartFile
+      
+      # @param [Disk] the remote disk
+      # @param [String] the name of the resource
+      def initialize(disk, name)
+        @disk, @name = disk, name
+        multiname = Utils.filename_with_sequence(name, 1)
+                  
+        if disk.key_exists?(name)
+          @multipart = false
+        elsif disk.key_exists?(multiname)
+          @multipart = true
+        else
+          raise Disk::RemoteFileNotFound
+        end
+      end
+      
+      attr_reader :disk, :name
       
+      # Whether the resource is broken into chunks on the remote store
+      def multipart?
+        @multipart
+      end
+      
+      # Downloads the resource to the destination. If the file is stored in parts it is download
+      # sequentially in pieces
+      def get(dest)
+        output = File.new(dest, "wb")
+        begin
+          if multipart?
+            seq = 1
+            while part = Utils.filename_with_sequence(name, seq) do
+              break unless disk.key_exists? part
+              output.write disk.read(part)
+              seq += 1
+            end
+          else
+            output.write disk.read(name)
+          end
+        rescue Exception => e
+          Mir.logger.error e
+        ensure
+          output.close
+        end
+      end
     end
+    
   end
 end

data/lib/mir/disk.rb

@@ -2,6 +2,9 @@
 module Mir
   module Disk
     
+    class IncompleteTransmission < StandardError ; end
+    class RemoteFileNotFound < StandardError ; end
+    
     # Returns a disk object from the settings specified
     def self.fetch(settings = {})
       case settings[:type]

data/lib/mir/index.rb

@@ -1,13 +1,19 @@
 require 'active_record'
 require "active_support/inflector"
 
-# Manages database operations for application
+# The index class is responsible for maintaining knowledge of files uploaded
+# onto the remote file system. The index does this by scanning the directory to
+# be synchronized and evaluating whether a file needs to be uploaded or has changed
+# since the last indexing date
+
 module Mir
   class Index
     
     MIGRATIONS_PATH = File.join(File.dirname(__FILE__), "..", "..", "db", "migrate")
     
     # Returns a databse object used to connect to the indexing database
+    #
+    # @param [String] the absolute path of the directory to be synchronized
     # @param [Hash] database configuration settings. See ActiveRecord#Base::establish_connection
     def initialize(sync_path, connection_params)
       @sync_path = sync_path
@@ -16,7 +22,9 @@ module Mir
     
     attr_reader :sync_path
     
+    #
     # Creates necessary database and tables if this is the first time connecting
+    #
     # @option opts [Boolean] :verbose Enable on ActiveRecord reporting
     # @option opts [Boolean] :force_flush Rebuild index no matter what
     def setup(options = {})
@@ -34,24 +42,50 @@ module Mir
       rebuild if !tables_created? or options[:force_flush]
     end
     
-    # Scans the filesystem and flags any resources which need updating
+    ##
+    # Scans the synchronization path and evaluates whether a resource has changed
+    # since the last index or is new and needs to be added to the index.
     def update
       Mir.logger.info "Updating backup index for '#{sync_path}'"
+      Models::AppSetting.last_indexed_at = @last_indexed_at = DateTime.now
+      
       Dir.glob(File.join(sync_path, "**", "*")) do |f|
         fname = relative_path(f)
         file = File.new(f)
         resource = Models::Resource.find_by_filename(fname)
 
         if resource.nil?
-          Mir.logger.info "Adding file to index #{fname}"
+          Mir.logger.debug "Adding file to index #{fname}"
           resource = Models::Resource.create_from_file_and_name(file, fname)
-        else
-          resource.flag_for_update unless resource.synchronized?(file)
-        end                
+        elsif !resource.synchronized?(file)
+          resource.flag_for_update
+        end
+        resource.update_attribute(:last_indexed_at, last_indexed_at)
       end
+      
       Mir.logger.info "Index updated"
     end
     
+    ##
+    # Returns any files not present since the last re-indexing. This is useful
+    # for finding files that have been deleted post-index.
+    # 
+    # @return [Mir::Models::Resource]
+    def orphans
+      Models::Resource.not_indexed_on(last_indexed_at)
+    end
+    
+    def last_indexed_at
+      @last_indexed_at ||= Models::AppSetting.last_indexed_at
+    end
+    
+    
+    ##
+    # Removes any files from the index that are no longer present locally
+    def clean!
+      Models::Resource.delete_all_except(last_indexed_at)
+    end
+    
     private
       # Returns the path of a file relative to the backup directory
       def relative_path(file)
@@ -63,10 +97,12 @@ module Mir
         tables.each { |t| ActiveRecord::Migration.drop_table(t.table_name) if t.table_exists? }
         ActiveRecord::Migration.drop_table(:schema_migrations) if @connection.table_exists? :schema_migrations
         ActiveRecord::Migrator.migrate MIGRATIONS_PATH
-        Models::AppSetting.create(:name => Models::AppSetting::SYNC_PATH, :value => sync_path)
-        Models::AppSetting.create(:name => Models::AppSetting::INSTALL_DATE, :value => DateTime.now.to_s)
+        Models::AppSetting.initialize_table(sync_path)
       end
       
+      
+      ##
+      # TODO: no reason to lazy load these activemodels
       def load_tables
         @tables = []
         models = File.join(File.dirname(__FILE__), "models", "*.rb")

data/lib/mir/models/app_setting.rb

@@ -1,13 +1,28 @@
 module Mir
   module Models
     class AppSetting < ActiveRecord::Base
-      
-      SYNC_PATH = "sync_path"
-      INSTALL_DATE = "installation_date"
+
+      ##
+      # Builds entries for the variables that will be used by this application
+      # 
+      # @param [String] the path to be synchronized with S3
+      def self.initialize_table(sync_path)
+        create(:name => :sync_path, :value => sync_path)
+        create(:name => :install_date, :value => DateTime.now)
+        create(:name => :last_indexed_at, :value => nil)
+      end
       
       def self.backup_path
-        record = self.where(:name => SYNC_PATH).first
-        record.value rescue nil
+        where(:name => :sync_path).first.value
+      end
+      
+      def self.last_indexed_at=(val)
+        record = where(:name => :last_indexed_at).first
+        record.update_attribute(:value, val)
+      end
+      
+      def self.last_indexed_at
+        where(:name => :last_indexed_at).first.value
       end
 
     end

data/lib/mir/models/resource.rb

@@ -5,6 +5,8 @@ module Mir
   module Models
     class Resource < ActiveRecord::Base
       
+      scope :not_indexed_on, lambda { |date| where("last_indexed_at != ?", date) }
+      
       # Builds a resource for the backup index from a file
       # @param [File] a file object
       # @param [String] the name of the file on the remote disk
@@ -20,6 +22,14 @@ module Mir
                :is_directory => is_dir)
       end
       
+      ##
+      # Removes all resources not that were not indexed on the specified date
+      #
+      # @param [DateTime]
+      def self.delete_all_except(index_date)
+        not_indexed_on(index_date).delete_all
+      end
+      
       # Returns true when jobs are still queued
       def self.pending_jobs?
         self.where(:queued => true).size > 0
@@ -49,15 +59,20 @@ module Mir
         end
       end
       
+      #
       # Compares a file asset to the index to deterimine whether the file needs to be updated
+      #
       # @param [String] a path to a file or directory
-      # @returns [Boolean] whether the file and index are in sync with each other
+      # @return [Boolean] returns true when the file's checksum is equal to the value stored in
+      #   the index, or when the file is a directory
       def synchronized?(file)
-        if File.exists? file
-          return true if File.directory? file
-          return Digest::MD5.file(file).to_s == self.checksum
+        if File.directory?(file)
+          return true
+        elsif !File.exist?(file) or in_progress? or queued?
+          return false
+        else
+          Digest::MD5.file(file).to_s == self.checksum
         end
-        false
       end
       
       # Whether the item can be synchronized to a remote disk
@@ -68,7 +83,9 @@ module Mir
       
       # Places the resource into a queueble state
       def flag_for_update
-        update_attributes :queued => true, :checksum => Digest::MD5.file(abs_path).to_s
+        update_attributes :queued => true, 
+                          :checksum => Digest::MD5.file(abs_path).to_s,
+                          :last_modified => File.new(abs_path).ctime
       end
       
       def start_progress

data/lib/mir/options.rb

@@ -5,9 +5,13 @@ require 'ostruct'
 module Mir
   class Options
     
-    USAGE_BANNER = "Usage: mir [options] [settings_file_path] [target]"
+    USAGE_BANNER = "Usage: mir [options] [target]"
     
     def self.parse(args)
+      if args.size < 1
+        puts USAGE_BANNER
+        exit
+      end
       options = OpenStruct.new
       options.debug = false
       options.verbose = false
@@ -30,6 +34,10 @@ module Mir
           options.copy = true
         end
         
+        opts.on("-d", "--debug", "Enable debug logging") do
+          options.debug = true
+        end
+        
         opts.on("--settings", String, "The YAML settings file") do |path|
           options.settings_path = path
         end

data/lib/mir/utils.rb

@@ -1,7 +1,49 @@
 module Mir
   module Utils
+    
+    # Generates filename for a split file
+    # filename_with_sequence("foobar.txt", 23) => foobar.txt.00000023
+    # @param [String] filename
+    # @param [Integer] the sequence number
+    def self.filename_with_sequence(name, seq)
+      [name, ".", "%08d" % seq].join
+    end
+    
     def self.try_create_dir(path)
       Dir.mkdir(path) unless Dir.exist?(path)
     end
+    
+    # Splits a file into pieces that may be reassembled later
+    # @param [File or String] File to be split
+    # @param [Integer] the number of bytes per each chunked file
+    # @param [String] where the split files should be stored
+    def self.split_file(file, chunk_size, dest)
+      try_create_dir(dest) unless Dir.exist?(dest)
+      
+      fname = File.join(dest, File.basename(file))
+      seq = 1
+      
+      open(file, "rb") do |f|
+        while split_data = f.read(chunk_size) do
+          split_name = self.filename_with_sequence(fname, seq)
+          open(split_name, "wb") { |dest| dest.write(split_data) }
+          seq += 1
+        end
+      end
+    end
+  
+    
+    # Recombines a file from pieces
+    # @param [String] the directory that holds the split files
+    # @param [String] the path to the assembled file
+    def self.recombine(source_dir, dest)
+      parts = Dir.glob(File.join(source_dir, "*"))
+      open(File.expand_path(dest), "wb") do |file|
+        parts.each do |part|
+          p_io = File.new(part)
+          file.write p_io.read
+        end
+      end
+    end
   end
 end

data/lib/mir/version.rb

@@ -1,5 +1,5 @@
 module Mir
-  VERSION = [0,1,2]
+  VERSION = [0,1,3]
   
   def self.version
     VERSION.join('.')

data/spec/disk/amazon_spec.rb

@@ -0,0 +1,20 @@
+require "spec_helper"
+
+describe Mir::Disk::Amazon do
+  
+  let(:settings) do
+    {
+      :bucket_name        => "bucket",
+      :access_key_id      => "xxx",
+      :secret_access_key  => "xxx"
+    }
+  end
+  
+  let(:disk) { Mir::Disk::Amazon.new(settings) }
+    
+  it "should default to 5mb chunk size for uploads" do
+    disk = Mir::Disk::Amazon.new(settings)
+    disk.chunk_size.should == 5*2**20
+  end
+  
+end

data/spec/index_spec.rb

@@ -0,0 +1,33 @@
+require "spec_helper"
+
+describe Mir::Index do
+  
+  describe "#update" do
+    let(:index) { Mir::Index.new("/tmp", {}) }
+    let(:fake_file) { mock("file", :directory? => false) }
+    let(:resource) { mock("resource") }
+    
+    before(:each) do
+      Mir::Models::AppSetting.should_receive(:last_indexed_at=)
+      Dir.should_receive(:glob).and_yield("filename")
+      File.should_receive(:new).and_return(fake_file)
+      resource.should_receive(:update_attribute).and_return(true)
+      index.stub!(:last_indexed_at)
+    end
+    
+    it "adds an asset to the index if it has not yet been added" do
+      Mir::Models::Resource.should_receive(:find_by_filename).and_return(nil)
+      Mir::Models::Resource.should_receive(:create_from_file_and_name).and_return(resource)
+      index.update
+    end
+    
+    it "should flag a resource if the local copy is out of sync with the index" do
+      Mir::Models::Resource.should_receive(:find_by_filename).and_return(resource)
+      resource.should_receive(:synchronized?).and_return(false)
+      resource.should_receive(:flag_for_update).and_return(true)
+      index.update
+    end
+  end
+  
+
+end

data/spec/spec_helper.rb

@@ -1,5 +1,8 @@
 require "mir"
+require "mir/models/app_setting"
+require "mir/models/resource"
 
 RSpec.configure do |config|
-  
+  Mir.logger = Logger.new($stdout)
+  Mir.logger.level = Logger::ERROR
 end

data/spec/utils_spec.rb

@@ -0,0 +1,43 @@
+require "spec_helper"
+require "tempfile"
+require "fileutils"
+require "digest/md5"
+
+describe Mir::Utils do
+  
+  context "Splitting files" do
+    before(:all) do
+      @tmp_dir = "/tmp/splitter-tests"
+      FileUtils.mkdir(@tmp_dir) unless File.exist? @tmp_dir
+      @fake_file_path = "/tmp/512byteFile"
+      @file = open(@fake_file_path, "w") do |f|
+        f.write <<-EOS
+          Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor 
+          incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud 
+          exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute 
+          irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla 
+          pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia 
+          deserunt mollit anim id est laborum.
+        EOS
+      end
+    end
+    
+    after(:all) do
+      FileUtils.rm(Dir.glob(File.join(@tmp_dir, "*")))
+      FileUtils.rm(@fake_file_path)
+    end
+    
+    it "should create smaller files from one large file" do
+      path = Mir::Utils.split_file(@fake_file_path, 8, @tmp_dir)
+      split_files = Dir.glob(File.join(@tmp_dir, "*"))
+      split_files.size.should == 64
+    end
+    
+    it "should recombine smaller files into one large file" do
+      dest = "/tmp/recombined.txt"
+      path = Mir::Utils.recombine(@tmp_dir, dest)
+      Digest::MD5.file(dest).should == Digest::MD5.file(@fake_file_path)
+    end
+
+  end
+end

metadata

@@ -2,7 +2,7 @@
 name: mir
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors: 
 - Nate Miller
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-09-05 00:00:00 -07:00
+date: 2011-09-22 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -91,6 +91,8 @@ extra_rdoc_files: []
 files: 
 - .gitignore
 - Gemfile
+- LICENSE
+- README.md
 - Rakefile
 - bin/mir
 - db/migrate/001_create_app_settings.rb
@@ -108,7 +110,10 @@ files:
 - lib/mir/utils.rb
 - lib/mir/version.rb
 - mir.gemspec
+- spec/disk/amazon_spec.rb
+- spec/index_spec.rb
 - spec/spec_helper.rb
+- spec/utils_spec.rb
 has_rdoc: true
 homepage: ""
 licenses: []
@@ -138,4 +143,7 @@ signing_key:
 specification_version: 3
 summary: A utility for backing up resources
 test_files: 
+- spec/disk/amazon_spec.rb
+- spec/index_spec.rb
 - spec/spec_helper.rb
+- spec/utils_spec.rb