RubyGems - mir - Versions diffs - 0.1.4 → 0.1.5 - Mend

mir 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (40) hide show

data/.gitignore +1 -0
data/doc/Mir.html +353 -0
data/doc/Mir/Application.html +1067 -0
data/doc/Mir/Config.html +660 -0
data/doc/Mir/Disk.html +211 -0
data/doc/Mir/Disk/Amazon.html +1563 -0
data/doc/Mir/Disk/IncompleteTransmission.html +116 -0
data/doc/Mir/Disk/MultiPartFile.html +586 -0
data/doc/Mir/Disk/RemoteFileNotFound.html +116 -0
data/doc/Mir/Index.html +785 -0
data/doc/Mir/Models.html +108 -0
data/doc/Mir/Models/AppSetting.html +370 -0
data/doc/Mir/Models/Resource.html +1140 -0
data/doc/Mir/Options.html +297 -0
data/doc/Mir/UndefinedConfigValue.html +116 -0
data/doc/Mir/Utils.html +582 -0
data/doc/_index.html +263 -0
data/doc/class_list.html +47 -0
data/doc/css/common.css +1 -0
data/doc/css/full_list.css +53 -0
data/doc/css/style.css +320 -0
data/doc/file.README.html +134 -0
data/doc/file_list.html +49 -0
data/doc/frames.html +13 -0
data/doc/index.html +134 -0
data/doc/js/app.js +205 -0
data/doc/js/full_list.js +150 -0
data/doc/js/jquery.js +16 -0
data/doc/method_list.html +590 -0
data/doc/top-level-namespace.html +103 -0
data/lib/mir/application.rb +92 -82
data/lib/mir/config.rb +8 -6
data/lib/mir/disk.rb +1 -0
data/lib/mir/disk/amazon.rb +33 -8
data/lib/mir/index.rb +24 -7
data/lib/mir/logger.rb +5 -0
data/lib/mir/models/resource.rb +5 -3
data/lib/mir/utils.rb +9 -3
data/lib/mir/version.rb +1 -1
metadata +30 -1

data/doc/top-level-namespace.html ADDED

@@ -0,0 +1,103 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+<title>
+  Top Level Namespace
+    &mdash; Documentation by YARD 0.7.2
+</title>
+  <link rel="stylesheet" href="css/style.css" type="text/css" media="screen" charset="utf-8" />
+  <link rel="stylesheet" href="css/common.css" type="text/css" media="screen" charset="utf-8" />
+<script type="text/javascript" charset="utf-8">
+  relpath = '';
+  if (relpath != '') relpath += '/';
+</script>
+  <script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
+  <script type="text/javascript" charset="utf-8" src="js/app.js"></script>
+  </head>
+  <body>
+    <script type="text/javascript" charset="utf-8">
+      if (window.top.frames.main) document.body.className = 'frames';
+    </script>
+    <div id="header">
+      <div id="menu">
+    <a href="_index.html">Index</a> &raquo;
+    <span class="title">Top Level Namespace</span>
+  <div class="noframes"><span class="title">(</span><a href="." target="_top">no frames</a><span class="title">)</span></div>
+</div>
+      <div id="search">
+    <a id="class_list_link" href="#">Class List</a>
+    <a id="method_list_link" href="#">Method List</a>
+    <a id="file_list_link" href="#">File List</a>
+</div>
+      <div class="clear"></div>
+    </div>
+    <iframe id="search_frame"></iframe>
+    <div id="content"><h1>Top Level Namespace
+</h1>
+<dl class="box">
+</dl>
+<div class="clear"></div>
+<h2>Defined Under Namespace</h2>
+<p class="children">
+      <strong class="modules">Modules:</strong> <span class='object_link'><a href="Mir.html" title="Mir (module)">Mir</a></span>
+</p>
+</div>
+    <div id="footer">
+  Generated on Fri Sep 23 18:24:39 2011 by
+  <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
+  0.7.2 (ruby-1.9.2).
+</div>
+  </body>
+</html>

data/lib/mir/application.rb CHANGED

@@ -9,13 +9,16 @@ module Mir
     DEFAULT_SETTINGS_FILE_NAME = "mir_settings.yml"
     DEFAULT_BATCH_SIZE = 100
-    # Creates a new Mir instance
+    # Creates a new Mir instance and starts the main execution thread
+    # @return [void]
     def self.start
       new.start
     end
     attr_reader :options, :disk, :index
+    # Creates a new Mir instance. Parses any command line arguments provided to the application
+    # @return [Mir::Application]
     def initialize
       @options = Mir::Options.parse(ARGV)
       Mir.logger = Logger.new(options.log_destination)
@@ -29,6 +32,7 @@ module Mir
     ##
     # Begins the synchronization operation after initializing the file index and remote storage
     # container
+    # @return [void]
     def start
       if options.copy && options.flush
         Mir.logger.error "Conflicting options: Cannot copy from remote source with an empty file index"
@@ -64,6 +68,92 @@ module Mir
       self.class.config
     end
+    ##
+    # Synchronize the target directory to the remote disk. Will create a file index if one does
+    # not already exist
+    # @param target [String] the absolute path of the folder that will be synchronized remotely
+    # @return [void]
+    def push(target)
+      Mir.logger.info "Starting push operation"
+      if Models::AppSetting.backup_path != target
+        Mir.logger.error "Target does not match directory stored in index"
+        exit
+      end
+      index.update
+      time = Benchmark.measure do
+        queue = WorkQueue.new(config.max_threads)
+        while Models::Resource.pending_jobs? do
+          Models::Resource.pending_sync_groups(DEFAULT_BATCH_SIZE) do |resources|
+            push_group(queue, resources)
+            handle_push_failures(resources)
+          end
+        end
+      end
+      # If any assets have been deleted locally, also remove them from remote disk
+      index.orphans.each { |orphan| disk.delete(orphan.abs_path) }
+      index.clean! # Remove orphans from index
+      puts "Completed push operation #{time}"
+      Mir.logger.info time
+    end
+    ##
+    # Copy the remote disk contents into the specified directory
+    # @param target [String] the absolute path to the destination of the S3 disk contents
+    # @return [void]
+    def pull(target)
+      write_dir = Utils.try_create_dir(target)
+      Mir.logger.info "Copying remote disk to #{write_dir.path} using #{config.max_threads} threads"
+      failed_downloads = []
+      time = Benchmark.measure do
+        queue = WorkQueue.new(config.max_threads)
+        Models::Resource.ordered_groups(DEFAULT_BATCH_SIZE) do |resources|
+          # Track the number of download attempts made per resource
+          batch = resources.inject({}) { |set, resource| set[resource] = 0; set }
+          while !batch.empty? do
+            batch.each do |resource, attempts|
+              dest = File.join(write_dir.path, resource.filename)
+              if resource.is_directory?
+                Utils.try_create_dir(dest)
+                batch.delete(resource)
+              elsif attempts >= config.max_download_attempts
+                Mir.logger.info "Resource #{resource.abs_path} failed to download"
+                failed_downloads << resource
+                batch.delete(resource)
+              elsif resource.synchronized? dest
+                Mir.logger.debug "Skipping already downloaded file #{resource.abs_path}"
+                batch.delete(resource)
+              else
+                batch[resource] += 1
+                queue.enqueue_b do
+                  Mir.logger.debug "Beginning download of #{resource.abs_path}"
+                  disk.copy(resource.abs_path, dest)
+                  if resource.synchronized? dest
+                    FileUtils.chmod_R 0755, dest # allow binaries to execute
+                    puts "Pulled #{dest}"
+                    batch.delete(resource)
+                  end
+                end
+              end
+            end
+            queue.join
+          end
+        end
+      end
+      Mir.logger.info time
+      puts "Completed pull operation #{time}"
+      unless failed_downloads.empty?
+        puts "The following files failed to download after several attempts:\n}"
+        puts failed_downloads.join("\n")
+      end
+    end
     private
       # Returns a path to the settings file. If the file was provided as an option it will always
       # be returned. When no option is passed, the file 'mir_settings.yml' will be searched for
@@ -81,36 +171,7 @@ module Mir
         nil
       end
-      ##
-      # Synchronize the local files to the remote disk
-      #
-      # @param [String] the absolute path of the folder that will be synchronized remotely
-      def push(target)
-        Mir.logger.info "Starting push operation"
-        if Models::AppSetting.backup_path != target
-          Mir.logger.error "Target does not match directory stored in index"
-          exit
-        end
-        index.update
-        time = Benchmark.measure do
-          queue = WorkQueue.new(config.max_threads)
-          while Models::Resource.pending_jobs? do
-            Models::Resource.pending_sync_groups(DEFAULT_BATCH_SIZE) do |resources|
-              push_group(queue, resources)
-              handle_push_failures(resources)
-            end
-          end
-        end
-        # If any assets have been deleted locally, also remove them from remote disk
-        index.orphans.each { |orphan| disk.delete(orphan.abs_path) }
-        index.clean! # Remove orphans from index
-        puts "Completed push operation #{time}"
-        Mir.logger.info time
-      end
       ##
       # Uploads a collection of resouces. Blocks until all items in queue have been processed
@@ -143,57 +204,6 @@ module Mir
           end
         end
       end
-      # Copy the remote disk contents into the specified directory
-      def pull(target)
-        write_dir = Utils.try_create_dir(target)
-        Mir.logger.info "Copying remote disk to #{write_dir.path} using #{config.max_threads} threads"
-        failed_downloads = []
-        time = Benchmark.measure do
-          queue = WorkQueue.new(config.max_threads)
-          Models::Resource.ordered_groups(DEFAULT_BATCH_SIZE) do |resources|
-            # Track the number of download attempts made per resource
-            batch = resources.inject({}) { |set, resource| set[resource] = 0; set }
-            while !batch.empty? do
-              batch.each do |resource, attempts|
-                dest = File.join(write_dir.path, resource.filename)
-                if resource.is_directory?
-                  Utils.try_create_dir(dest)
-                  batch.delete(resource)
-                elsif attempts >= config.max_download_attempts
-                  Mir.logger.info "Resource #{resource.abs_path} failed to download"
-                  failed_downloads << resource
-                  batch.delete(resource)
-                elsif resource.synchronized? dest
-                  Mir.logger.debug "Skipping already downloaded file #{resource.abs_path}"
-                  batch.delete(resource)
-                else
-                  batch[resource] += 1
-                  queue.enqueue_b do
-                    Mir.logger.debug "Beginning download of #{resource.abs_path}"
-                    disk.copy(resource.abs_path, dest)
-                    if resource.synchronized? dest
-                      FileUtils.chmod_R 0755, dest # allow binaries to execute
-                      puts "Pulled #{dest}"
-                      batch.delete(resource)
-                    end
-                  end
-                end
-              end
-              queue.join
-            end
-          end
-        end
-        Mir.logger.info time
-        puts "Completed pull operation #{time}"
-        unless failed_downloads.empty?
-          puts "The following files failed to download after several attempts:\n}"
-          puts failed_downloads.join("\n")
-        end
-      end
   end
 end

data/lib/mir/config.rb CHANGED

@@ -1,10 +1,14 @@
 # The config class is used for storage of user settings and preferences releated to
-# S3 storage
+#   S3 storage
 module Mir
   class UndefinedConfigValue < StandardError; end
   class Config
+    # Creates a new config instance
+    # @param config_file [String] path to the configuration file
+    # @yield [Mir::Config]
+    # @return [Mir::Config]
     def initialize(config_file = nil)
       @config_file = config_file
       @settings, @database = nil, nil
@@ -19,9 +23,9 @@ module Mir
     attr_accessor :max_upload_attempts, :max_download_attempts, :max_threads
     # Validates configuration settings
-    # TODO: move this into a class method responsible for parsing the YAML file. We
-    # shouldn't have to explicitly check for a valid object unless the user has provided
-    # a settings file
+    # @todo move this into a class method responsible for parsing the YAML file. We
+    #   shouldn't have to explicitly check for a valid object unless the user has provided
+    #   a settings file
     def valid?
       if @config_file.nil? or !File.exist?(@config_file)
         Mir.logger.error("Configuration file not found")
@@ -47,8 +51,6 @@ module Mir
       true
     end
     def method_missing(meth, *args, &block)
       val = @settings.send(meth)
       unless val.nil?

data/lib/mir/disk.rb CHANGED

@@ -6,6 +6,7 @@ module Mir
     class RemoteFileNotFound < StandardError ; end
     # Returns a disk object from the settings specified
+    # @return [Mir::Disk]
     def self.fetch(settings = {})
       case settings[:type]
       when "s3"

data/lib/mir/disk/amazon.rb CHANGED

@@ -13,11 +13,10 @@ module Mir
       attr_reader :bucket_name, :connection
-      #
       # Converts a path name to a key that can be stored on s3
       #
-      # @param [String] the path to the file
-      # @return [String] an S3-safe key with leading slashes removed
+      # @param path [String] the path to the file
+      # @return [String]
       def self.s3_key(path)
         if path[0] == File::SEPARATOR
           path[1..-1]
@@ -26,6 +25,13 @@ module Mir
         end
       end
+      ##
+      # Attempts to create a new connection to Amazon S3
+      # @option settings [Symbol] :bucket_name The name of the remote bucket
+      # @option settings [Symbol] :access_key_id The access key id for the amazon account
+      # @option settings [Symbol] :secret_access_key The secret access key for the amazon account
+      # @option settings [Symbol] :chunk_size The maximum number of bytes to use for each chunk
+      #   of a file sent to S3
       def initialize(settings = {})
         @bucket_name = settings[:bucket_name]
         @access_key_id = settings[:access_key_id]
@@ -35,22 +41,28 @@ module Mir
       end
       # Returns the buckets available from S3
+      # @return [Hash]
       def collections
         @connection.list_bucket.select(:key)
       end
+      # Sets the maximum number of bytes to be used per chunk sent to S3
+      # @param n [Integer] Number of bytes
+      # @return [void]
       def chunk_size=(n)
         raise ArgumentError unless n > 0
         @chunk_size = n
       end
+      # The size in bytes of each chunk
+      # @return [Integer]
       def chunk_size
         @chunk_size
       end
       # Whether the key exists in S3
       #
-      # @param [String] the S3 key name
+      # @param key [String] the S3 key name
       # @return [Boolean]
       def key_exists?(key)
         begin
@@ -63,8 +75,9 @@ module Mir
       end
       # Copies the remote resource to the local filesystem
-      # @param [String] the remote name of the resource to copy
-      # @param [String] the local name of the destination
+      # @param from [String] the remote name of the resource to copy
+      # @param dest [String] the local name of the destination
+      # @return [void]
       def copy(from, dest)
         open(dest, 'w') do |file|
           key = self.class.s3_key(from)
@@ -73,15 +86,23 @@ module Mir
         end
       end
-      # Retrieves the complete object from S3 without streaming
+      # Retrieves the complete object from S3. Note this method will not stream the object
+      # and will return the value of the object stored on S3
+      #
+      # @param key [String] the S3 key name of the object
+      # @return [String]
       def read(key)
         connection.get_object(bucket_name, key)
       end
+      # Whether a connection to S3 has been established
+      # @return [Boolean]
       def connected?
         @connection_success
       end
+      # Retrieves the bucket from S3
+      # @return [RightAws::S3::Bucket]
       def volume
         connection.bucket(bucket_name, true)
       end
@@ -111,6 +132,7 @@ module Mir
       #
       # @param [String] the absolute path of the file to be written
       # @raise [Disk::IncompleteTransmission] raised when remote resource is different from local file
+      # @return [void]
       def write(file_path)
         key = self.class.s3_key(file_path)
@@ -154,6 +176,8 @@ module Mir
         Digest::MD5.file(filename).to_s == remote_md5
       end
+      # Attempts to establish a connection with Amazon S3
+      # @return [RightAws::S3Interface|Boolean]
       def try_connect
         begin
           conn = RightAws::S3Interface.new(@access_key_id, @secret_access_key, {
@@ -171,7 +195,8 @@ module Mir
       # Yields a temp file object that is immediately discarded after use
       #
       # @param [String] the filename
-      # @yields [Tempfile]
+      # @yield [Tempfile]
+      # @return [void]
       def temp_file(name, &block)
         file = Tempfile.new(File.basename(name))
         begin