mir 0.1.2

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,2 @@
+source "http://rubygems.org"
+gemspec

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new

data/bin/mir

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require 'mir'
+Mir::Application.start

data/db/migrate/001_create_app_settings.rb

@@ -0,0 +1,13 @@
+class CreateAppSettings < ActiveRecord::Migration
+  def self.up
+    create_table :app_settings do |t|
+      t.string :name
+      t.string :value
+    end
+    add_index :app_settings, :name
+  end
+
+  def self.down
+    drop_table :app_settings
+  end
+end

data/db/migrate/002_create_resources.rb

@@ -0,0 +1,24 @@
+class CreateResources < ActiveRecord::Migration
+  def self.up
+    create_table :resources do |t|
+      t.string :filename, :null => false
+      t.integer :size, :default => 0, :null => false
+      t.string :checksum, :limit => 32
+      t.datetime :last_modified
+      t.datetime :add_date, :default => DateTime.now
+      t.datetime :last_synchronized
+      t.boolean :is_directory, :default => false, :null => false
+      t.boolean :in_progress, :default => false
+      t.boolean :queued, :default => false
+      t.integer :times_failed, :default => 0, :null => false
+    end
+    
+    add_index :resources, :filename, :unique => true
+    add_index :resources, :in_progress
+    add_index :resources, :queued
+  end
+
+  def self.down
+    drop_table :resources
+  end
+end

data/lib/mir/application.rb

@@ -0,0 +1,147 @@
+require "benchmark"
+require "fileutils"
+require "work_queue"
+
+module Mir
+  
+  class Application
+    
+    DEFAULT_SETTINGS_FILE_NAME = "mir_settings.yml"
+    DEFAULT_BATCH_SIZE = 20
+        
+    # Creates a new Mir instance
+    def self.start
+      new.start
+    end
+    
+    attr_reader :options, :disk, :index
+    
+    def initialize
+      @options = Mir::Options.parse(ARGV)
+      Mir.logger = Logger.new(options.log_destination)
+    end
+    
+    def start
+      if options.copy && options.flush
+        Mir.logger.error "Conflicting options: Cannot copy from remote source with an empty file index"
+        exit
+      end
+      
+      @@config = Config.new find_settings_file
+      param_path = File.expand_path(ARGV[0])
+      exit unless config.valid?
+      
+      # Initialize our remote disk
+      @disk = Disk.fetch(config.cloud_provider)
+      exit unless @disk.connected?
+      
+      # Initialize our local index
+      @index = Index.new(param_path, config.database)
+      @index.setup(:verbose => options.verbose, :force_flush => options.flush)
+      
+      options.copy ? pull(param_path) : push(param_path)
+    end
+  
+    def self.config
+      @@config
+    end
+    
+    def config
+      @@config
+    end
+    
+    private
+      # Returns a path to the settings file. If the file was provided as an option it will always
+      # be returned. When no option is passed, the file 'mir_settings.yml' will be searched for
+      # in the following paths: $HOME, /etc/mir
+      # @return [String] path to the settings or nil if none is found
+      def find_settings_file
+        if !options.settings_path.nil?
+          return options.settings_path
+        else
+          ["~", "/etc/mir"].each do |dir|
+            path = File.expand_path(File.join(dir, DEFAULT_SETTINGS_FILE_NAME))
+            return path.to_s if File.exist?(path)
+          end
+        end
+        nil
+      end
+      
+      # Synchronize the local files to the remote disk
+      def push(target)
+        Mir.logger.info "Starting push operation"
+        
+        if Models::AppSetting.backup_path != target
+          Mir.logger.error "Target does not match directory stored in index"
+          exit
+        end
+        
+        index.update
+        
+        time = Benchmark.measure do
+          queue = WorkQueue.new(config.max_threads)
+          while Models::Resource.pending_jobs? do
+            Models::Resource.pending_sync_groups(DEFAULT_BATCH_SIZE) do |resources|
+              push_group(queue, resources)
+              handle_push_failures(resources)
+            end
+          end
+        end
+        Mir.logger.info time
+      end
+      
+      
+      # Uploads a collection of resouces. Blocks until all items in queue have been processed
+      # @param [WorkQueue] a submission queue to manage resource uploads
+      # @param [Array] an array of Models::Resource objects that need to be uploaded
+      def push_group(work_queue, resources)
+        resources.each do |resource|
+          work_queue.enqueue_b do
+            resource.start_progress
+            disk.write resource.abs_path
+            resource.update_success
+          end
+        end
+        work_queue.join
+      end
+      
+      # Scans a collection of resources for jobs that did no complete successfully and flags them
+      # for resubmission
+      # @param [Array] an array of Models::Resources
+      def handle_push_failures(resources)
+        resources.each do |resource|
+          if resource.in_progress?
+            Mir.logger.info "Resource '#{resource.abs_path}' failed to upload"
+            resource.update_failure 
+          end
+        end
+      end
+      
+      # Copy the remote disk contents into the specified directory
+      def pull(target)
+        Utils.try_create_dir(target)
+        write_dir = Dir.new(target)
+        Mir.logger.info "Copying remote disk to #{write_dir.path} using #{config.max_threads} threads"
+        
+        time = Benchmark.measure do 
+          queue = WorkQueue.new(config.max_threads)
+        
+          # loop through each resource. If the resource is a directory, create the path
+          # otherwise download the file
+          Models::Resource.ordered_groups(DEFAULT_BATCH_SIZE) do |resources|
+            resources.each do |resource|
+              dest = File.join(write_dir.path, resource.filename)          
+              if resource.is_directory?  
+                Utils.try_create_dir(dest)
+              elsif !resource.synchronized?(dest)
+                queue.enqueue_b { @disk.copy(resource.abs_path, dest) }
+              end
+            end
+            queue.join
+          end
+        end
+        Mir.logger.info time
+      end
+          
+  end
+end

data/lib/mir/config.rb

@@ -0,0 +1,62 @@
+
+
+# The config class is used for storage of user settings and preferences releated to 
+# S3 storage
+module Mir
+  class UndefinedConfigValue < StandardError; end
+    
+  class Config
+    
+    def initialize(config_file = nil)
+      @config_file = config_file
+      @settings, @database = nil, nil
+    end
+    
+    # Validates configuration settings
+    def valid?
+      if @config_file.nil? or !File.exist?(@config_file)
+        Mir.logger.error("Configuration file not found")
+        return false
+      end
+      
+      if File.directory?(@config_file)
+        Mir.logger.error("Received directory instead of settings file")
+        return false
+      end
+      
+      File.open(@config_file) do |f|
+        yml = YAML::load(f)
+        if yml.key? "settings"
+          @settings = OpenStruct.new yml["settings"]
+          @settings.database = symbolize_keys(@settings.database)
+          @settings.cloud_provider = symbolize_keys(@settings.cloud_provider)
+        else
+          Mir.logger.error("Malformed config file")
+          return false
+        end
+      end
+      true
+    end
+    
+    
+        
+    def method_missing(meth, *args, &block)
+      val = @settings.send(meth)
+      unless val.nil?
+        val
+      else
+        raise UndefinedConfigValue
+      end
+    end
+    
+    private
+      def symbolize_keys(hsh)
+        symbolized = hsh.inject({}) do |opts, (k,v)|
+          opts[(k.to_sym rescue k) || k] = v
+          opts
+        end
+        symbolized
+      end
+    
+  end
+end

data/lib/mir/disk/amazon.rb

@@ -0,0 +1,69 @@
+require "right_aws"
+
+module Mir
+  module Disk
+    class Amazon
+      
+      attr_reader :bucket_name
+      
+      def self.key_name(path)
+        if path[0] == File::SEPARATOR
+          path[1..-1] 
+        else
+          path
+        end
+      end
+      
+      def initialize(settings = {})
+        @bucket_name = settings[:bucket_name]
+        @access_key_id = settings[:access_key_id]
+        @secret_access_key = settings[:secret_access_key]
+        @connection = try_connect
+      end
+      
+      # Returns the buckets available from S3
+      def collections
+        @connection.list_bucket.select(:key)
+      end
+      
+      # Copies the remote resource to the local filesystem
+      # @param [String] the remote name of the resource to copy
+      # @param [String] the local name of the destination
+      def copy(from, to)
+        open(to, 'w') do |file|
+          @connection.get(bucket_name, self.class.key_name(from)) { |chunk| file.write(chunk) }
+        end
+        Mir.logger.info "Completed download '#{to}'"
+      end
+      
+      def connected?
+        @connection_success
+      end
+      
+      def volume
+        @connection.bucket(bucket_name, true)
+      end
+      
+      def write(file_path)
+        @connection.put(bucket_name, self.class.key_name(file_path), File.open(file_path))
+        Mir.logger.info "Completed upload #{file_path}"
+      end
+      
+      private
+        def try_connect
+          begin
+            conn = RightAws::S3Interface.new(@access_key_id, @secret_access_key, {
+              :multi_thread => true,
+              :logger => Mir.logger
+            })
+            @connection_success = true
+            return conn
+          rescue Exception => e
+            @connection_success = false
+            Mir.logger.error "Could not establish connection with S3: '#{e.message}'"
+          end
+        end
+      
+    end
+  end
+end

data/lib/mir/disk.rb

@@ -0,0 +1,17 @@
+# The remote storage container where objects are saved
+module Mir
+  module Disk
+    
+    # Returns a disk object from the settings specified
+    def self.fetch(settings = {})
+      case settings[:type]
+      when "s3"
+        Mir::Disk::Amazon.new(settings)
+      else
+        Mir.logger.error "Could not find specified cloud provider"
+      end
+    end
+
+
+  end
+end

data/lib/mir/index.rb

@@ -0,0 +1,92 @@
+require 'active_record'
+require "active_support/inflector"
+
+# Manages database operations for application
+module Mir
+  class Index
+    
+    MIGRATIONS_PATH = File.join(File.dirname(__FILE__), "..", "..", "db", "migrate")
+    
+    # Returns a databse object used to connect to the indexing database
+    # @param [Hash] database configuration settings. See ActiveRecord#Base::establish_connection
+    def initialize(sync_path, connection_params)
+      @sync_path = sync_path
+      @connection_params = connection_params
+    end
+    
+    attr_reader :sync_path
+    
+    # Creates necessary database and tables if this is the first time connecting
+    # @option opts [Boolean] :verbose Enable on ActiveRecord reporting
+    # @option opts [Boolean] :force_flush Rebuild index no matter what
+    def setup(options = {})
+      options[:force_flush] ||= false
+      options[:verbose] ||= false
+      @connection = ActiveRecord::Base.establish_connection(@connection_params).connection
+      ActiveRecord::Base.timestamped_migrations = false
+      
+      if options[:verbose]
+        ActiveRecord::Base.logger = Mir.logger
+        ActiveRecord::Migration.verbose = true
+      end
+      
+      load_tables
+      rebuild if !tables_created? or options[:force_flush]
+    end
+    
+    # Scans the filesystem and flags any resources which need updating
+    def update
+      Mir.logger.info "Updating backup index for '#{sync_path}'"
+      Dir.glob(File.join(sync_path, "**", "*")) do |f|
+        fname = relative_path(f)
+        file = File.new(f)
+        resource = Models::Resource.find_by_filename(fname)
+
+        if resource.nil?
+          Mir.logger.info "Adding file to index #{fname}"
+          resource = Models::Resource.create_from_file_and_name(file, fname)
+        else
+          resource.flag_for_update unless resource.synchronized?(file)
+        end                
+      end
+      Mir.logger.info "Index updated"
+    end
+    
+    private
+      # Returns the path of a file relative to the backup directory
+      def relative_path(file)
+        File.absolute_path(file).gsub(sync_path,'')
+      end
+      
+      # Regenerates the file system index for the backup directory
+      def rebuild
+        tables.each { |t| ActiveRecord::Migration.drop_table(t.table_name) if t.table_exists? }
+        ActiveRecord::Migration.drop_table(:schema_migrations) if @connection.table_exists? :schema_migrations
+        ActiveRecord::Migrator.migrate MIGRATIONS_PATH
+        Models::AppSetting.create(:name => Models::AppSetting::SYNC_PATH, :value => sync_path)
+        Models::AppSetting.create(:name => Models::AppSetting::INSTALL_DATE, :value => DateTime.now.to_s)
+      end
+      
+      def load_tables
+        @tables = []
+        models = File.join(File.dirname(__FILE__), "models", "*.rb")
+        
+        # load the AR models for the application
+        Dir.glob(models) do |f|
+          require f
+          name = "Models::" + ActiveSupport::Inflector.camelize(File.basename(f, ".rb"))
+          @tables << eval(name)
+        end
+      end
+      
+      # Returns the activerecord classes for each table used by the application
+      def tables
+        @tables
+      end
+      
+      # Checks whether any of the tables required by the applicaiton exist
+      def tables_created?
+        tables.any? { |t| t.table_exists? }
+      end
+  end
+end

data/lib/mir/logger.rb

@@ -0,0 +1,10 @@
+require 'logger'
+
+module Mir
+  def self.logger=(logger)
+    @@logger = logger
+  end
+  def self.logger
+    @@logger
+  end
+end

data/lib/mir/models/app_setting.rb

@@ -0,0 +1,15 @@
+module Mir
+  module Models
+    class AppSetting < ActiveRecord::Base
+      
+      SYNC_PATH = "sync_path"
+      INSTALL_DATE = "installation_date"
+      
+      def self.backup_path
+        record = self.where(:name => SYNC_PATH).first
+        record.value rescue nil
+      end
+
+    end
+  end
+end

data/lib/mir/models/resource.rb

@@ -0,0 +1,97 @@
+require 'digest/md5'
+
+# Represents a local file asset
+module Mir
+  module Models
+    class Resource < ActiveRecord::Base
+      
+      # Builds a resource for the backup index from a file
+      # @param [File] a file object
+      # @param [String] the name of the file on the remote disk
+      # @returns [Resource] a new Resource instance that with a queued status
+      def self.create_from_file_and_name(file, name)
+        is_dir = File.directory?(file)
+        create(:filename => name,
+               :size => file.size,
+               :last_modified => file.ctime,
+               :add_date => DateTime.now,
+               :queued => !is_dir,
+               :checksum => is_dir ? nil : Digest::MD5.file(file).to_s,
+               :is_directory => is_dir)
+      end
+      
+      # Returns true when jobs are still queued
+      def self.pending_jobs?
+        self.where(:queued => true).size > 0
+      end
+      
+      # Yields a Models::Resource object that needs to be synchronized
+      def self.pending_sync_groups(response_size, &block)
+        qry = lambda { Resource.where(:queued => true, :is_directory => false) }
+        chunked_groups(qry, response_size) { |chunk| yield chunk }
+      end
+      
+      # Returns groups of file resources ordered by name
+      # @param [Integer] the number of records to return per chunk
+      # @yields [Array] instances of Models::Resource
+      def self.ordered_groups(group_size = 10)
+        qry = lambda { Resource.order(:filename) }
+        chunked_groups(qry, group_size) { |chunk| yield chunk }
+      end
+      
+      def self.chunked_groups(qry_func, chunk_size)
+        num_results = Resource.count
+        offset = 0
+        pages = (num_results/chunk_size.to_f).ceil
+        pages.times do |i|
+          response = qry_func.call().limit(chunk_size).offset(i*chunk_size)
+          yield response
+        end
+      end
+      
+      # Compares a file asset to the index to deterimine whether the file needs to be updated
+      # @param [String] a path to a file or directory
+      # @returns [Boolean] whether the file and index are in sync with each other
+      def synchronized?(file)
+        if File.exists? file
+          return true if File.directory? file
+          return Digest::MD5.file(file).to_s == self.checksum
+        end
+        false
+      end
+      
+      # Whether the item can be synchronized to a remote disk
+      # @returns [Boolean] true when the resource is not a directory
+      def synchronizable?
+        !is_directory?
+      end
+      
+      # Places the resource into a queueble state
+      def flag_for_update
+        update_attributes :queued => true, :checksum => Digest::MD5.file(abs_path).to_s
+      end
+      
+      def start_progress
+        update_attribute :in_progress, true
+      end
+      
+      def update_success
+        update_attributes :in_progress => false, 
+                          :last_synchronized => DateTime.now, 
+                          :queued => false,
+                          :times_failed => 0
+      end
+      
+      def update_failure
+        num_times_failed = times_failed + 1
+        will_requeue = (num_times_failed < Mir::Application.config.max_upload_retries)
+        update_attributes :times_failed => num_times_failed, :in_progress => false, :queued => will_requeue
+      end
+      
+      def abs_path
+        File.join(Models::AppSetting.backup_path, filename)
+      end
+
+    end
+  end
+end

data/lib/mir/options.rb

@@ -0,0 +1,60 @@
+require 'optparse'
+require 'yaml'
+require 'ostruct'
+
+module Mir
+  class Options
+    
+    USAGE_BANNER = "Usage: mir [options] [settings_file_path] [target]"
+    
+    def self.parse(args)
+      options = OpenStruct.new
+      options.debug = false
+      options.verbose = false
+      options.settings_file = nil
+      options.log_destination = STDOUT
+      options.flush = false
+      options.copy = false
+      options.settings_path = nil
+      
+      opts_parser = OptionParser.new do |opts|
+        opts.banner = USAGE_BANNER
+        opts.separator ""
+        opts.separator "Specific options:"
+        
+        opts.on("--flush", "Flush the file index") do
+          options.flush = true
+        end
+        
+        opts.on("-c", "--copy", "Copy the remote files to target") do
+          options.copy = true
+        end
+        
+        opts.on("--settings", String, "The YAML settings file") do |path|
+          options.settings_path = path
+        end
+        
+        opts.on("-l", "--log-path LOG_FILE", String, "Location for storing execution logs") do |log_file|
+          options.log_destination = log_file
+        end
+        
+        opts.on("-v", "--verbose", "Run verbosely") do |v|
+          options.verbose = true
+        end
+        
+        opts.on_tail("-h", "--help", "Show this message") do
+          puts opts
+          exit
+        end
+        
+        opts.on_tail("--version", "Show version") do
+          puts Mir.version
+          exit
+        end
+      end
+      
+      opts_parser.parse!(args)
+      options
+    end
+  end  
+end

data/lib/mir/utils.rb

@@ -0,0 +1,7 @@
+module Mir
+  module Utils
+    def self.try_create_dir(path)
+      Dir.mkdir(path) unless Dir.exist?(path)
+    end
+  end
+end

data/lib/mir/version.rb

@@ -0,0 +1,7 @@
+module Mir
+  VERSION = [0,1,2]
+  
+  def self.version
+    VERSION.join('.')
+  end
+end

data/lib/mir.rb

@@ -0,0 +1,12 @@
+require "mir/application"
+require "mir/config"
+require "mir/disk"
+require "mir/disk/amazon"
+require "mir/index"
+require "mir/logger"
+require "mir/options"
+require "mir/utils"
+require "mir/version"
+
+module Mir
+end

data/mir.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "mir/version"
+
+Gem::Specification.new do |s|
+  s.name        = "mir"
+  s.version     = Mir.version
+  s.authors     = ["Nate Miller"]
+  s.email       = ["nate@natemiller.org"]
+  s.homepage    = ""
+  s.summary     = %q{A utility for backing up resources}
+  s.description = %q{A commandline tool useful for automating the back up of files to Amazon S3}
+
+  s.rubyforge_project = "mir"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "rspec"
+  
+  s.add_runtime_dependency "bundler"
+  s.add_runtime_dependency "right_aws"
+  s.add_runtime_dependency "activerecord"
+  s.add_runtime_dependency "activesupport"
+  s.add_runtime_dependency "work_queue"
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+require "mir"
+
+RSpec.configure do |config|
+  
+end

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification 
+name: mir
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.1.2
+platform: ruby
+authors: 
+- Nate Miller
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-09-05 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: bundler
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: right_aws
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: work_queue
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id006
+description: A commandline tool useful for automating the back up of files to Amazon S3
+email: 
+- nate@natemiller.org
+executables: 
+- mir
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- Rakefile
+- bin/mir
+- db/migrate/001_create_app_settings.rb
+- db/migrate/002_create_resources.rb
+- lib/mir.rb
+- lib/mir/application.rb
+- lib/mir/config.rb
+- lib/mir/disk.rb
+- lib/mir/disk/amazon.rb
+- lib/mir/index.rb
+- lib/mir/logger.rb
+- lib/mir/models/app_setting.rb
+- lib/mir/models/resource.rb
+- lib/mir/options.rb
+- lib/mir/utils.rb
+- lib/mir/version.rb
+- mir.gemspec
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: mir
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: A utility for backing up resources
+test_files: 
+- spec/spec_helper.rb