mir 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +4 -0
- data/Gemfile +2 -0
- data/Rakefile +3 -0
- data/bin/mir +3 -0
- data/db/migrate/001_create_app_settings.rb +13 -0
- data/db/migrate/002_create_resources.rb +24 -0
- data/lib/mir/application.rb +147 -0
- data/lib/mir/config.rb +62 -0
- data/lib/mir/disk/amazon.rb +69 -0
- data/lib/mir/disk.rb +17 -0
- data/lib/mir/index.rb +92 -0
- data/lib/mir/logger.rb +10 -0
- data/lib/mir/models/app_setting.rb +15 -0
- data/lib/mir/models/resource.rb +97 -0
- data/lib/mir/options.rb +60 -0
- data/lib/mir/utils.rb +7 -0
- data/lib/mir/version.rb +7 -0
- data/lib/mir.rb +12 -0
- data/mir.gemspec +28 -0
- data/spec/spec_helper.rb +5 -0
- metadata +141 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Rakefile
ADDED
data/bin/mir
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
class CreateResources < ActiveRecord::Migration
|
2
|
+
def self.up
|
3
|
+
create_table :resources do |t|
|
4
|
+
t.string :filename, :null => false
|
5
|
+
t.integer :size, :default => 0, :null => false
|
6
|
+
t.string :checksum, :limit => 32
|
7
|
+
t.datetime :last_modified
|
8
|
+
t.datetime :add_date, :default => DateTime.now
|
9
|
+
t.datetime :last_synchronized
|
10
|
+
t.boolean :is_directory, :default => false, :null => false
|
11
|
+
t.boolean :in_progress, :default => false
|
12
|
+
t.boolean :queued, :default => false
|
13
|
+
t.integer :times_failed, :default => 0, :null => false
|
14
|
+
end
|
15
|
+
|
16
|
+
add_index :resources, :filename, :unique => true
|
17
|
+
add_index :resources, :in_progress
|
18
|
+
add_index :resources, :queued
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.down
|
22
|
+
drop_table :resources
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,147 @@
|
|
1
|
+
require "benchmark"
|
2
|
+
require "fileutils"
|
3
|
+
require "work_queue"
|
4
|
+
|
5
|
+
module Mir
|
6
|
+
|
7
|
+
class Application
|
8
|
+
|
9
|
+
DEFAULT_SETTINGS_FILE_NAME = "mir_settings.yml"
|
10
|
+
DEFAULT_BATCH_SIZE = 20
|
11
|
+
|
12
|
+
# Creates a new Mir instance
|
13
|
+
def self.start
|
14
|
+
new.start
|
15
|
+
end
|
16
|
+
|
17
|
+
attr_reader :options, :disk, :index
|
18
|
+
|
19
|
+
def initialize
|
20
|
+
@options = Mir::Options.parse(ARGV)
|
21
|
+
Mir.logger = Logger.new(options.log_destination)
|
22
|
+
end
|
23
|
+
|
24
|
+
def start
|
25
|
+
if options.copy && options.flush
|
26
|
+
Mir.logger.error "Conflicting options: Cannot copy from remote source with an empty file index"
|
27
|
+
exit
|
28
|
+
end
|
29
|
+
|
30
|
+
@@config = Config.new find_settings_file
|
31
|
+
param_path = File.expand_path(ARGV[0])
|
32
|
+
exit unless config.valid?
|
33
|
+
|
34
|
+
# Initialize our remote disk
|
35
|
+
@disk = Disk.fetch(config.cloud_provider)
|
36
|
+
exit unless @disk.connected?
|
37
|
+
|
38
|
+
# Initialize our local index
|
39
|
+
@index = Index.new(param_path, config.database)
|
40
|
+
@index.setup(:verbose => options.verbose, :force_flush => options.flush)
|
41
|
+
|
42
|
+
options.copy ? pull(param_path) : push(param_path)
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.config
|
46
|
+
@@config
|
47
|
+
end
|
48
|
+
|
49
|
+
def config
|
50
|
+
@@config
|
51
|
+
end
|
52
|
+
|
53
|
+
private
|
54
|
+
# Returns a path to the settings file. If the file was provided as an option it will always
|
55
|
+
# be returned. When no option is passed, the file 'mir_settings.yml' will be searched for
|
56
|
+
# in the following paths: $HOME, /etc/mir
|
57
|
+
# @return [String] path to the settings or nil if none is found
|
58
|
+
def find_settings_file
|
59
|
+
if !options.settings_path.nil?
|
60
|
+
return options.settings_path
|
61
|
+
else
|
62
|
+
["~", "/etc/mir"].each do |dir|
|
63
|
+
path = File.expand_path(File.join(dir, DEFAULT_SETTINGS_FILE_NAME))
|
64
|
+
return path.to_s if File.exist?(path)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
nil
|
68
|
+
end
|
69
|
+
|
70
|
+
# Synchronize the local files to the remote disk
|
71
|
+
def push(target)
|
72
|
+
Mir.logger.info "Starting push operation"
|
73
|
+
|
74
|
+
if Models::AppSetting.backup_path != target
|
75
|
+
Mir.logger.error "Target does not match directory stored in index"
|
76
|
+
exit
|
77
|
+
end
|
78
|
+
|
79
|
+
index.update
|
80
|
+
|
81
|
+
time = Benchmark.measure do
|
82
|
+
queue = WorkQueue.new(config.max_threads)
|
83
|
+
while Models::Resource.pending_jobs? do
|
84
|
+
Models::Resource.pending_sync_groups(DEFAULT_BATCH_SIZE) do |resources|
|
85
|
+
push_group(queue, resources)
|
86
|
+
handle_push_failures(resources)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
Mir.logger.info time
|
91
|
+
end
|
92
|
+
|
93
|
+
|
94
|
+
# Uploads a collection of resouces. Blocks until all items in queue have been processed
|
95
|
+
# @param [WorkQueue] a submission queue to manage resource uploads
|
96
|
+
# @param [Array] an array of Models::Resource objects that need to be uploaded
|
97
|
+
def push_group(work_queue, resources)
|
98
|
+
resources.each do |resource|
|
99
|
+
work_queue.enqueue_b do
|
100
|
+
resource.start_progress
|
101
|
+
disk.write resource.abs_path
|
102
|
+
resource.update_success
|
103
|
+
end
|
104
|
+
end
|
105
|
+
work_queue.join
|
106
|
+
end
|
107
|
+
|
108
|
+
# Scans a collection of resources for jobs that did no complete successfully and flags them
|
109
|
+
# for resubmission
|
110
|
+
# @param [Array] an array of Models::Resources
|
111
|
+
def handle_push_failures(resources)
|
112
|
+
resources.each do |resource|
|
113
|
+
if resource.in_progress?
|
114
|
+
Mir.logger.info "Resource '#{resource.abs_path}' failed to upload"
|
115
|
+
resource.update_failure
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
# Copy the remote disk contents into the specified directory
|
121
|
+
def pull(target)
|
122
|
+
Utils.try_create_dir(target)
|
123
|
+
write_dir = Dir.new(target)
|
124
|
+
Mir.logger.info "Copying remote disk to #{write_dir.path} using #{config.max_threads} threads"
|
125
|
+
|
126
|
+
time = Benchmark.measure do
|
127
|
+
queue = WorkQueue.new(config.max_threads)
|
128
|
+
|
129
|
+
# loop through each resource. If the resource is a directory, create the path
|
130
|
+
# otherwise download the file
|
131
|
+
Models::Resource.ordered_groups(DEFAULT_BATCH_SIZE) do |resources|
|
132
|
+
resources.each do |resource|
|
133
|
+
dest = File.join(write_dir.path, resource.filename)
|
134
|
+
if resource.is_directory?
|
135
|
+
Utils.try_create_dir(dest)
|
136
|
+
elsif !resource.synchronized?(dest)
|
137
|
+
queue.enqueue_b { @disk.copy(resource.abs_path, dest) }
|
138
|
+
end
|
139
|
+
end
|
140
|
+
queue.join
|
141
|
+
end
|
142
|
+
end
|
143
|
+
Mir.logger.info time
|
144
|
+
end
|
145
|
+
|
146
|
+
end
|
147
|
+
end
|
data/lib/mir/config.rb
ADDED
@@ -0,0 +1,62 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
# The config class is used for storage of user settings and preferences releated to
|
4
|
+
# S3 storage
|
5
|
+
module Mir
|
6
|
+
class UndefinedConfigValue < StandardError; end
|
7
|
+
|
8
|
+
class Config
|
9
|
+
|
10
|
+
def initialize(config_file = nil)
|
11
|
+
@config_file = config_file
|
12
|
+
@settings, @database = nil, nil
|
13
|
+
end
|
14
|
+
|
15
|
+
# Validates configuration settings
|
16
|
+
def valid?
|
17
|
+
if @config_file.nil? or !File.exist?(@config_file)
|
18
|
+
Mir.logger.error("Configuration file not found")
|
19
|
+
return false
|
20
|
+
end
|
21
|
+
|
22
|
+
if File.directory?(@config_file)
|
23
|
+
Mir.logger.error("Received directory instead of settings file")
|
24
|
+
return false
|
25
|
+
end
|
26
|
+
|
27
|
+
File.open(@config_file) do |f|
|
28
|
+
yml = YAML::load(f)
|
29
|
+
if yml.key? "settings"
|
30
|
+
@settings = OpenStruct.new yml["settings"]
|
31
|
+
@settings.database = symbolize_keys(@settings.database)
|
32
|
+
@settings.cloud_provider = symbolize_keys(@settings.cloud_provider)
|
33
|
+
else
|
34
|
+
Mir.logger.error("Malformed config file")
|
35
|
+
return false
|
36
|
+
end
|
37
|
+
end
|
38
|
+
true
|
39
|
+
end
|
40
|
+
|
41
|
+
|
42
|
+
|
43
|
+
def method_missing(meth, *args, &block)
|
44
|
+
val = @settings.send(meth)
|
45
|
+
unless val.nil?
|
46
|
+
val
|
47
|
+
else
|
48
|
+
raise UndefinedConfigValue
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
private
|
53
|
+
def symbolize_keys(hsh)
|
54
|
+
symbolized = hsh.inject({}) do |opts, (k,v)|
|
55
|
+
opts[(k.to_sym rescue k) || k] = v
|
56
|
+
opts
|
57
|
+
end
|
58
|
+
symbolized
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require "right_aws"
|
2
|
+
|
3
|
+
module Mir
|
4
|
+
module Disk
|
5
|
+
class Amazon
|
6
|
+
|
7
|
+
attr_reader :bucket_name
|
8
|
+
|
9
|
+
def self.key_name(path)
|
10
|
+
if path[0] == File::SEPARATOR
|
11
|
+
path[1..-1]
|
12
|
+
else
|
13
|
+
path
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def initialize(settings = {})
|
18
|
+
@bucket_name = settings[:bucket_name]
|
19
|
+
@access_key_id = settings[:access_key_id]
|
20
|
+
@secret_access_key = settings[:secret_access_key]
|
21
|
+
@connection = try_connect
|
22
|
+
end
|
23
|
+
|
24
|
+
# Returns the buckets available from S3
|
25
|
+
def collections
|
26
|
+
@connection.list_bucket.select(:key)
|
27
|
+
end
|
28
|
+
|
29
|
+
# Copies the remote resource to the local filesystem
|
30
|
+
# @param [String] the remote name of the resource to copy
|
31
|
+
# @param [String] the local name of the destination
|
32
|
+
def copy(from, to)
|
33
|
+
open(to, 'w') do |file|
|
34
|
+
@connection.get(bucket_name, self.class.key_name(from)) { |chunk| file.write(chunk) }
|
35
|
+
end
|
36
|
+
Mir.logger.info "Completed download '#{to}'"
|
37
|
+
end
|
38
|
+
|
39
|
+
def connected?
|
40
|
+
@connection_success
|
41
|
+
end
|
42
|
+
|
43
|
+
def volume
|
44
|
+
@connection.bucket(bucket_name, true)
|
45
|
+
end
|
46
|
+
|
47
|
+
def write(file_path)
|
48
|
+
@connection.put(bucket_name, self.class.key_name(file_path), File.open(file_path))
|
49
|
+
Mir.logger.info "Completed upload #{file_path}"
|
50
|
+
end
|
51
|
+
|
52
|
+
private
|
53
|
+
def try_connect
|
54
|
+
begin
|
55
|
+
conn = RightAws::S3Interface.new(@access_key_id, @secret_access_key, {
|
56
|
+
:multi_thread => true,
|
57
|
+
:logger => Mir.logger
|
58
|
+
})
|
59
|
+
@connection_success = true
|
60
|
+
return conn
|
61
|
+
rescue Exception => e
|
62
|
+
@connection_success = false
|
63
|
+
Mir.logger.error "Could not establish connection with S3: '#{e.message}'"
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
data/lib/mir/disk.rb
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
# The remote storage container where objects are saved
|
2
|
+
module Mir
|
3
|
+
module Disk
|
4
|
+
|
5
|
+
# Returns a disk object from the settings specified
|
6
|
+
def self.fetch(settings = {})
|
7
|
+
case settings[:type]
|
8
|
+
when "s3"
|
9
|
+
Mir::Disk::Amazon.new(settings)
|
10
|
+
else
|
11
|
+
Mir.logger.error "Could not find specified cloud provider"
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
|
16
|
+
end
|
17
|
+
end
|
data/lib/mir/index.rb
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require "active_support/inflector"
|
3
|
+
|
4
|
+
# Manages database operations for application
|
5
|
+
module Mir
|
6
|
+
class Index
|
7
|
+
|
8
|
+
MIGRATIONS_PATH = File.join(File.dirname(__FILE__), "..", "..", "db", "migrate")
|
9
|
+
|
10
|
+
# Returns a databse object used to connect to the indexing database
|
11
|
+
# @param [Hash] database configuration settings. See ActiveRecord#Base::establish_connection
|
12
|
+
def initialize(sync_path, connection_params)
|
13
|
+
@sync_path = sync_path
|
14
|
+
@connection_params = connection_params
|
15
|
+
end
|
16
|
+
|
17
|
+
attr_reader :sync_path
|
18
|
+
|
19
|
+
# Creates necessary database and tables if this is the first time connecting
|
20
|
+
# @option opts [Boolean] :verbose Enable on ActiveRecord reporting
|
21
|
+
# @option opts [Boolean] :force_flush Rebuild index no matter what
|
22
|
+
def setup(options = {})
|
23
|
+
options[:force_flush] ||= false
|
24
|
+
options[:verbose] ||= false
|
25
|
+
@connection = ActiveRecord::Base.establish_connection(@connection_params).connection
|
26
|
+
ActiveRecord::Base.timestamped_migrations = false
|
27
|
+
|
28
|
+
if options[:verbose]
|
29
|
+
ActiveRecord::Base.logger = Mir.logger
|
30
|
+
ActiveRecord::Migration.verbose = true
|
31
|
+
end
|
32
|
+
|
33
|
+
load_tables
|
34
|
+
rebuild if !tables_created? or options[:force_flush]
|
35
|
+
end
|
36
|
+
|
37
|
+
# Scans the filesystem and flags any resources which need updating
|
38
|
+
def update
|
39
|
+
Mir.logger.info "Updating backup index for '#{sync_path}'"
|
40
|
+
Dir.glob(File.join(sync_path, "**", "*")) do |f|
|
41
|
+
fname = relative_path(f)
|
42
|
+
file = File.new(f)
|
43
|
+
resource = Models::Resource.find_by_filename(fname)
|
44
|
+
|
45
|
+
if resource.nil?
|
46
|
+
Mir.logger.info "Adding file to index #{fname}"
|
47
|
+
resource = Models::Resource.create_from_file_and_name(file, fname)
|
48
|
+
else
|
49
|
+
resource.flag_for_update unless resource.synchronized?(file)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
Mir.logger.info "Index updated"
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
# Returns the path of a file relative to the backup directory
|
57
|
+
def relative_path(file)
|
58
|
+
File.absolute_path(file).gsub(sync_path,'')
|
59
|
+
end
|
60
|
+
|
61
|
+
# Regenerates the file system index for the backup directory
|
62
|
+
def rebuild
|
63
|
+
tables.each { |t| ActiveRecord::Migration.drop_table(t.table_name) if t.table_exists? }
|
64
|
+
ActiveRecord::Migration.drop_table(:schema_migrations) if @connection.table_exists? :schema_migrations
|
65
|
+
ActiveRecord::Migrator.migrate MIGRATIONS_PATH
|
66
|
+
Models::AppSetting.create(:name => Models::AppSetting::SYNC_PATH, :value => sync_path)
|
67
|
+
Models::AppSetting.create(:name => Models::AppSetting::INSTALL_DATE, :value => DateTime.now.to_s)
|
68
|
+
end
|
69
|
+
|
70
|
+
def load_tables
|
71
|
+
@tables = []
|
72
|
+
models = File.join(File.dirname(__FILE__), "models", "*.rb")
|
73
|
+
|
74
|
+
# load the AR models for the application
|
75
|
+
Dir.glob(models) do |f|
|
76
|
+
require f
|
77
|
+
name = "Models::" + ActiveSupport::Inflector.camelize(File.basename(f, ".rb"))
|
78
|
+
@tables << eval(name)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
# Returns the activerecord classes for each table used by the application
|
83
|
+
def tables
|
84
|
+
@tables
|
85
|
+
end
|
86
|
+
|
87
|
+
# Checks whether any of the tables required by the applicaiton exist
|
88
|
+
def tables_created?
|
89
|
+
tables.any? { |t| t.table_exists? }
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
data/lib/mir/logger.rb
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
module Mir
|
2
|
+
module Models
|
3
|
+
class AppSetting < ActiveRecord::Base
|
4
|
+
|
5
|
+
SYNC_PATH = "sync_path"
|
6
|
+
INSTALL_DATE = "installation_date"
|
7
|
+
|
8
|
+
def self.backup_path
|
9
|
+
record = self.where(:name => SYNC_PATH).first
|
10
|
+
record.value rescue nil
|
11
|
+
end
|
12
|
+
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
require 'digest/md5'
|
2
|
+
|
3
|
+
# Represents a local file asset
|
4
|
+
module Mir
|
5
|
+
module Models
|
6
|
+
class Resource < ActiveRecord::Base
|
7
|
+
|
8
|
+
# Builds a resource for the backup index from a file
|
9
|
+
# @param [File] a file object
|
10
|
+
# @param [String] the name of the file on the remote disk
|
11
|
+
# @returns [Resource] a new Resource instance that with a queued status
|
12
|
+
def self.create_from_file_and_name(file, name)
|
13
|
+
is_dir = File.directory?(file)
|
14
|
+
create(:filename => name,
|
15
|
+
:size => file.size,
|
16
|
+
:last_modified => file.ctime,
|
17
|
+
:add_date => DateTime.now,
|
18
|
+
:queued => !is_dir,
|
19
|
+
:checksum => is_dir ? nil : Digest::MD5.file(file).to_s,
|
20
|
+
:is_directory => is_dir)
|
21
|
+
end
|
22
|
+
|
23
|
+
# Returns true when jobs are still queued
|
24
|
+
def self.pending_jobs?
|
25
|
+
self.where(:queued => true).size > 0
|
26
|
+
end
|
27
|
+
|
28
|
+
# Yields a Models::Resource object that needs to be synchronized
|
29
|
+
def self.pending_sync_groups(response_size, &block)
|
30
|
+
qry = lambda { Resource.where(:queued => true, :is_directory => false) }
|
31
|
+
chunked_groups(qry, response_size) { |chunk| yield chunk }
|
32
|
+
end
|
33
|
+
|
34
|
+
# Returns groups of file resources ordered by name
|
35
|
+
# @param [Integer] the number of records to return per chunk
|
36
|
+
# @yields [Array] instances of Models::Resource
|
37
|
+
def self.ordered_groups(group_size = 10)
|
38
|
+
qry = lambda { Resource.order(:filename) }
|
39
|
+
chunked_groups(qry, group_size) { |chunk| yield chunk }
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.chunked_groups(qry_func, chunk_size)
|
43
|
+
num_results = Resource.count
|
44
|
+
offset = 0
|
45
|
+
pages = (num_results/chunk_size.to_f).ceil
|
46
|
+
pages.times do |i|
|
47
|
+
response = qry_func.call().limit(chunk_size).offset(i*chunk_size)
|
48
|
+
yield response
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
# Compares a file asset to the index to deterimine whether the file needs to be updated
|
53
|
+
# @param [String] a path to a file or directory
|
54
|
+
# @returns [Boolean] whether the file and index are in sync with each other
|
55
|
+
def synchronized?(file)
|
56
|
+
if File.exists? file
|
57
|
+
return true if File.directory? file
|
58
|
+
return Digest::MD5.file(file).to_s == self.checksum
|
59
|
+
end
|
60
|
+
false
|
61
|
+
end
|
62
|
+
|
63
|
+
# Whether the item can be synchronized to a remote disk
|
64
|
+
# @returns [Boolean] true when the resource is not a directory
|
65
|
+
def synchronizable?
|
66
|
+
!is_directory?
|
67
|
+
end
|
68
|
+
|
69
|
+
# Places the resource into a queueble state
|
70
|
+
def flag_for_update
|
71
|
+
update_attributes :queued => true, :checksum => Digest::MD5.file(abs_path).to_s
|
72
|
+
end
|
73
|
+
|
74
|
+
def start_progress
|
75
|
+
update_attribute :in_progress, true
|
76
|
+
end
|
77
|
+
|
78
|
+
def update_success
|
79
|
+
update_attributes :in_progress => false,
|
80
|
+
:last_synchronized => DateTime.now,
|
81
|
+
:queued => false,
|
82
|
+
:times_failed => 0
|
83
|
+
end
|
84
|
+
|
85
|
+
def update_failure
|
86
|
+
num_times_failed = times_failed + 1
|
87
|
+
will_requeue = (num_times_failed < Mir::Application.config.max_upload_retries)
|
88
|
+
update_attributes :times_failed => num_times_failed, :in_progress => false, :queued => will_requeue
|
89
|
+
end
|
90
|
+
|
91
|
+
def abs_path
|
92
|
+
File.join(Models::AppSetting.backup_path, filename)
|
93
|
+
end
|
94
|
+
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
data/lib/mir/options.rb
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'yaml'
|
3
|
+
require 'ostruct'
|
4
|
+
|
5
|
+
module Mir
|
6
|
+
class Options
|
7
|
+
|
8
|
+
USAGE_BANNER = "Usage: mir [options] [settings_file_path] [target]"
|
9
|
+
|
10
|
+
def self.parse(args)
|
11
|
+
options = OpenStruct.new
|
12
|
+
options.debug = false
|
13
|
+
options.verbose = false
|
14
|
+
options.settings_file = nil
|
15
|
+
options.log_destination = STDOUT
|
16
|
+
options.flush = false
|
17
|
+
options.copy = false
|
18
|
+
options.settings_path = nil
|
19
|
+
|
20
|
+
opts_parser = OptionParser.new do |opts|
|
21
|
+
opts.banner = USAGE_BANNER
|
22
|
+
opts.separator ""
|
23
|
+
opts.separator "Specific options:"
|
24
|
+
|
25
|
+
opts.on("--flush", "Flush the file index") do
|
26
|
+
options.flush = true
|
27
|
+
end
|
28
|
+
|
29
|
+
opts.on("-c", "--copy", "Copy the remote files to target") do
|
30
|
+
options.copy = true
|
31
|
+
end
|
32
|
+
|
33
|
+
opts.on("--settings", String, "The YAML settings file") do |path|
|
34
|
+
options.settings_path = path
|
35
|
+
end
|
36
|
+
|
37
|
+
opts.on("-l", "--log-path LOG_FILE", String, "Location for storing execution logs") do |log_file|
|
38
|
+
options.log_destination = log_file
|
39
|
+
end
|
40
|
+
|
41
|
+
opts.on("-v", "--verbose", "Run verbosely") do |v|
|
42
|
+
options.verbose = true
|
43
|
+
end
|
44
|
+
|
45
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
46
|
+
puts opts
|
47
|
+
exit
|
48
|
+
end
|
49
|
+
|
50
|
+
opts.on_tail("--version", "Show version") do
|
51
|
+
puts Mir.version
|
52
|
+
exit
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
opts_parser.parse!(args)
|
57
|
+
options
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
data/lib/mir/utils.rb
ADDED
data/lib/mir/version.rb
ADDED
data/lib/mir.rb
ADDED
data/mir.gemspec
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "mir/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "mir"
|
7
|
+
s.version = Mir.version
|
8
|
+
s.authors = ["Nate Miller"]
|
9
|
+
s.email = ["nate@natemiller.org"]
|
10
|
+
s.homepage = ""
|
11
|
+
s.summary = %q{A utility for backing up resources}
|
12
|
+
s.description = %q{A commandline tool useful for automating the back up of files to Amazon S3}
|
13
|
+
|
14
|
+
s.rubyforge_project = "mir"
|
15
|
+
|
16
|
+
s.files = `git ls-files`.split("\n")
|
17
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
18
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
19
|
+
s.require_paths = ["lib"]
|
20
|
+
|
21
|
+
s.add_development_dependency "rspec"
|
22
|
+
|
23
|
+
s.add_runtime_dependency "bundler"
|
24
|
+
s.add_runtime_dependency "right_aws"
|
25
|
+
s.add_runtime_dependency "activerecord"
|
26
|
+
s.add_runtime_dependency "activesupport"
|
27
|
+
s.add_runtime_dependency "work_queue"
|
28
|
+
end
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,141 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: mir
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease:
|
5
|
+
version: 0.1.2
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Nate Miller
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
|
13
|
+
date: 2011-09-05 00:00:00 -07:00
|
14
|
+
default_executable:
|
15
|
+
dependencies:
|
16
|
+
- !ruby/object:Gem::Dependency
|
17
|
+
name: rspec
|
18
|
+
prerelease: false
|
19
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
20
|
+
none: false
|
21
|
+
requirements:
|
22
|
+
- - ">="
|
23
|
+
- !ruby/object:Gem::Version
|
24
|
+
version: "0"
|
25
|
+
type: :development
|
26
|
+
version_requirements: *id001
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler
|
29
|
+
prerelease: false
|
30
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
31
|
+
none: false
|
32
|
+
requirements:
|
33
|
+
- - ">="
|
34
|
+
- !ruby/object:Gem::Version
|
35
|
+
version: "0"
|
36
|
+
type: :runtime
|
37
|
+
version_requirements: *id002
|
38
|
+
- !ruby/object:Gem::Dependency
|
39
|
+
name: right_aws
|
40
|
+
prerelease: false
|
41
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
42
|
+
none: false
|
43
|
+
requirements:
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: "0"
|
47
|
+
type: :runtime
|
48
|
+
version_requirements: *id003
|
49
|
+
- !ruby/object:Gem::Dependency
|
50
|
+
name: activerecord
|
51
|
+
prerelease: false
|
52
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
53
|
+
none: false
|
54
|
+
requirements:
|
55
|
+
- - ">="
|
56
|
+
- !ruby/object:Gem::Version
|
57
|
+
version: "0"
|
58
|
+
type: :runtime
|
59
|
+
version_requirements: *id004
|
60
|
+
- !ruby/object:Gem::Dependency
|
61
|
+
name: activesupport
|
62
|
+
prerelease: false
|
63
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
64
|
+
none: false
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: "0"
|
69
|
+
type: :runtime
|
70
|
+
version_requirements: *id005
|
71
|
+
- !ruby/object:Gem::Dependency
|
72
|
+
name: work_queue
|
73
|
+
prerelease: false
|
74
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
75
|
+
none: false
|
76
|
+
requirements:
|
77
|
+
- - ">="
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: "0"
|
80
|
+
type: :runtime
|
81
|
+
version_requirements: *id006
|
82
|
+
description: A commandline tool useful for automating the back up of files to Amazon S3
|
83
|
+
email:
|
84
|
+
- nate@natemiller.org
|
85
|
+
executables:
|
86
|
+
- mir
|
87
|
+
extensions: []
|
88
|
+
|
89
|
+
extra_rdoc_files: []
|
90
|
+
|
91
|
+
files:
|
92
|
+
- .gitignore
|
93
|
+
- Gemfile
|
94
|
+
- Rakefile
|
95
|
+
- bin/mir
|
96
|
+
- db/migrate/001_create_app_settings.rb
|
97
|
+
- db/migrate/002_create_resources.rb
|
98
|
+
- lib/mir.rb
|
99
|
+
- lib/mir/application.rb
|
100
|
+
- lib/mir/config.rb
|
101
|
+
- lib/mir/disk.rb
|
102
|
+
- lib/mir/disk/amazon.rb
|
103
|
+
- lib/mir/index.rb
|
104
|
+
- lib/mir/logger.rb
|
105
|
+
- lib/mir/models/app_setting.rb
|
106
|
+
- lib/mir/models/resource.rb
|
107
|
+
- lib/mir/options.rb
|
108
|
+
- lib/mir/utils.rb
|
109
|
+
- lib/mir/version.rb
|
110
|
+
- mir.gemspec
|
111
|
+
- spec/spec_helper.rb
|
112
|
+
has_rdoc: true
|
113
|
+
homepage: ""
|
114
|
+
licenses: []
|
115
|
+
|
116
|
+
post_install_message:
|
117
|
+
rdoc_options: []
|
118
|
+
|
119
|
+
require_paths:
|
120
|
+
- lib
|
121
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
122
|
+
none: false
|
123
|
+
requirements:
|
124
|
+
- - ">="
|
125
|
+
- !ruby/object:Gem::Version
|
126
|
+
version: "0"
|
127
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
128
|
+
none: false
|
129
|
+
requirements:
|
130
|
+
- - ">="
|
131
|
+
- !ruby/object:Gem::Version
|
132
|
+
version: "0"
|
133
|
+
requirements: []
|
134
|
+
|
135
|
+
rubyforge_project: mir
|
136
|
+
rubygems_version: 1.6.2
|
137
|
+
signing_key:
|
138
|
+
specification_version: 3
|
139
|
+
summary: A utility for backing up resources
|
140
|
+
test_files:
|
141
|
+
- spec/spec_helper.rb
|