easy_sync 0.0.5 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +8 -0
- data/.rspec +2 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +85 -0
- data/CLAUDE.md +133 -0
- data/Gemfile +2 -0
- data/README.md +607 -21
- data/Rakefile +6 -8
- data/bin/easy_sync +8 -1
- data/docs/fine-placement.md +174 -0
- data/docs/hardware-notes.md +71 -0
- data/docs/integrity-scan.md +435 -0
- data/docs/parallel-scrub.md +280 -0
- data/docs/performance.md +146 -0
- data/docs/tripwire.md +232 -0
- data/easy_sync.gemspec +25 -16
- data/lib/easy_sync/cli.rb +1246 -0
- data/lib/easy_sync/config.rb +202 -0
- data/lib/easy_sync/error.rb +5 -0
- data/lib/easy_sync/jbod/backblaze.rb +113 -0
- data/lib/easy_sync/jbod/benchmarker.rb +149 -0
- data/lib/easy_sync/jbod/cleaner.rb +93 -0
- data/lib/easy_sync/jbod/dashboard.rb +611 -0
- data/lib/easy_sync/jbod/keep_awake.rb +33 -0
- data/lib/easy_sync/jbod/manifest.rb +1068 -0
- data/lib/easy_sync/jbod/mirror.rb +204 -0
- data/lib/easy_sync/jbod/models.rb +165 -0
- data/lib/easy_sync/jbod/page_cache.rb +56 -0
- data/lib/easy_sync/jbod/placement.rb +71 -0
- data/lib/easy_sync/jbod/planner.rb +76 -0
- data/lib/easy_sync/jbod/purger.rb +161 -0
- data/lib/easy_sync/jbod/restorer.rb +142 -0
- data/lib/easy_sync/jbod/run_lock.rb +98 -0
- data/lib/easy_sync/jbod/run_log.rb +57 -0
- data/lib/easy_sync/jbod/runner.rb +615 -0
- data/lib/easy_sync/jbod/scrub_pool.rb +136 -0
- data/lib/easy_sync/jbod/scrubber.rb +321 -0
- data/lib/easy_sync/jbod/share_scan.rb +26 -0
- data/lib/easy_sync/jbod/sync_eta.rb +68 -0
- data/lib/easy_sync/jbod/tripwire.rb +80 -0
- data/lib/easy_sync/jbod/volume_info.rb +283 -0
- data/lib/easy_sync/jbod.rb +35 -0
- data/lib/easy_sync/shell.rb +62 -0
- data/lib/easy_sync/templates/dashboard.html.erb +419 -0
- data/lib/easy_sync/version.rb +3 -1
- data/lib/easy_sync.rb +7 -3
- metadata +84 -45
- data/Gemfile.lock +0 -55
- data/Guardfile +0 -28
- data/lib/easy_sync/rsync.rb +0 -60
- data/lib/easy_sync/sync_logger.rb +0 -20
- data/lib/easy_sync/sync_runner.rb +0 -44
- data/spec/easy_sync/rsync_spec.rb +0 -25
- data/spec/easy_sync/sync_runner_spec.rb +0 -54
- data/spec/spec_helper.rb +0 -33
- data/spec/support.rb +0 -89
data/Gemfile.lock
DELETED
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
PATH
|
|
2
|
-
remote: .
|
|
3
|
-
specs:
|
|
4
|
-
easy_sync (0.0.5)
|
|
5
|
-
|
|
6
|
-
GEM
|
|
7
|
-
remote: https://rubygems.org/
|
|
8
|
-
specs:
|
|
9
|
-
celluloid (0.16.0)
|
|
10
|
-
timers (~> 4.0.0)
|
|
11
|
-
coderay (1.1.0)
|
|
12
|
-
ffi (1.9.6)
|
|
13
|
-
formatador (0.2.5)
|
|
14
|
-
guard (2.10.5)
|
|
15
|
-
formatador (>= 0.2.4)
|
|
16
|
-
listen (~> 2.7)
|
|
17
|
-
lumberjack (~> 1.0)
|
|
18
|
-
nenv (~> 0.1)
|
|
19
|
-
pry (>= 0.9.12)
|
|
20
|
-
thor (>= 0.18.1)
|
|
21
|
-
guard-minitest (2.3.2)
|
|
22
|
-
guard (~> 2.0)
|
|
23
|
-
minitest (>= 3.0)
|
|
24
|
-
hitimes (1.2.2)
|
|
25
|
-
listen (2.8.4)
|
|
26
|
-
celluloid (>= 0.15.2)
|
|
27
|
-
rb-fsevent (>= 0.9.3)
|
|
28
|
-
rb-inotify (>= 0.9)
|
|
29
|
-
lumberjack (1.0.9)
|
|
30
|
-
method_source (0.8.2)
|
|
31
|
-
minitest (5.5.0)
|
|
32
|
-
nenv (0.1.1)
|
|
33
|
-
pry (0.10.1)
|
|
34
|
-
coderay (~> 1.1.0)
|
|
35
|
-
method_source (~> 0.8.1)
|
|
36
|
-
slop (~> 3.4)
|
|
37
|
-
rake (10.4.2)
|
|
38
|
-
rb-fsevent (0.9.4)
|
|
39
|
-
rb-inotify (0.9.5)
|
|
40
|
-
ffi (>= 0.5.0)
|
|
41
|
-
slop (3.6.0)
|
|
42
|
-
terminal-notifier-guard (1.6.4)
|
|
43
|
-
thor (0.19.1)
|
|
44
|
-
timers (4.0.1)
|
|
45
|
-
hitimes
|
|
46
|
-
|
|
47
|
-
PLATFORMS
|
|
48
|
-
ruby
|
|
49
|
-
|
|
50
|
-
DEPENDENCIES
|
|
51
|
-
bundler (~> 1.5)
|
|
52
|
-
easy_sync!
|
|
53
|
-
guard-minitest
|
|
54
|
-
rake
|
|
55
|
-
terminal-notifier-guard
|
data/Guardfile
DELETED
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
# A sample Guardfile
|
|
2
|
-
# More info at https://github.com/guard/guard#readme
|
|
3
|
-
|
|
4
|
-
guard :minitest do
|
|
5
|
-
# with Minitest::Unit
|
|
6
|
-
#watch(%r{^test/(.*)\/?test_(.*)\.rb$})
|
|
7
|
-
#watch(%r{^lib/(.*/)?([^/]+)\.rb$}) { |m| "test/#{m[1]}test_#{m[2]}.rb" }
|
|
8
|
-
#watch(%r{^test/test_helper\.rb$}) { 'test' }
|
|
9
|
-
|
|
10
|
-
# with Minitest::Spec
|
|
11
|
-
watch(%r{^spec/(.*)_spec\.rb$})
|
|
12
|
-
#watch(%r{^lib/(.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" }
|
|
13
|
-
watch(%r{^spec/spec_helper\.rb$}) { 'spec' }
|
|
14
|
-
|
|
15
|
-
# Rails 4
|
|
16
|
-
# watch(%r{^app/(.+)\.rb$}) { |m| "test/#{m[1]}_test.rb" }
|
|
17
|
-
# watch(%r{^app/controllers/application_controller\.rb$}) { 'test/controllers' }
|
|
18
|
-
# watch(%r{^app/controllers/(.+)_controller\.rb$}) { |m| "test/integration/#{m[1]}_test.rb" }
|
|
19
|
-
# watch(%r{^app/views/(.+)_mailer/.+}) { |m| "test/mailers/#{m[1]}_mailer_test.rb" }
|
|
20
|
-
# watch(%r{^lib/(.+)\.rb$}) { |m| "test/lib/#{m[1]}_test.rb" }
|
|
21
|
-
# watch(%r{^test/.+_test\.rb$})
|
|
22
|
-
# watch(%r{^test/test_helper\.rb$}) { 'test' }
|
|
23
|
-
|
|
24
|
-
# Rails < 4
|
|
25
|
-
# watch(%r{^app/controllers/(.*)\.rb$}) { |m| "test/functional/#{m[1]}_test.rb" }
|
|
26
|
-
# watch(%r{^app/helpers/(.*)\.rb$}) { |m| "test/helpers/#{m[1]}_test.rb" }
|
|
27
|
-
# watch(%r{^app/models/(.*)\.rb$}) { |m| "test/unit/#{m[1]}_test.rb" }
|
|
28
|
-
end
|
data/lib/easy_sync/rsync.rb
DELETED
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
require 'logger'
|
|
2
|
-
require 'fileutils'
|
|
3
|
-
|
|
4
|
-
module EasySync
|
|
5
|
-
|
|
6
|
-
class Rsync
|
|
7
|
-
attr_reader :sync_name, :source, :destination, :exclude_file, :last_snapshot, :current, :logging
|
|
8
|
-
|
|
9
|
-
def initialize(options)
|
|
10
|
-
@sync_name = options[:sync_name]
|
|
11
|
-
@source = options[:source]
|
|
12
|
-
@destination = options[:destination]
|
|
13
|
-
@exclude_file = options[:exclude_file]
|
|
14
|
-
@logging = options.fetch(:logging){:off}
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
def latest_snapshot
|
|
18
|
-
@last_snapshot = Dir["#{destination}/*"].max
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
def current_snapshot
|
|
22
|
-
@current = File.join destination, Time.now.strftime("%Y-%m-%d")
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
def remove_old_backups
|
|
26
|
-
backups = Dir["#{destination}/*"]
|
|
27
|
-
|
|
28
|
-
(backups - backups.last(5)).each do |r|
|
|
29
|
-
puts "Removing #{r}"
|
|
30
|
-
FileUtils.remove_dir(r, true)
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
def sync
|
|
36
|
-
commands = [
|
|
37
|
-
"rsync",
|
|
38
|
-
"-avhiPH",
|
|
39
|
-
"--exclude-from", "#{exclude_file}",
|
|
40
|
-
"--link-dest", "#{latest_snapshot}"
|
|
41
|
-
]
|
|
42
|
-
|
|
43
|
-
commands << ['--log-file', "#{ENV['HOME']}/easy_sync.log" ] if logging == :on
|
|
44
|
-
commands << ["#{source}", "#{current_snapshot}"]
|
|
45
|
-
commands = commands.flatten
|
|
46
|
-
|
|
47
|
-
puts "\n\n\n------------------ Running #{sync_name} ------------------\n\n\n"
|
|
48
|
-
|
|
49
|
-
remove_old_backups
|
|
50
|
-
|
|
51
|
-
puts "latest snapshot #{@last_snapshot}"
|
|
52
|
-
|
|
53
|
-
IO.popen(commands).each_line do |l|
|
|
54
|
-
puts l
|
|
55
|
-
end
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
end
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
require 'yaml'
|
|
2
|
-
|
|
3
|
-
module EasySync
|
|
4
|
-
|
|
5
|
-
class SyncRunner
|
|
6
|
-
attr_reader :config
|
|
7
|
-
|
|
8
|
-
def initialize()
|
|
9
|
-
handle_config
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
def run
|
|
13
|
-
config[:tasks].each do |c|
|
|
14
|
-
c[:logging] = config[:logging]
|
|
15
|
-
Rsync.new(c).sync
|
|
16
|
-
end
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
private
|
|
20
|
-
|
|
21
|
-
def handle_config
|
|
22
|
-
path = "#{ENV["HOME"]}/.easy_syncrc.yml"
|
|
23
|
-
generate_yaml(path) unless File.exist? path
|
|
24
|
-
@config = YAML.load_file path
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
def generate_yaml(path)
|
|
28
|
-
File.open(path, 'w') do |f|
|
|
29
|
-
h = {:logging => :on,
|
|
30
|
-
:tasks => [{
|
|
31
|
-
sync_name: "sample_sync",
|
|
32
|
-
source: "[/example/path]",
|
|
33
|
-
destination: "[/example/path]",
|
|
34
|
-
exclude_file: "[/example/path]"
|
|
35
|
-
}]}
|
|
36
|
-
f.puts h.to_yaml
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
$stderr.puts "Generated sample config file: #{path}\n\n"
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
end
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
describe Rsync do
|
|
4
|
-
|
|
5
|
-
let(:task){config_file_data[:tasks].first}
|
|
6
|
-
|
|
7
|
-
it "should find the latest snapshot directory" do
|
|
8
|
-
Rsync.new(task) \
|
|
9
|
-
.latest_snapshot.must_equal File.join(destination_directory, "2013-12-30")
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
it "should create the new snapshot directory" do
|
|
13
|
-
Rsync.new(task) \
|
|
14
|
-
.current_snapshot.must_equal File.join(destination_directory, Time.now.strftime("%Y-%m-%d"))
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
it "should log to a file" do
|
|
18
|
-
log_file = "#{temp_directory}/easy_sync.log"
|
|
19
|
-
task[:logging] = :on
|
|
20
|
-
|
|
21
|
-
Rsync.new(task).sync
|
|
22
|
-
File.exist?(log_file).must_equal true
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
end
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
describe SyncRunner do
|
|
4
|
-
|
|
5
|
-
it "should create template config if no config file passed" do
|
|
6
|
-
SyncRunner.new
|
|
7
|
-
config = YAML.load_file default_config_path
|
|
8
|
-
config.must_equal blank_config
|
|
9
|
-
end
|
|
10
|
-
|
|
11
|
-
it "should use default config file" do
|
|
12
|
-
File.open(default_config_path, "w"){|f| f.puts config_file_data.merge({logging: :off}).to_yaml}
|
|
13
|
-
SyncRunner.new.config[:logging].must_equal :off
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
it "should sync files between source and destination" do
|
|
17
|
-
create_config_file
|
|
18
|
-
SyncRunner.new.run
|
|
19
|
-
Dir["#{File.join(destination_directory, Time.now.strftime("%Y-%m-%d"))}/*"].map{|f| File.basename f}.must_equal ["file1.txt", "file2.txt", "file3.txt", "file4.txt"]
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
it "should create hard links to the files that didn't change in the new snapshot directory" do
|
|
23
|
-
create_config_file
|
|
24
|
-
SyncRunner.new.run
|
|
25
|
-
|
|
26
|
-
snapshot_file = Dir["#{File.join(destination_directory, "2013-12-30")}/*"].first
|
|
27
|
-
new_file = Dir["#{File.join(destination_directory, Time.now.strftime("%Y-%m-%d"))}/*"].first
|
|
28
|
-
|
|
29
|
-
File.stat(snapshot_file).ino.must_equal File.stat(new_file).ino
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
it "should use exclude-from option" do
|
|
33
|
-
exclude_file = "#{destination_directory}/exclude.txt"
|
|
34
|
-
File.open(exclude_file, "w"){|f| f.puts "file4.txt"}
|
|
35
|
-
|
|
36
|
-
config_file_data[:tasks].first[:exclude_file] = exclude_file
|
|
37
|
-
File.open(default_config_path, 'w'){|f| f.puts config_file_data.to_yaml}
|
|
38
|
-
|
|
39
|
-
SyncRunner.new.run
|
|
40
|
-
|
|
41
|
-
Dir["#{File.join(destination_directory, Time.now.strftime("%Y-%m-%d"))}/*"].map{|f| File.basename f}.must_equal ["file1.txt", "file2.txt", "file3.txt"]
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
it "should apply log setting to all task" do
|
|
45
|
-
log_file = "#{temp_directory}/easy_sync.log"
|
|
46
|
-
create_config_file
|
|
47
|
-
s = SyncRunner.new
|
|
48
|
-
s.config[:logging] = :on
|
|
49
|
-
s.run
|
|
50
|
-
|
|
51
|
-
File.exist?(log_file).must_equal true
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
end
|
data/spec/spec_helper.rb
DELETED
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
require "minitest/autorun"
|
|
2
|
-
require "minitest/pride"
|
|
3
|
-
require 'pry'
|
|
4
|
-
|
|
5
|
-
require 'easy_sync'
|
|
6
|
-
require_relative 'support'
|
|
7
|
-
|
|
8
|
-
include EasySync
|
|
9
|
-
|
|
10
|
-
class MiniTest::Spec
|
|
11
|
-
|
|
12
|
-
before do
|
|
13
|
-
create_root_directory
|
|
14
|
-
create_source_directory
|
|
15
|
-
create_destination_directory
|
|
16
|
-
create_snapshot_directories
|
|
17
|
-
create_source_files
|
|
18
|
-
copy_to_latest
|
|
19
|
-
|
|
20
|
-
# Silences output, comment out to debug test cases
|
|
21
|
-
$stdout = StringIO.new
|
|
22
|
-
$stderr = StringIO.new
|
|
23
|
-
ENV['HOME'] = temp_directory
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
after do
|
|
28
|
-
FileUtils.rm_r temp_directory
|
|
29
|
-
$stdout = STDOUT
|
|
30
|
-
$stderr = STDERR
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
end
|
data/spec/support.rb
DELETED
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
require 'fileutils'
|
|
2
|
-
require 'tmpdir'
|
|
3
|
-
require 'time'
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
def create_root_directory
|
|
7
|
-
@temp_directory = Dir.mktmpdir('test_data', File.expand_path('./', File.dirname(__FILE__)))
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
def temp_directory
|
|
11
|
-
@temp_directory
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
def create_source_directory
|
|
15
|
-
@source_directory = File.join(temp_directory, 'source')
|
|
16
|
-
Dir.mkdir @source_directory
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
def create_destination_directory
|
|
20
|
-
@destination_directory = File.join(temp_directory, 'backup')
|
|
21
|
-
Dir.mkdir @destination_directory
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
def source_directory
|
|
25
|
-
"#{@source_directory}/"
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
def destination_directory
|
|
29
|
-
@destination_directory
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
def create_snapshot_directories
|
|
33
|
-
%w{2013-12-30 2013-11-10 2013-10-10 2013-11-07}.each do |date|
|
|
34
|
-
path = File.join destination_directory, date
|
|
35
|
-
|
|
36
|
-
Dir.mkdir path
|
|
37
|
-
time = Time.parse date
|
|
38
|
-
|
|
39
|
-
File.utime(time, time, path)
|
|
40
|
-
|
|
41
|
-
end
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
def create_source_files
|
|
45
|
-
%w{file1.txt file2.txt file3.txt file4.txt}.each do |file|
|
|
46
|
-
File.open(File.join(source_directory, file), 'w'){|f| f.puts "This is some sample text"}
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
def copy_to_latest
|
|
52
|
-
%w{file1.txt file2.txt file3.txt}.each do |file|
|
|
53
|
-
FileUtils.cp File.join(source_directory, file), File.join(destination_directory, "2013-12-30")
|
|
54
|
-
end
|
|
55
|
-
end
|
|
56
|
-
|
|
57
|
-
def blank_config
|
|
58
|
-
@blank_config ||= {
|
|
59
|
-
logging: :on,
|
|
60
|
-
tasks: [{
|
|
61
|
-
sync_name: "sample_sync",
|
|
62
|
-
source: "[/example/path]",
|
|
63
|
-
destination: "[/example/path]",
|
|
64
|
-
exclude_file: "[/example/path]"
|
|
65
|
-
}]
|
|
66
|
-
}
|
|
67
|
-
end
|
|
68
|
-
|
|
69
|
-
def config_file_data
|
|
70
|
-
@config_file_data ||= {
|
|
71
|
-
logging: :off,
|
|
72
|
-
tasks: [
|
|
73
|
-
{
|
|
74
|
-
sync_name: "test_sync",
|
|
75
|
-
source: "#{source_directory}",
|
|
76
|
-
destination: "#{destination_directory}",
|
|
77
|
-
exclude_file: ""
|
|
78
|
-
}]
|
|
79
|
-
}
|
|
80
|
-
end
|
|
81
|
-
|
|
82
|
-
def create_config_file
|
|
83
|
-
File.open(default_config_path, 'w'){|f| f.puts config_file_data.to_yaml}
|
|
84
|
-
default_config_path
|
|
85
|
-
end
|
|
86
|
-
|
|
87
|
-
def default_config_path
|
|
88
|
-
"#{temp_directory}/.easy_syncrc.yml"
|
|
89
|
-
end
|