easy_sync 0.0.5 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +8 -0
  3. data/.rspec +2 -0
  4. data/.ruby-version +1 -0
  5. data/CHANGELOG.md +85 -0
  6. data/CLAUDE.md +133 -0
  7. data/Gemfile +2 -0
  8. data/README.md +607 -21
  9. data/Rakefile +6 -8
  10. data/bin/easy_sync +8 -1
  11. data/docs/fine-placement.md +174 -0
  12. data/docs/hardware-notes.md +71 -0
  13. data/docs/integrity-scan.md +435 -0
  14. data/docs/parallel-scrub.md +280 -0
  15. data/docs/performance.md +146 -0
  16. data/docs/tripwire.md +232 -0
  17. data/easy_sync.gemspec +25 -16
  18. data/lib/easy_sync/cli.rb +1246 -0
  19. data/lib/easy_sync/config.rb +202 -0
  20. data/lib/easy_sync/error.rb +5 -0
  21. data/lib/easy_sync/jbod/backblaze.rb +113 -0
  22. data/lib/easy_sync/jbod/benchmarker.rb +149 -0
  23. data/lib/easy_sync/jbod/cleaner.rb +93 -0
  24. data/lib/easy_sync/jbod/dashboard.rb +611 -0
  25. data/lib/easy_sync/jbod/keep_awake.rb +33 -0
  26. data/lib/easy_sync/jbod/manifest.rb +1068 -0
  27. data/lib/easy_sync/jbod/mirror.rb +204 -0
  28. data/lib/easy_sync/jbod/models.rb +165 -0
  29. data/lib/easy_sync/jbod/page_cache.rb +56 -0
  30. data/lib/easy_sync/jbod/placement.rb +71 -0
  31. data/lib/easy_sync/jbod/planner.rb +76 -0
  32. data/lib/easy_sync/jbod/purger.rb +161 -0
  33. data/lib/easy_sync/jbod/restorer.rb +142 -0
  34. data/lib/easy_sync/jbod/run_lock.rb +98 -0
  35. data/lib/easy_sync/jbod/run_log.rb +57 -0
  36. data/lib/easy_sync/jbod/runner.rb +615 -0
  37. data/lib/easy_sync/jbod/scrub_pool.rb +136 -0
  38. data/lib/easy_sync/jbod/scrubber.rb +321 -0
  39. data/lib/easy_sync/jbod/share_scan.rb +26 -0
  40. data/lib/easy_sync/jbod/sync_eta.rb +68 -0
  41. data/lib/easy_sync/jbod/tripwire.rb +80 -0
  42. data/lib/easy_sync/jbod/volume_info.rb +283 -0
  43. data/lib/easy_sync/jbod.rb +35 -0
  44. data/lib/easy_sync/shell.rb +62 -0
  45. data/lib/easy_sync/templates/dashboard.html.erb +419 -0
  46. data/lib/easy_sync/version.rb +3 -1
  47. data/lib/easy_sync.rb +7 -3
  48. metadata +84 -45
  49. data/Gemfile.lock +0 -55
  50. data/Guardfile +0 -28
  51. data/lib/easy_sync/rsync.rb +0 -60
  52. data/lib/easy_sync/sync_logger.rb +0 -20
  53. data/lib/easy_sync/sync_runner.rb +0 -44
  54. data/spec/easy_sync/rsync_spec.rb +0 -25
  55. data/spec/easy_sync/sync_runner_spec.rb +0 -54
  56. data/spec/spec_helper.rb +0 -33
  57. data/spec/support.rb +0 -89
data/Gemfile.lock DELETED
@@ -1,55 +0,0 @@
1
- PATH
2
- remote: .
3
- specs:
4
- easy_sync (0.0.5)
5
-
6
- GEM
7
- remote: https://rubygems.org/
8
- specs:
9
- celluloid (0.16.0)
10
- timers (~> 4.0.0)
11
- coderay (1.1.0)
12
- ffi (1.9.6)
13
- formatador (0.2.5)
14
- guard (2.10.5)
15
- formatador (>= 0.2.4)
16
- listen (~> 2.7)
17
- lumberjack (~> 1.0)
18
- nenv (~> 0.1)
19
- pry (>= 0.9.12)
20
- thor (>= 0.18.1)
21
- guard-minitest (2.3.2)
22
- guard (~> 2.0)
23
- minitest (>= 3.0)
24
- hitimes (1.2.2)
25
- listen (2.8.4)
26
- celluloid (>= 0.15.2)
27
- rb-fsevent (>= 0.9.3)
28
- rb-inotify (>= 0.9)
29
- lumberjack (1.0.9)
30
- method_source (0.8.2)
31
- minitest (5.5.0)
32
- nenv (0.1.1)
33
- pry (0.10.1)
34
- coderay (~> 1.1.0)
35
- method_source (~> 0.8.1)
36
- slop (~> 3.4)
37
- rake (10.4.2)
38
- rb-fsevent (0.9.4)
39
- rb-inotify (0.9.5)
40
- ffi (>= 0.5.0)
41
- slop (3.6.0)
42
- terminal-notifier-guard (1.6.4)
43
- thor (0.19.1)
44
- timers (4.0.1)
45
- hitimes
46
-
47
- PLATFORMS
48
- ruby
49
-
50
- DEPENDENCIES
51
- bundler (~> 1.5)
52
- easy_sync!
53
- guard-minitest
54
- rake
55
- terminal-notifier-guard
data/Guardfile DELETED
@@ -1,28 +0,0 @@
1
- # A sample Guardfile
2
- # More info at https://github.com/guard/guard#readme
3
-
4
- guard :minitest do
5
- # with Minitest::Unit
6
- #watch(%r{^test/(.*)\/?test_(.*)\.rb$})
7
- #watch(%r{^lib/(.*/)?([^/]+)\.rb$}) { |m| "test/#{m[1]}test_#{m[2]}.rb" }
8
- #watch(%r{^test/test_helper\.rb$}) { 'test' }
9
-
10
- # with Minitest::Spec
11
- watch(%r{^spec/(.*)_spec\.rb$})
12
- #watch(%r{^lib/(.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" }
13
- watch(%r{^spec/spec_helper\.rb$}) { 'spec' }
14
-
15
- # Rails 4
16
- # watch(%r{^app/(.+)\.rb$}) { |m| "test/#{m[1]}_test.rb" }
17
- # watch(%r{^app/controllers/application_controller\.rb$}) { 'test/controllers' }
18
- # watch(%r{^app/controllers/(.+)_controller\.rb$}) { |m| "test/integration/#{m[1]}_test.rb" }
19
- # watch(%r{^app/views/(.+)_mailer/.+}) { |m| "test/mailers/#{m[1]}_mailer_test.rb" }
20
- # watch(%r{^lib/(.+)\.rb$}) { |m| "test/lib/#{m[1]}_test.rb" }
21
- # watch(%r{^test/.+_test\.rb$})
22
- # watch(%r{^test/test_helper\.rb$}) { 'test' }
23
-
24
- # Rails < 4
25
- # watch(%r{^app/controllers/(.*)\.rb$}) { |m| "test/functional/#{m[1]}_test.rb" }
26
- # watch(%r{^app/helpers/(.*)\.rb$}) { |m| "test/helpers/#{m[1]}_test.rb" }
27
- # watch(%r{^app/models/(.*)\.rb$}) { |m| "test/unit/#{m[1]}_test.rb" }
28
- end
@@ -1,60 +0,0 @@
1
- require 'logger'
2
- require 'fileutils'
3
-
4
- module EasySync
5
-
6
- class Rsync
7
- attr_reader :sync_name, :source, :destination, :exclude_file, :last_snapshot, :current, :logging
8
-
9
- def initialize(options)
10
- @sync_name = options[:sync_name]
11
- @source = options[:source]
12
- @destination = options[:destination]
13
- @exclude_file = options[:exclude_file]
14
- @logging = options.fetch(:logging){:off}
15
- end
16
-
17
- def latest_snapshot
18
- @last_snapshot = Dir["#{destination}/*"].max
19
- end
20
-
21
- def current_snapshot
22
- @current = File.join destination, Time.now.strftime("%Y-%m-%d")
23
- end
24
-
25
- def remove_old_backups
26
- backups = Dir["#{destination}/*"]
27
-
28
- (backups - backups.last(5)).each do |r|
29
- puts "Removing #{r}"
30
- FileUtils.remove_dir(r, true)
31
- end
32
-
33
- end
34
-
35
- def sync
36
- commands = [
37
- "rsync",
38
- "-avhiPH",
39
- "--exclude-from", "#{exclude_file}",
40
- "--link-dest", "#{latest_snapshot}"
41
- ]
42
-
43
- commands << ['--log-file', "#{ENV['HOME']}/easy_sync.log" ] if logging == :on
44
- commands << ["#{source}", "#{current_snapshot}"]
45
- commands = commands.flatten
46
-
47
- puts "\n\n\n------------------ Running #{sync_name} ------------------\n\n\n"
48
-
49
- remove_old_backups
50
-
51
- puts "latest snapshot #{@last_snapshot}"
52
-
53
- IO.popen(commands).each_line do |l|
54
- puts l
55
- end
56
- end
57
-
58
- end
59
-
60
- end
@@ -1,20 +0,0 @@
1
- require 'logger'
2
-
3
-
4
- module EasySync
5
-
6
- class SyncLogger
7
- attr_reader :log
8
-
9
- def initialize(path)
10
- @log = Logger.new(path)
11
- end
12
-
13
- def info(message)
14
- puts message
15
- log.info message
16
- end
17
-
18
- end
19
-
20
- end
@@ -1,44 +0,0 @@
1
- require 'yaml'
2
-
3
- module EasySync
4
-
5
- class SyncRunner
6
- attr_reader :config
7
-
8
- def initialize()
9
- handle_config
10
- end
11
-
12
- def run
13
- config[:tasks].each do |c|
14
- c[:logging] = config[:logging]
15
- Rsync.new(c).sync
16
- end
17
- end
18
-
19
- private
20
-
21
- def handle_config
22
- path = "#{ENV["HOME"]}/.easy_syncrc.yml"
23
- generate_yaml(path) unless File.exist? path
24
- @config = YAML.load_file path
25
- end
26
-
27
- def generate_yaml(path)
28
- File.open(path, 'w') do |f|
29
- h = {:logging => :on,
30
- :tasks => [{
31
- sync_name: "sample_sync",
32
- source: "[/example/path]",
33
- destination: "[/example/path]",
34
- exclude_file: "[/example/path]"
35
- }]}
36
- f.puts h.to_yaml
37
- end
38
-
39
- $stderr.puts "Generated sample config file: #{path}\n\n"
40
- end
41
-
42
- end
43
-
44
- end
@@ -1,25 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe Rsync do
4
-
5
- let(:task){config_file_data[:tasks].first}
6
-
7
- it "should find the latest snapshot directory" do
8
- Rsync.new(task) \
9
- .latest_snapshot.must_equal File.join(destination_directory, "2013-12-30")
10
- end
11
-
12
- it "should create the new snapshot directory" do
13
- Rsync.new(task) \
14
- .current_snapshot.must_equal File.join(destination_directory, Time.now.strftime("%Y-%m-%d"))
15
- end
16
-
17
- it "should log to a file" do
18
- log_file = "#{temp_directory}/easy_sync.log"
19
- task[:logging] = :on
20
-
21
- Rsync.new(task).sync
22
- File.exist?(log_file).must_equal true
23
- end
24
-
25
- end
@@ -1,54 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe SyncRunner do
4
-
5
- it "should create template config if no config file passed" do
6
- SyncRunner.new
7
- config = YAML.load_file default_config_path
8
- config.must_equal blank_config
9
- end
10
-
11
- it "should use default config file" do
12
- File.open(default_config_path, "w"){|f| f.puts config_file_data.merge({logging: :off}).to_yaml}
13
- SyncRunner.new.config[:logging].must_equal :off
14
- end
15
-
16
- it "should sync files between source and destination" do
17
- create_config_file
18
- SyncRunner.new.run
19
- Dir["#{File.join(destination_directory, Time.now.strftime("%Y-%m-%d"))}/*"].map{|f| File.basename f}.must_equal ["file1.txt", "file2.txt", "file3.txt", "file4.txt"]
20
- end
21
-
22
- it "should create hard links to the files that didn't change in the new snapshot directory" do
23
- create_config_file
24
- SyncRunner.new.run
25
-
26
- snapshot_file = Dir["#{File.join(destination_directory, "2013-12-30")}/*"].first
27
- new_file = Dir["#{File.join(destination_directory, Time.now.strftime("%Y-%m-%d"))}/*"].first
28
-
29
- File.stat(snapshot_file).ino.must_equal File.stat(new_file).ino
30
- end
31
-
32
- it "should use exclude-from option" do
33
- exclude_file = "#{destination_directory}/exclude.txt"
34
- File.open(exclude_file, "w"){|f| f.puts "file4.txt"}
35
-
36
- config_file_data[:tasks].first[:exclude_file] = exclude_file
37
- File.open(default_config_path, 'w'){|f| f.puts config_file_data.to_yaml}
38
-
39
- SyncRunner.new.run
40
-
41
- Dir["#{File.join(destination_directory, Time.now.strftime("%Y-%m-%d"))}/*"].map{|f| File.basename f}.must_equal ["file1.txt", "file2.txt", "file3.txt"]
42
- end
43
-
44
- it "should apply log setting to all task" do
45
- log_file = "#{temp_directory}/easy_sync.log"
46
- create_config_file
47
- s = SyncRunner.new
48
- s.config[:logging] = :on
49
- s.run
50
-
51
- File.exist?(log_file).must_equal true
52
- end
53
-
54
- end
data/spec/spec_helper.rb DELETED
@@ -1,33 +0,0 @@
1
- require "minitest/autorun"
2
- require "minitest/pride"
3
- require 'pry'
4
-
5
- require 'easy_sync'
6
- require_relative 'support'
7
-
8
- include EasySync
9
-
10
- class MiniTest::Spec
11
-
12
- before do
13
- create_root_directory
14
- create_source_directory
15
- create_destination_directory
16
- create_snapshot_directories
17
- create_source_files
18
- copy_to_latest
19
-
20
- # Silences output, comment out to debug test cases
21
- $stdout = StringIO.new
22
- $stderr = StringIO.new
23
- ENV['HOME'] = temp_directory
24
- end
25
-
26
-
27
- after do
28
- FileUtils.rm_r temp_directory
29
- $stdout = STDOUT
30
- $stderr = STDERR
31
- end
32
-
33
- end
data/spec/support.rb DELETED
@@ -1,89 +0,0 @@
1
- require 'fileutils'
2
- require 'tmpdir'
3
- require 'time'
4
-
5
-
6
- def create_root_directory
7
- @temp_directory = Dir.mktmpdir('test_data', File.expand_path('./', File.dirname(__FILE__)))
8
- end
9
-
10
- def temp_directory
11
- @temp_directory
12
- end
13
-
14
- def create_source_directory
15
- @source_directory = File.join(temp_directory, 'source')
16
- Dir.mkdir @source_directory
17
- end
18
-
19
- def create_destination_directory
20
- @destination_directory = File.join(temp_directory, 'backup')
21
- Dir.mkdir @destination_directory
22
- end
23
-
24
- def source_directory
25
- "#{@source_directory}/"
26
- end
27
-
28
- def destination_directory
29
- @destination_directory
30
- end
31
-
32
- def create_snapshot_directories
33
- %w{2013-12-30 2013-11-10 2013-10-10 2013-11-07}.each do |date|
34
- path = File.join destination_directory, date
35
-
36
- Dir.mkdir path
37
- time = Time.parse date
38
-
39
- File.utime(time, time, path)
40
-
41
- end
42
- end
43
-
44
- def create_source_files
45
- %w{file1.txt file2.txt file3.txt file4.txt}.each do |file|
46
- File.open(File.join(source_directory, file), 'w'){|f| f.puts "This is some sample text"}
47
- end
48
-
49
- end
50
-
51
- def copy_to_latest
52
- %w{file1.txt file2.txt file3.txt}.each do |file|
53
- FileUtils.cp File.join(source_directory, file), File.join(destination_directory, "2013-12-30")
54
- end
55
- end
56
-
57
- def blank_config
58
- @blank_config ||= {
59
- logging: :on,
60
- tasks: [{
61
- sync_name: "sample_sync",
62
- source: "[/example/path]",
63
- destination: "[/example/path]",
64
- exclude_file: "[/example/path]"
65
- }]
66
- }
67
- end
68
-
69
- def config_file_data
70
- @config_file_data ||= {
71
- logging: :off,
72
- tasks: [
73
- {
74
- sync_name: "test_sync",
75
- source: "#{source_directory}",
76
- destination: "#{destination_directory}",
77
- exclude_file: ""
78
- }]
79
- }
80
- end
81
-
82
- def create_config_file
83
- File.open(default_config_path, 'w'){|f| f.puts config_file_data.to_yaml}
84
- default_config_path
85
- end
86
-
87
- def default_config_path
88
- "#{temp_directory}/.easy_syncrc.yml"
89
- end