easy_sync 0.0.5 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +8 -0
- data/.rspec +2 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +85 -0
- data/CLAUDE.md +133 -0
- data/Gemfile +2 -0
- data/README.md +607 -21
- data/Rakefile +6 -8
- data/bin/easy_sync +8 -1
- data/docs/fine-placement.md +174 -0
- data/docs/hardware-notes.md +71 -0
- data/docs/integrity-scan.md +435 -0
- data/docs/parallel-scrub.md +280 -0
- data/docs/performance.md +146 -0
- data/docs/tripwire.md +232 -0
- data/easy_sync.gemspec +25 -16
- data/lib/easy_sync/cli.rb +1246 -0
- data/lib/easy_sync/config.rb +202 -0
- data/lib/easy_sync/error.rb +5 -0
- data/lib/easy_sync/jbod/backblaze.rb +113 -0
- data/lib/easy_sync/jbod/benchmarker.rb +149 -0
- data/lib/easy_sync/jbod/cleaner.rb +93 -0
- data/lib/easy_sync/jbod/dashboard.rb +611 -0
- data/lib/easy_sync/jbod/keep_awake.rb +33 -0
- data/lib/easy_sync/jbod/manifest.rb +1068 -0
- data/lib/easy_sync/jbod/mirror.rb +204 -0
- data/lib/easy_sync/jbod/models.rb +165 -0
- data/lib/easy_sync/jbod/page_cache.rb +56 -0
- data/lib/easy_sync/jbod/placement.rb +71 -0
- data/lib/easy_sync/jbod/planner.rb +76 -0
- data/lib/easy_sync/jbod/purger.rb +161 -0
- data/lib/easy_sync/jbod/restorer.rb +142 -0
- data/lib/easy_sync/jbod/run_lock.rb +98 -0
- data/lib/easy_sync/jbod/run_log.rb +57 -0
- data/lib/easy_sync/jbod/runner.rb +615 -0
- data/lib/easy_sync/jbod/scrub_pool.rb +136 -0
- data/lib/easy_sync/jbod/scrubber.rb +321 -0
- data/lib/easy_sync/jbod/share_scan.rb +26 -0
- data/lib/easy_sync/jbod/sync_eta.rb +68 -0
- data/lib/easy_sync/jbod/tripwire.rb +80 -0
- data/lib/easy_sync/jbod/volume_info.rb +283 -0
- data/lib/easy_sync/jbod.rb +35 -0
- data/lib/easy_sync/shell.rb +62 -0
- data/lib/easy_sync/templates/dashboard.html.erb +419 -0
- data/lib/easy_sync/version.rb +3 -1
- data/lib/easy_sync.rb +7 -3
- metadata +84 -45
- data/Gemfile.lock +0 -55
- data/Guardfile +0 -28
- data/lib/easy_sync/rsync.rb +0 -60
- data/lib/easy_sync/sync_logger.rb +0 -20
- data/lib/easy_sync/sync_runner.rb +0 -44
- data/spec/easy_sync/rsync_spec.rb +0 -25
- data/spec/easy_sync/sync_runner_spec.rb +0 -54
- data/spec/spec_helper.rb +0 -33
- data/spec/support.rb +0 -89
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'fileutils'
|
|
4
|
+
require 'tempfile'
|
|
5
|
+
|
|
6
|
+
module EasySync
|
|
7
|
+
module Jbod
|
|
8
|
+
# Mirrors one source folder onto one destination folder with rsync.
|
|
9
|
+
#
|
|
10
|
+
# Nothing is ever deleted by rsync itself. Each folder gets two passes: a
|
|
11
|
+
# read-only check (`rsync -n --delete --itemize-changes --stats`, see
|
|
12
|
+
# #check) run before any copying starts, then a copy pass with no deletion
|
|
13
|
+
# flags at all. The check's "*deleting path" lines say which files on the
|
|
14
|
+
# drive no longer exist on the source (they feed the grace-period purge in
|
|
15
|
+
# Runner), and its ">f" lines say which existing files the copy would
|
|
16
|
+
# overwrite (the tripwire, docs/tripwire.md). (`--delete --max-delete=0`
|
|
17
|
+
# looked like a one-pass alternative, but rsync then only prints
|
|
18
|
+
# "N skipped" without naming the files, and exits 25.)
|
|
19
|
+
class Mirror
|
|
20
|
+
MIN_VERSION = [3, 0, 0].freeze
|
|
21
|
+
# Anchored to the transfer root, directories only: a share's root-files
|
|
22
|
+
# unit copies its loose top-level files and nothing below them.
|
|
23
|
+
ROOT_ONLY = '--exclude=/*/'
|
|
24
|
+
|
|
25
|
+
Result = Struct.new(:exit_status, :bytes_transferred, :total_size_bytes, :disk_full, :output,
|
|
26
|
+
keyword_init: true) do
|
|
27
|
+
def success? = exit_status.zero?
|
|
28
|
+
def disk_full? = !!disk_full
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# What the read-only check found for one folder. +replaced+ are existing
|
|
32
|
+
# files on the drive the copy would overwrite; +missing+ is
|
|
33
|
+
# [[path, kind], ...] on the drive but gone from the source (for the
|
|
34
|
+
# purge); +junk+ are the missing paths that only match exclude_folders
|
|
35
|
+
# (already-copied junk, not something lost on the NAS); +new_files+ is
|
|
36
|
+
# how many files the copy would add; +source_files+ is the source's
|
|
37
|
+
# regular-file count from --stats. +known+ are paths an earlier run
|
|
38
|
+
# already found missing (Runner fills it in from pending_deletions):
|
|
39
|
+
# they wait out the grace period on the drive and are not news, so an
|
|
40
|
+
# accepted bulk delete doesn't trip again on every run until it's purged.
|
|
41
|
+
Check = Struct.new(:replaced, :missing, :junk, :new_files, :source_files, :known, keyword_init: true) do
|
|
42
|
+
def missing_files
|
|
43
|
+
skip = Set.new(junk) + Array(known)
|
|
44
|
+
missing.select { |path, kind| kind == 'file' && !skip.include?(path) }.map(&:first)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def changed = replaced.size + missing_files.size
|
|
48
|
+
|
|
49
|
+
# Files on the drive, worked out without walking it: what the source
|
|
50
|
+
# has, minus what the copy would add, plus what only the drive has.
|
|
51
|
+
def files_on_drive = [source_files.to_i - new_files + missing.count { |_, kind| kind == 'file' }, 0].max
|
|
52
|
+
def samples(limit = 10) = (replaced + missing_files).first(limit)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# +excludes+ are rsync patterns (no slash, so they match at any depth):
|
|
56
|
+
# the config's exclude_folders. They apply to both passes.
|
|
57
|
+
def initialize(shell: Shell.new, extra_args: [], excludes: [])
|
|
58
|
+
@shell = shell
|
|
59
|
+
@extra_args = Array(extra_args)
|
|
60
|
+
@patterns = Array(excludes)
|
|
61
|
+
@excludes = @patterns.map { |e| "--exclude=#{e}" }
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# The copy pass. Extra args (from config, or --dry-run) apply here.
|
|
65
|
+
# --partial keeps a killed transfer's in-progress file instead of deleting
|
|
66
|
+
# it, so a multi-GB file interrupted mid-copy resumes next run instead of
|
|
67
|
+
# restarting from zero.
|
|
68
|
+
def command(source, destination, root_only: false)
|
|
69
|
+
argv = ['rsync', '-a', '--partial', '--stats', '--info=progress2', '--itemize-changes', *@excludes]
|
|
70
|
+
argv << ROOT_ONLY if root_only
|
|
71
|
+
argv += @extra_args
|
|
72
|
+
argv + [with_slash(source), with_slash(destination)]
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# The check probe: never copies, never deletes, only reports.
|
|
76
|
+
# --stats adds the source's file count, for the tripwire's ratio.
|
|
77
|
+
# --delete-excluded makes it also report excluded junk that an earlier
|
|
78
|
+
# run copied before the exclusion existed, so the purge clears it.
|
|
79
|
+
# A root-files unit (root_only) drops --delete-excluded: its subfolders
|
|
80
|
+
# are excluded, and they belong to other folders, so they must never be
|
|
81
|
+
# reported as extraneous. Without --delete-excluded rsync protects them.
|
|
82
|
+
def probe_command(source, destination, root_only: false)
|
|
83
|
+
return ['rsync', '-an', '--itemize-changes', '--stats', '--delete', *@excludes, ROOT_ONLY,
|
|
84
|
+
with_slash(source), with_slash(destination)] if root_only
|
|
85
|
+
|
|
86
|
+
['rsync', '-an', '--itemize-changes', '--stats', '--delete', '--delete-excluded', *@excludes,
|
|
87
|
+
with_slash(source), with_slash(destination)]
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Raises unless the rsync on PATH is new enough for --itemize-changes deletion reporting.
|
|
91
|
+
def self.check_version!(shell)
|
|
92
|
+
result = shell.capture(['rsync', '--version'])
|
|
93
|
+
version = result.output[/rsync\s+version\s+(\d+)\.(\d+)\.(\d+)/, 0]
|
|
94
|
+
raise Error, 'rsync not found on PATH' unless result.success? && version
|
|
95
|
+
|
|
96
|
+
numbers = version.scan(/\d+/).map(&:to_i)
|
|
97
|
+
return numbers.join('.') if (numbers <=> MIN_VERSION) >= 0
|
|
98
|
+
|
|
99
|
+
raise Error, "rsync #{numbers.join('.')} is too old; 3.0 or newer is required (brew install rsync)"
|
|
100
|
+
rescue Errno::ENOENT
|
|
101
|
+
raise Error, 'rsync not found on PATH'
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def sync(source, destination, root_only: false)
|
|
105
|
+
raise Error, "source folder #{source} does not exist" unless Dir.exist?(source)
|
|
106
|
+
|
|
107
|
+
FileUtils.mkdir_p(File.dirname(destination))
|
|
108
|
+
result = @shell.run(command(source, destination, root_only: root_only))
|
|
109
|
+
stats = self.class.parse_stats(result.output)
|
|
110
|
+
Result.new(exit_status: result.status, output: result.output,
|
|
111
|
+
disk_full: self.class.disk_full?(result.output), **stats)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Runs the check probe. Returns a Check, or nil if the probe failed (the
|
|
115
|
+
# caller then neither copies the folder nor records anything for it).
|
|
116
|
+
def check(source, destination, root_only: false)
|
|
117
|
+
result = @shell.capture(probe_command(source, destination, root_only: root_only))
|
|
118
|
+
return nil unless result.success?
|
|
119
|
+
|
|
120
|
+
missing = self.class.parse_extraneous(result.output)
|
|
121
|
+
changes = self.class.parse_file_changes(result.output)
|
|
122
|
+
Check.new(replaced: changes[:replaced], missing: missing, new_files: changes[:new],
|
|
123
|
+
junk: missing.map(&:first).select { |path| junk?(path) },
|
|
124
|
+
source_files: self.class.regular_files(result.output))
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Re-copies files `scrub` flagged as corrupt/unreadable, overwriting the
|
|
128
|
+
# bad copy on the drive. `-I` ignores rsync's quick size/mtime check
|
|
129
|
+
# (which would otherwise skip a file whose size and mtime never
|
|
130
|
+
# changed - the whole reason rot goes unnoticed). No --partial: an
|
|
131
|
+
# interrupted refetch must leave the old, still-flagged file in place,
|
|
132
|
+
# not a half-written one - rsync only renames its temp file over the
|
|
133
|
+
# target once that file's transfer completes.
|
|
134
|
+
def refetch(source, destination, relative_paths)
|
|
135
|
+
list = Tempfile.new('easy_sync-refetch')
|
|
136
|
+
begin
|
|
137
|
+
list.write(relative_paths.join("\0"))
|
|
138
|
+
list.close
|
|
139
|
+
@shell.run(['rsync', '-a', '-I', '--stats', '--from0', "--files-from=#{list.path}",
|
|
140
|
+
with_slash(source), with_slash(destination)])
|
|
141
|
+
ensure
|
|
142
|
+
list.unlink
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# True when rsync's own output says the destination ran out of space.
|
|
147
|
+
# Deliberately a text match rather than a specific exit status: rsync
|
|
148
|
+
# reports this the same way (exit 11) as other unrelated I/O errors.
|
|
149
|
+
def self.disk_full?(output)
|
|
150
|
+
output.match?(/No space left on device/i)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Pulls the two numbers we keep out of `rsync --stats` output.
|
|
154
|
+
def self.parse_stats(output)
|
|
155
|
+
{
|
|
156
|
+
total_size_bytes: stat(output, 'Total file size'),
|
|
157
|
+
bytes_transferred: stat(output, 'Total transferred file size')
|
|
158
|
+
}
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# "*deleting some/dir/" and "*deleting some/file" lines -> [[path, kind], ...]
|
|
162
|
+
def self.parse_extraneous(output)
|
|
163
|
+
output.scan(/^\*deleting\s+(.+?)\r?$/).map do |(path)|
|
|
164
|
+
path.end_with?('/') ? [path.chomp('/'), 'dir'] : [path, 'file']
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# ">f+++++++++ path" is a new file; any other ">f" line is an existing
|
|
169
|
+
# file the copy would overwrite. Directories, symlinks and
|
|
170
|
+
# attribute-only (".f") lines change no file's content.
|
|
171
|
+
def self.parse_file_changes(output)
|
|
172
|
+
changes = { replaced: [], new: 0 }
|
|
173
|
+
output.scan(/^>f(\S+) (.+?)\r?$/) do |flags, path|
|
|
174
|
+
flags.start_with?('+') ? changes[:new] += 1 : changes[:replaced] << path
|
|
175
|
+
end
|
|
176
|
+
changes
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# "Number of files: 1,234 (reg: 1,000, dir: 234)" -> 1000. A source
|
|
180
|
+
# with no regular files leaves "reg:" out entirely.
|
|
181
|
+
def self.regular_files(output)
|
|
182
|
+
line = output[/^Number of files:.*$/] or return 0
|
|
183
|
+
line[/reg:\s*([\d,]+)/, 1].to_s.delete(',').to_i
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def self.stat(output, label)
|
|
187
|
+
match = output[/^#{Regexp.escape(label)}:\s*([\d,.]+)/, 1]
|
|
188
|
+
match && match.delete(',').to_i
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
private
|
|
192
|
+
|
|
193
|
+
# A path the probe reports only because --delete-excluded finds
|
|
194
|
+
# already-copied junk: one of its components matches exclude_folders.
|
|
195
|
+
def junk?(path)
|
|
196
|
+
path.split('/').any? { |part| @patterns.any? { |pat| File.fnmatch?(pat, part) } }
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def with_slash(path)
|
|
200
|
+
path.end_with?('/') ? path : "#{path}/"
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
end
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'time'
|
|
4
|
+
|
|
5
|
+
module EasySync
|
|
6
|
+
module Jbod
|
|
7
|
+
Drive = Struct.new(:serial_number, :friendly_name, :capacity_bytes, :added_date, :volume_uuid, :model,
|
|
8
|
+
:last_seen_at, :last_used_bytes, :last_free_bytes,
|
|
9
|
+
:smart_status, :smart_detail, :smart_checked_at, :power_on_hours, :retired_at,
|
|
10
|
+
keyword_init: true) do
|
|
11
|
+
def retired? = !retired_at.nil?
|
|
12
|
+
|
|
13
|
+
def used_fraction
|
|
14
|
+
return nil if last_used_bytes.nil? || capacity_bytes.to_i.zero?
|
|
15
|
+
|
|
16
|
+
last_used_bytes.to_f / capacity_bytes
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# smartctl's Device Model has no separate "vendor" field for ATA/NVMe
|
|
20
|
+
# drives (that's a SCSI/USB-bridge thing); the maker is only ever
|
|
21
|
+
# visible as a token baked into the model string itself, or - for
|
|
22
|
+
# Seagate - not spelled out at all, just implied by the "ST" model
|
|
23
|
+
# prefix every Seagate drive uses.
|
|
24
|
+
BRANDS = {
|
|
25
|
+
'WDC' => 'Western Digital', 'WD' => 'Western Digital', 'TOSHIBA' => 'Toshiba', 'HGST' => 'HGST',
|
|
26
|
+
'SAMSUNG' => 'Samsung', 'HITACHI' => 'Hitachi', 'APPLE' => 'Apple', 'SEAGATE' => 'Seagate',
|
|
27
|
+
'CRUCIAL' => 'Crucial', 'KINGSTON' => 'Kingston', 'INTEL' => 'Intel'
|
|
28
|
+
}.freeze
|
|
29
|
+
|
|
30
|
+
def manufacturer
|
|
31
|
+
return nil unless model
|
|
32
|
+
|
|
33
|
+
token = model[/\A[A-Za-z]+/]
|
|
34
|
+
return BRANDS[token.upcase] if token && BRANDS.key?(token.upcase)
|
|
35
|
+
|
|
36
|
+
'Seagate' if model.match?(/\AST\d/i)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# model, with the manufacturer spelled out plainly instead of a raw
|
|
40
|
+
# token (or, for Seagate, added - its model numbers don't carry one).
|
|
41
|
+
def branded_model
|
|
42
|
+
return nil unless model
|
|
43
|
+
return model unless manufacturer
|
|
44
|
+
|
|
45
|
+
rest = model.sub(/\A(?:WDC|WD|TOSHIBA|HGST|SAMSUNG|HITACHI|APPLE|SEAGATE|CRUCIAL|KINGSTON|INTEL)[\s-]+/i, '')
|
|
46
|
+
"#{manufacturer} #{rest}"
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# SMART's Power_On_Hours counts only time actually spinning/powered,
|
|
50
|
+
# unlike calendar age: a 5-year-old drive that sat on a shelf can show
|
|
51
|
+
# a fraction of the wear of one bought last year and run constantly.
|
|
52
|
+
def power_on_label
|
|
53
|
+
return nil unless power_on_hours
|
|
54
|
+
|
|
55
|
+
years = power_on_hours / 24.0 / 365
|
|
56
|
+
return format('%d days (%d hrs)', (power_on_hours / 24.0).round, power_on_hours) if years < 1
|
|
57
|
+
|
|
58
|
+
format('%.1f yrs (%d hrs)', years, power_on_hours)
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# scope 'tree' is a folder copied with everything under it. scope 'root'
|
|
63
|
+
# is a share's loose top-level files only (folder_path is the share name);
|
|
64
|
+
# the share's subfolders are their own 'tree' folders, possibly elsewhere.
|
|
65
|
+
Folder = Struct.new(:folder_path, :drive_serial, :size_bytes, :assigned_at,
|
|
66
|
+
:last_synced_at, :last_sync_status, :scope, keyword_init: true) do
|
|
67
|
+
def root? = scope == 'root'
|
|
68
|
+
def share = folder_path.split('/').first
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# One folder held back (or let through with --accept-changes) by the
|
|
72
|
+
# tripwire in one sync run. +samples+ is a few of its changed paths.
|
|
73
|
+
TripRecord = Struct.new(:id, :run_started_at, :folder_path, :scope, :replaced, :missing, :files_on_drive,
|
|
74
|
+
:samples, :accepted_at, keyword_init: true) do
|
|
75
|
+
def changed = replaced.to_i + missing.to_i
|
|
76
|
+
def accepted? = !accepted_at.nil?
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
HistoryEntry = Struct.new(:id, :folder_path, :drive_serial, :event, :recorded_at, :note,
|
|
80
|
+
keyword_init: true)
|
|
81
|
+
|
|
82
|
+
SyncRun = Struct.new(:id, :folder_path, :drive_serial, :started_at, :finished_at, :exit_status,
|
|
83
|
+
:bytes_transferred, :total_size_bytes, keyword_init: true)
|
|
84
|
+
|
|
85
|
+
# One whole `sync` run, summed over its folders (Manifest#run_summaries).
|
|
86
|
+
# copied counts folders that transferred any bytes; the rest only
|
|
87
|
+
# confirmed nothing had changed.
|
|
88
|
+
RunSummary = Struct.new(:run_started_at, :last_finished_at, :folders, :copied, :bytes_transferred, :failed,
|
|
89
|
+
keyword_init: true)
|
|
90
|
+
|
|
91
|
+
# A path on a drive that is a candidate for deletion, either because
|
|
92
|
+
# rsync reported it gone from the NAS (cause 'missing_on_nas';
|
|
93
|
+
# relative_path is '' with kind 'folder' when the whole folder is gone)
|
|
94
|
+
# or because the folder was reassigned off +drive_serial+ (cause
|
|
95
|
+
# 'reassigned', always whole-folder; relative_path there holds the old
|
|
96
|
+
# drive's serial instead of a real path, only so a second reassignment of
|
|
97
|
+
# the same folder before the first cleanup runs gets its own row rather
|
|
98
|
+
# than colliding on the folder_path+relative_path uniqueness).
|
|
99
|
+
PendingDeletion = Struct.new(:id, :folder_path, :relative_path, :kind, :drive_serial, :cause, :first_missing_at,
|
|
100
|
+
:last_missing_at, :missing_runs, keyword_init: true) do
|
|
101
|
+
def whole_folder? = kind == 'folder'
|
|
102
|
+
def reassigned? = cause == 'reassigned'
|
|
103
|
+
|
|
104
|
+
def expires_at(grace_days)
|
|
105
|
+
Time.parse(first_missing_at) + (grace_days * 86_400)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def expired?(now:, grace_days:, grace_runs:)
|
|
109
|
+
missing_runs >= grace_runs && expires_at(grace_days) <= now
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
Deletion = Struct.new(:id, :folder_path, :relative_path, :kind, :drive_serial, :first_missing_at, :deleted_at,
|
|
114
|
+
keyword_init: true)
|
|
115
|
+
|
|
116
|
+
# One tracked file on one drive, for `scrub` (see docs/integrity-scan.md).
|
|
117
|
+
# digest is the SHA-256 baseline (nil until the first hash); status is
|
|
118
|
+
# 'ok', 'corrupt' (hash mismatch), 'unreadable' (a read error), or
|
|
119
|
+
# 'unresolved' (still bad after sync refetched it - never refetched
|
|
120
|
+
# again automatically).
|
|
121
|
+
FileChecksum = Struct.new(:drive_serial, :folder_path, :relative_path, :size_bytes, :mtime, :digest,
|
|
122
|
+
:verified_at, :status, :failed_at, :refetched_at, keyword_init: true) do
|
|
123
|
+
def ok? = status == 'ok'
|
|
124
|
+
def flagged? = %w[corrupt unreadable].include?(status)
|
|
125
|
+
def unresolved? = status == 'unresolved'
|
|
126
|
+
def refetched? = !refetched_at.nil?
|
|
127
|
+
def label = "#{folder_path}/#{relative_path}"
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# One `benchmark` run on one drive (see Jbod::Benchmarker). MB/s is
|
|
131
|
+
# MiB/s, the same unit scrub reports. used_bytes is how full the drive was
|
|
132
|
+
# at the time: a fuller drive writes to slower inner tracks, so a gradual
|
|
133
|
+
# decline that tracks it is expected, not a warning sign.
|
|
134
|
+
DriveBenchmark = Struct.new(:id, :drive_serial, :run_at, :bytes, :write_mb_s, :read_mb_s, :used_bytes,
|
|
135
|
+
keyword_init: true)
|
|
136
|
+
|
|
137
|
+
# One row per folder seen on the NAS at the last completed placement
|
|
138
|
+
# pass: placed (assigned to a drive, synced or queued), unplaced (no drive
|
|
139
|
+
# has room, or nothing is mounted) or empty (no real files).
|
|
140
|
+
SourceEntry = Struct.new(:folder_path, :size_bytes, :state, :detail, :seen_at, keyword_init: true) do
|
|
141
|
+
def share = folder_path.split('/').first
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# SMART health as last read from the drive. status is one of
|
|
145
|
+
# 'ok' (self-assessment passed, no bad-sector counters), 'warning' (passed
|
|
146
|
+
# but reallocated/pending/uncorrectable sectors or an NVMe critical flag:
|
|
147
|
+
# the drive is starting to fail), 'failing' (self-assessment FAILED), or
|
|
148
|
+
# 'unknown' (SMART not exposed by the enclosure, smartctl missing, etc.).
|
|
149
|
+
# reallocated_sector_ct is the raw counter, tracked separately over time
|
|
150
|
+
# (see Manifest#record_smart_check) so a drive whose count is old and
|
|
151
|
+
# unchanging can be told apart from one that's actively climbing.
|
|
152
|
+
# other_bad is true when pending/uncorrectable sectors, media errors, or
|
|
153
|
+
# an NVMe critical-warning flag also contributed to a 'warning' status -
|
|
154
|
+
# those are never downgraded by reallocated-count history.
|
|
155
|
+
Health = Struct.new(:status, :detail, :source, :power_on_hours, :reallocated_sector_ct, :other_bad,
|
|
156
|
+
keyword_init: true)
|
|
157
|
+
|
|
158
|
+
# A registered drive that is currently mounted, with live usage numbers.
|
|
159
|
+
MountedDrive = Struct.new(:drive, :mount_point, :capacity_bytes, :used_bytes, :free_bytes,
|
|
160
|
+
keyword_init: true) do
|
|
161
|
+
def serial_number = drive.serial_number
|
|
162
|
+
def friendly_name = drive.friendly_name
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'fiddle'
|
|
4
|
+
|
|
5
|
+
module EasySync
|
|
6
|
+
module Jbod
|
|
7
|
+
# Drops one file's pages from the macOS unified buffer cache, so the read
|
|
8
|
+
# that follows comes off the drive. F_NOCACHE alone is not enough: it
|
|
9
|
+
# stops a read from *adding* pages to the cache, but pages already there
|
|
10
|
+
# (a file sync just wrote, or anything read recently) are still served
|
|
11
|
+
# from RAM. Measured on jbod-test-1 (USB): a just-synced 100 MB file read
|
|
12
|
+
# at 12,037 MB/s with F_NOCACHE, and at 156 MB/s after this. msync with
|
|
13
|
+
# MS_INVALIDATE is the BSD/macOS stand-in for Linux's
|
|
14
|
+
# posix_fadvise(DONTNEED) (vmtouch uses it the same way); MS_SYNC first
|
|
15
|
+
# flushes any pending writes, so nothing unwritten is ever discarded.
|
|
16
|
+
# Best effort: any failure just leaves the cache as it was.
|
|
17
|
+
module PageCache
|
|
18
|
+
PROT_READ = 0x01
|
|
19
|
+
MAP_SHARED = 0x01
|
|
20
|
+
MS_SYNC = 0x10
|
|
21
|
+
MS_INVALIDATE = 0x02
|
|
22
|
+
|
|
23
|
+
module_function
|
|
24
|
+
|
|
25
|
+
def evict(io)
|
|
26
|
+
len = io.size
|
|
27
|
+
return false if len.zero?
|
|
28
|
+
|
|
29
|
+
addr = functions[:mmap].call(nil, len, PROT_READ, MAP_SHARED, io.fileno, 0)
|
|
30
|
+
return false if [-1, (2**64) - 1].include?(addr.to_i)
|
|
31
|
+
|
|
32
|
+
begin
|
|
33
|
+
functions[:msync].call(addr, len, MS_SYNC | MS_INVALIDATE).zero?
|
|
34
|
+
ensure
|
|
35
|
+
functions[:munmap].call(addr, len)
|
|
36
|
+
end
|
|
37
|
+
rescue StandardError, Fiddle::DLError
|
|
38
|
+
false
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def functions
|
|
42
|
+
@functions ||= begin
|
|
43
|
+
libc = Fiddle.dlopen(nil)
|
|
44
|
+
{
|
|
45
|
+
mmap: Fiddle::Function.new(libc['mmap'], [Fiddle::TYPE_VOIDP, Fiddle::TYPE_SIZE_T, Fiddle::TYPE_INT,
|
|
46
|
+
Fiddle::TYPE_INT, Fiddle::TYPE_INT, Fiddle::TYPE_LONG_LONG],
|
|
47
|
+
Fiddle::TYPE_VOIDP),
|
|
48
|
+
msync: Fiddle::Function.new(libc['msync'], [Fiddle::TYPE_VOIDP, Fiddle::TYPE_SIZE_T, Fiddle::TYPE_INT],
|
|
49
|
+
Fiddle::TYPE_INT),
|
|
50
|
+
munmap: Fiddle::Function.new(libc['munmap'], [Fiddle::TYPE_VOIDP, Fiddle::TYPE_SIZE_T], Fiddle::TYPE_INT)
|
|
51
|
+
}
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module EasySync
|
|
4
|
+
module Jbod
|
|
5
|
+
# Decides where a brand-new folder goes. Pure logic, no I/O.
|
|
6
|
+
#
|
|
7
|
+
# Rule: a drive in +prefer+ (one already holding part of the same share)
|
|
8
|
+
# wins if the folder fits there, so a share stays on one drive while it
|
|
9
|
+
# can; otherwise the mounted drive with the most free space wins. A folder
|
|
10
|
+
# is never placed on a drive it would not fit on, and existing folders are
|
|
11
|
+
# never moved (that is what makes the JBOD layout browsable by hand).
|
|
12
|
+
module Placement
|
|
13
|
+
class NoMountedDrives < Error; end
|
|
14
|
+
class DoesNotFit < Error; end
|
|
15
|
+
|
|
16
|
+
# +candidates+ are MountedDrive structs. +size_bytes+ is the folder size
|
|
17
|
+
# (nil when unknown, in which case only the free-space ordering applies).
|
|
18
|
+
# +reserve_bytes+ is headroom to leave on the drive after placement.
|
|
19
|
+
def self.choose(candidates, size_bytes: nil, reserve_bytes: 0, prefer: [])
|
|
20
|
+
raise NoMountedDrives, 'no registered drives are mounted' if candidates.empty?
|
|
21
|
+
|
|
22
|
+
preferred = candidates.select do |c|
|
|
23
|
+
prefer.include?(c.serial_number) && (size_bytes.nil? || c.free_bytes - reserve_bytes >= size_bytes)
|
|
24
|
+
end
|
|
25
|
+
pool = preferred.empty? ? candidates : preferred
|
|
26
|
+
best = pool.max_by { |c| [c.free_bytes, c.friendly_name.to_s] }
|
|
27
|
+
if size_bytes && (best.free_bytes - reserve_bytes) < size_bytes
|
|
28
|
+
raise DoesNotFit,
|
|
29
|
+
"#{format_bytes(size_bytes)} does not fit on #{best.friendly_name} " \
|
|
30
|
+
"(#{format_bytes(best.free_bytes)} free, #{format_bytes(reserve_bytes)} reserved)"
|
|
31
|
+
end
|
|
32
|
+
best
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# "2gb", "500 MB", 2147483648 -> bytes. Used for config values and CLI flags.
|
|
36
|
+
def self.parse_size(value)
|
|
37
|
+
return value.to_i if value.is_a?(Numeric)
|
|
38
|
+
|
|
39
|
+
m = value.to_s.strip.match(/\A([\d.]+)\s*(tb|gb|mb|kb|b)?\z/i) or raise Error, "cannot parse size #{value.inspect} (try 2gb)"
|
|
40
|
+
(m[1].to_f * { nil => 1, 'b' => 1, 'kb' => 1024, 'mb' => 1024**2, 'gb' => 1024**3, 'tb' => 1024**4 }[m[2]&.downcase]).to_i
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
UNITS = %w[B KB MB GB TB PB].freeze
|
|
44
|
+
|
|
45
|
+
def self.format_bytes(bytes)
|
|
46
|
+
return '—' if bytes.nil?
|
|
47
|
+
|
|
48
|
+
value = bytes.to_f
|
|
49
|
+
unit = 0
|
|
50
|
+
while value >= 1024 && unit < UNITS.size - 1
|
|
51
|
+
value /= 1024
|
|
52
|
+
unit += 1
|
|
53
|
+
end
|
|
54
|
+
unit.zero? ? "#{bytes} B" : format('%.1f %s', value, UNITS[unit])
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# "3d 4h", "2h 34m", "45m", or "12s". Shared by `status` and the
|
|
58
|
+
# dashboard so an elapsed/remaining time reads the same everywhere.
|
|
59
|
+
def self.format_duration(seconds)
|
|
60
|
+
days, rem = seconds.to_i.divmod(86_400)
|
|
61
|
+
hours, rem = rem.divmod(3600)
|
|
62
|
+
minutes, secs = rem.divmod(60)
|
|
63
|
+
return "#{days}d #{hours}h" if days.positive?
|
|
64
|
+
return "#{hours}h #{minutes}m" if hours.positive?
|
|
65
|
+
return "#{minutes}m #{secs}s" if minutes.positive?
|
|
66
|
+
|
|
67
|
+
"#{secs}s"
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module EasySync
|
|
4
|
+
module Jbod
|
|
5
|
+
# `easy_sync plan`: measures each configured share and checks that every
|
|
6
|
+
# top-level folder (each is placed on its own) fits on the largest drive
|
|
7
|
+
# in the fleet. Reads only; never places or copies anything.
|
|
8
|
+
class Planner
|
|
9
|
+
Row = Struct.new(:source, :mounted, :size_bytes, :subfolders, :largest_subfolder, :largest_name, :loose_files,
|
|
10
|
+
:reason, :fits, keyword_init: true)
|
|
11
|
+
|
|
12
|
+
def initialize(settings, shell: Shell.new, largest_drive_bytes: nil)
|
|
13
|
+
@settings = settings
|
|
14
|
+
@shell = shell
|
|
15
|
+
@largest = largest_drive_bytes
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# +only+ restricts to sources whose basename or full path is in the list
|
|
19
|
+
# (e.g. easy_sync plan pro, matching /Volumes/pro). nil measures all.
|
|
20
|
+
def rows(only: nil)
|
|
21
|
+
sources = Array(@settings[:sources]).map { |e| Runner::Source.from_config(e) }
|
|
22
|
+
sources = sources.select { |s| only.include?(s.name) || only.include?(s.path) } if only
|
|
23
|
+
sources.map { |src| measure(src) }
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def measure(source)
|
|
29
|
+
row = Row.new(source: source, mounted: Dir.exist?(source.path) && !Dir.empty?(source.path), loose_files: 0,
|
|
30
|
+
subfolders: 0)
|
|
31
|
+
return row.tap { |r| r.reason = 'not mounted (or empty)' } unless row.mounted
|
|
32
|
+
|
|
33
|
+
excluded = Array(@settings[:exclude_folders])
|
|
34
|
+
children = Dir.children(source.path).sort.reject { |n| excluded.include?(n) || n.start_with?('.') }
|
|
35
|
+
dirs = children.select { |n| File.directory?(File.join(source.path, n)) }
|
|
36
|
+
row.loose_files = children.count { |n| File.file?(File.join(source.path, n)) }
|
|
37
|
+
row.subfolders = dirs.size
|
|
38
|
+
|
|
39
|
+
sizes = du(dirs.map { |n| File.join(source.path, n) })
|
|
40
|
+
row.size_bytes = sizes.values.sum
|
|
41
|
+
row.largest_name, row.largest_subfolder = sizes.max_by { |_, v| v } || [nil, 0]
|
|
42
|
+
check_fit(row)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# One `du -sk` over every subfolder; the tool's own `du` cost was
|
|
46
|
+
# measured at ~6 s for 2,380 movie folders over SMB.
|
|
47
|
+
def du(paths)
|
|
48
|
+
return {} if paths.empty?
|
|
49
|
+
|
|
50
|
+
result = @shell.capture(['du', '-sk', *paths])
|
|
51
|
+
result.output.each_line.with_object({}) do |line, h|
|
|
52
|
+
kb, path = line.chomp.split("\t", 2)
|
|
53
|
+
next unless kb && path
|
|
54
|
+
|
|
55
|
+
h[File.basename(path.chomp('/'))] = kb.to_i * 1024
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def check_fit(row)
|
|
60
|
+
largest = @largest
|
|
61
|
+
if largest.nil?
|
|
62
|
+
row.reason = 'register a drive (or pass --largest-drive) to check whether every folder fits'
|
|
63
|
+
elsif row.largest_subfolder.to_i > largest
|
|
64
|
+
row.fits = false
|
|
65
|
+
row.reason = "WARNING: #{row.largest_name} alone is #{Placement.format_bytes(row.largest_subfolder)}, " \
|
|
66
|
+
"bigger than the largest drive currently registered (#{Placement.format_bytes(largest)}); " \
|
|
67
|
+
'it cannot be placed on this fleet today, but will fit once you add a bigger drive'
|
|
68
|
+
else
|
|
69
|
+
row.fits = true
|
|
70
|
+
row.reason = 'every folder fits on the largest drive'
|
|
71
|
+
end
|
|
72
|
+
row
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|