easy_sync 0.0.5 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +8 -0
  3. data/.rspec +2 -0
  4. data/.ruby-version +1 -0
  5. data/CHANGELOG.md +85 -0
  6. data/CLAUDE.md +133 -0
  7. data/Gemfile +2 -0
  8. data/README.md +607 -21
  9. data/Rakefile +6 -8
  10. data/bin/easy_sync +8 -1
  11. data/docs/fine-placement.md +174 -0
  12. data/docs/hardware-notes.md +71 -0
  13. data/docs/integrity-scan.md +435 -0
  14. data/docs/parallel-scrub.md +280 -0
  15. data/docs/performance.md +146 -0
  16. data/docs/tripwire.md +232 -0
  17. data/easy_sync.gemspec +25 -16
  18. data/lib/easy_sync/cli.rb +1246 -0
  19. data/lib/easy_sync/config.rb +202 -0
  20. data/lib/easy_sync/error.rb +5 -0
  21. data/lib/easy_sync/jbod/backblaze.rb +113 -0
  22. data/lib/easy_sync/jbod/benchmarker.rb +149 -0
  23. data/lib/easy_sync/jbod/cleaner.rb +93 -0
  24. data/lib/easy_sync/jbod/dashboard.rb +611 -0
  25. data/lib/easy_sync/jbod/keep_awake.rb +33 -0
  26. data/lib/easy_sync/jbod/manifest.rb +1068 -0
  27. data/lib/easy_sync/jbod/mirror.rb +204 -0
  28. data/lib/easy_sync/jbod/models.rb +165 -0
  29. data/lib/easy_sync/jbod/page_cache.rb +56 -0
  30. data/lib/easy_sync/jbod/placement.rb +71 -0
  31. data/lib/easy_sync/jbod/planner.rb +76 -0
  32. data/lib/easy_sync/jbod/purger.rb +161 -0
  33. data/lib/easy_sync/jbod/restorer.rb +142 -0
  34. data/lib/easy_sync/jbod/run_lock.rb +98 -0
  35. data/lib/easy_sync/jbod/run_log.rb +57 -0
  36. data/lib/easy_sync/jbod/runner.rb +615 -0
  37. data/lib/easy_sync/jbod/scrub_pool.rb +136 -0
  38. data/lib/easy_sync/jbod/scrubber.rb +321 -0
  39. data/lib/easy_sync/jbod/share_scan.rb +26 -0
  40. data/lib/easy_sync/jbod/sync_eta.rb +68 -0
  41. data/lib/easy_sync/jbod/tripwire.rb +80 -0
  42. data/lib/easy_sync/jbod/volume_info.rb +283 -0
  43. data/lib/easy_sync/jbod.rb +35 -0
  44. data/lib/easy_sync/shell.rb +62 -0
  45. data/lib/easy_sync/templates/dashboard.html.erb +419 -0
  46. data/lib/easy_sync/version.rb +3 -1
  47. data/lib/easy_sync.rb +7 -3
  48. metadata +84 -45
  49. data/Gemfile.lock +0 -55
  50. data/Guardfile +0 -28
  51. data/lib/easy_sync/rsync.rb +0 -60
  52. data/lib/easy_sync/sync_logger.rb +0 -20
  53. data/lib/easy_sync/sync_runner.rb +0 -44
  54. data/spec/easy_sync/rsync_spec.rb +0 -25
  55. data/spec/easy_sync/sync_runner_spec.rb +0 -54
  56. data/spec/spec_helper.rb +0 -33
  57. data/spec/support.rb +0 -89
@@ -0,0 +1,204 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'fileutils'
4
+ require 'tempfile'
5
+
6
+ module EasySync
7
+ module Jbod
8
+ # Mirrors one source folder onto one destination folder with rsync.
9
+ #
10
+ # Nothing is ever deleted by rsync itself. Each folder gets two passes: a
11
+ # read-only check (`rsync -n --delete --itemize-changes --stats`, see
12
+ # #check) run before any copying starts, then a copy pass with no deletion
13
+ # flags at all. The check's "*deleting path" lines say which files on the
14
+ # drive no longer exist on the source (they feed the grace-period purge in
15
+ # Runner), and its ">f" lines say which existing files the copy would
16
+ # overwrite (the tripwire, docs/tripwire.md). (`--delete --max-delete=0`
17
+ # looked like a one-pass alternative, but rsync then only prints
18
+ # "N skipped" without naming the files, and exits 25.)
19
+ class Mirror
20
+ MIN_VERSION = [3, 0, 0].freeze
21
+ # Anchored to the transfer root, directories only: a share's root-files
22
+ # unit copies its loose top-level files and nothing below them.
23
+ ROOT_ONLY = '--exclude=/*/'
24
+
25
+ Result = Struct.new(:exit_status, :bytes_transferred, :total_size_bytes, :disk_full, :output,
26
+ keyword_init: true) do
27
+ def success? = exit_status.zero?
28
+ def disk_full? = !!disk_full
29
+ end
30
+
31
+ # What the read-only check found for one folder. +replaced+ are existing
32
+ # files on the drive the copy would overwrite; +missing+ is
33
+ # [[path, kind], ...] on the drive but gone from the source (for the
34
+ # purge); +junk+ are the missing paths that only match exclude_folders
35
+ # (already-copied junk, not something lost on the NAS); +new_files+ is
36
+ # how many files the copy would add; +source_files+ is the source's
37
+ # regular-file count from --stats. +known+ are paths an earlier run
38
+ # already found missing (Runner fills it in from pending_deletions):
39
+ # they wait out the grace period on the drive and are not news, so an
40
+ # accepted bulk delete doesn't trip again on every run until it's purged.
41
+ Check = Struct.new(:replaced, :missing, :junk, :new_files, :source_files, :known, keyword_init: true) do
42
+ def missing_files
43
+ skip = Set.new(junk) + Array(known)
44
+ missing.select { |path, kind| kind == 'file' && !skip.include?(path) }.map(&:first)
45
+ end
46
+
47
+ def changed = replaced.size + missing_files.size
48
+
49
+ # Files on the drive, worked out without walking it: what the source
50
+ # has, minus what the copy would add, plus what only the drive has.
51
+ def files_on_drive = [source_files.to_i - new_files + missing.count { |_, kind| kind == 'file' }, 0].max
52
+ def samples(limit = 10) = (replaced + missing_files).first(limit)
53
+ end
54
+
55
+ # +excludes+ are rsync patterns (no slash, so they match at any depth):
56
+ # the config's exclude_folders. They apply to both passes.
57
+ def initialize(shell: Shell.new, extra_args: [], excludes: [])
58
+ @shell = shell
59
+ @extra_args = Array(extra_args)
60
+ @patterns = Array(excludes)
61
+ @excludes = @patterns.map { |e| "--exclude=#{e}" }
62
+ end
63
+
64
+ # The copy pass. Extra args (from config, or --dry-run) apply here.
65
+ # --partial keeps a killed transfer's in-progress file instead of deleting
66
+ # it, so a multi-GB file interrupted mid-copy resumes next run instead of
67
+ # restarting from zero.
68
+ def command(source, destination, root_only: false)
69
+ argv = ['rsync', '-a', '--partial', '--stats', '--info=progress2', '--itemize-changes', *@excludes]
70
+ argv << ROOT_ONLY if root_only
71
+ argv += @extra_args
72
+ argv + [with_slash(source), with_slash(destination)]
73
+ end
74
+
75
+ # The check probe: never copies, never deletes, only reports.
76
+ # --stats adds the source's file count, for the tripwire's ratio.
77
+ # --delete-excluded makes it also report excluded junk that an earlier
78
+ # run copied before the exclusion existed, so the purge clears it.
79
+ # A root-files unit (root_only) drops --delete-excluded: its subfolders
80
+ # are excluded, and they belong to other folders, so they must never be
81
+ # reported as extraneous. Without --delete-excluded rsync protects them.
82
+ def probe_command(source, destination, root_only: false)
83
+ return ['rsync', '-an', '--itemize-changes', '--stats', '--delete', *@excludes, ROOT_ONLY,
84
+ with_slash(source), with_slash(destination)] if root_only
85
+
86
+ ['rsync', '-an', '--itemize-changes', '--stats', '--delete', '--delete-excluded', *@excludes,
87
+ with_slash(source), with_slash(destination)]
88
+ end
89
+
90
+ # Raises unless the rsync on PATH is new enough for --itemize-changes deletion reporting.
91
+ def self.check_version!(shell)
92
+ result = shell.capture(['rsync', '--version'])
93
+ version = result.output[/rsync\s+version\s+(\d+)\.(\d+)\.(\d+)/, 0]
94
+ raise Error, 'rsync not found on PATH' unless result.success? && version
95
+
96
+ numbers = version.scan(/\d+/).map(&:to_i)
97
+ return numbers.join('.') if (numbers <=> MIN_VERSION) >= 0
98
+
99
+ raise Error, "rsync #{numbers.join('.')} is too old; 3.0 or newer is required (brew install rsync)"
100
+ rescue Errno::ENOENT
101
+ raise Error, 'rsync not found on PATH'
102
+ end
103
+
104
+ def sync(source, destination, root_only: false)
105
+ raise Error, "source folder #{source} does not exist" unless Dir.exist?(source)
106
+
107
+ FileUtils.mkdir_p(File.dirname(destination))
108
+ result = @shell.run(command(source, destination, root_only: root_only))
109
+ stats = self.class.parse_stats(result.output)
110
+ Result.new(exit_status: result.status, output: result.output,
111
+ disk_full: self.class.disk_full?(result.output), **stats)
112
+ end
113
+
114
+ # Runs the check probe. Returns a Check, or nil if the probe failed (the
115
+ # caller then neither copies the folder nor records anything for it).
116
+ def check(source, destination, root_only: false)
117
+ result = @shell.capture(probe_command(source, destination, root_only: root_only))
118
+ return nil unless result.success?
119
+
120
+ missing = self.class.parse_extraneous(result.output)
121
+ changes = self.class.parse_file_changes(result.output)
122
+ Check.new(replaced: changes[:replaced], missing: missing, new_files: changes[:new],
123
+ junk: missing.map(&:first).select { |path| junk?(path) },
124
+ source_files: self.class.regular_files(result.output))
125
+ end
126
+
127
+ # Re-copies files `scrub` flagged as corrupt/unreadable, overwriting the
128
+ # bad copy on the drive. `-I` ignores rsync's quick size/mtime check
129
+ # (which would otherwise skip a file whose size and mtime never
130
+ # changed - the whole reason rot goes unnoticed). No --partial: an
131
+ # interrupted refetch must leave the old, still-flagged file in place,
132
+ # not a half-written one - rsync only renames its temp file over the
133
+ # target once that file's transfer completes.
134
+ def refetch(source, destination, relative_paths)
135
+ list = Tempfile.new('easy_sync-refetch')
136
+ begin
137
+ list.write(relative_paths.join("\0"))
138
+ list.close
139
+ @shell.run(['rsync', '-a', '-I', '--stats', '--from0', "--files-from=#{list.path}",
140
+ with_slash(source), with_slash(destination)])
141
+ ensure
142
+ list.unlink
143
+ end
144
+ end
145
+
146
+ # True when rsync's own output says the destination ran out of space.
147
+ # Deliberately a text match rather than a specific exit status: rsync
148
+ # reports this the same way (exit 11) as other unrelated I/O errors.
149
+ def self.disk_full?(output)
150
+ output.match?(/No space left on device/i)
151
+ end
152
+
153
+ # Pulls the two numbers we keep out of `rsync --stats` output.
154
+ def self.parse_stats(output)
155
+ {
156
+ total_size_bytes: stat(output, 'Total file size'),
157
+ bytes_transferred: stat(output, 'Total transferred file size')
158
+ }
159
+ end
160
+
161
+ # "*deleting some/dir/" and "*deleting some/file" lines -> [[path, kind], ...]
162
+ def self.parse_extraneous(output)
163
+ output.scan(/^\*deleting\s+(.+?)\r?$/).map do |(path)|
164
+ path.end_with?('/') ? [path.chomp('/'), 'dir'] : [path, 'file']
165
+ end
166
+ end
167
+
168
+ # ">f+++++++++ path" is a new file; any other ">f" line is an existing
169
+ # file the copy would overwrite. Directories, symlinks and
170
+ # attribute-only (".f") lines change no file's content.
171
+ def self.parse_file_changes(output)
172
+ changes = { replaced: [], new: 0 }
173
+ output.scan(/^>f(\S+) (.+?)\r?$/) do |flags, path|
174
+ flags.start_with?('+') ? changes[:new] += 1 : changes[:replaced] << path
175
+ end
176
+ changes
177
+ end
178
+
179
+ # "Number of files: 1,234 (reg: 1,000, dir: 234)" -> 1000. A source
180
+ # with no regular files leaves "reg:" out entirely.
181
+ def self.regular_files(output)
182
+ line = output[/^Number of files:.*$/] or return 0
183
+ line[/reg:\s*([\d,]+)/, 1].to_s.delete(',').to_i
184
+ end
185
+
186
+ def self.stat(output, label)
187
+ match = output[/^#{Regexp.escape(label)}:\s*([\d,.]+)/, 1]
188
+ match && match.delete(',').to_i
189
+ end
190
+
191
+ private
192
+
193
+ # A path the probe reports only because --delete-excluded finds
194
+ # already-copied junk: one of its components matches exclude_folders.
195
+ def junk?(path)
196
+ path.split('/').any? { |part| @patterns.any? { |pat| File.fnmatch?(pat, part) } }
197
+ end
198
+
199
+ def with_slash(path)
200
+ path.end_with?('/') ? path : "#{path}/"
201
+ end
202
+ end
203
+ end
204
+ end
@@ -0,0 +1,165 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'time'
4
+
5
+ module EasySync
6
+ module Jbod
7
+ Drive = Struct.new(:serial_number, :friendly_name, :capacity_bytes, :added_date, :volume_uuid, :model,
8
+ :last_seen_at, :last_used_bytes, :last_free_bytes,
9
+ :smart_status, :smart_detail, :smart_checked_at, :power_on_hours, :retired_at,
10
+ keyword_init: true) do
11
+ def retired? = !retired_at.nil?
12
+
13
+ def used_fraction
14
+ return nil if last_used_bytes.nil? || capacity_bytes.to_i.zero?
15
+
16
+ last_used_bytes.to_f / capacity_bytes
17
+ end
18
+
19
+ # smartctl's Device Model has no separate "vendor" field for ATA/NVMe
20
+ # drives (that's a SCSI/USB-bridge thing); the maker is only ever
21
+ # visible as a token baked into the model string itself, or - for
22
+ # Seagate - not spelled out at all, just implied by the "ST" model
23
+ # prefix every Seagate drive uses.
24
+ BRANDS = {
25
+ 'WDC' => 'Western Digital', 'WD' => 'Western Digital', 'TOSHIBA' => 'Toshiba', 'HGST' => 'HGST',
26
+ 'SAMSUNG' => 'Samsung', 'HITACHI' => 'Hitachi', 'APPLE' => 'Apple', 'SEAGATE' => 'Seagate',
27
+ 'CRUCIAL' => 'Crucial', 'KINGSTON' => 'Kingston', 'INTEL' => 'Intel'
28
+ }.freeze
29
+
30
+ def manufacturer
31
+ return nil unless model
32
+
33
+ token = model[/\A[A-Za-z]+/]
34
+ return BRANDS[token.upcase] if token && BRANDS.key?(token.upcase)
35
+
36
+ 'Seagate' if model.match?(/\AST\d/i)
37
+ end
38
+
39
+ # model, with the manufacturer spelled out plainly instead of a raw
40
+ # token (or, for Seagate, added - its model numbers don't carry one).
41
+ def branded_model
42
+ return nil unless model
43
+ return model unless manufacturer
44
+
45
+ rest = model.sub(/\A(?:WDC|WD|TOSHIBA|HGST|SAMSUNG|HITACHI|APPLE|SEAGATE|CRUCIAL|KINGSTON|INTEL)[\s-]+/i, '')
46
+ "#{manufacturer} #{rest}"
47
+ end
48
+
49
+ # SMART's Power_On_Hours counts only time actually spinning/powered,
50
+ # unlike calendar age: a 5-year-old drive that sat on a shelf can show
51
+ # a fraction of the wear of one bought last year and run constantly.
52
+ def power_on_label
53
+ return nil unless power_on_hours
54
+
55
+ years = power_on_hours / 24.0 / 365
56
+ return format('%d days (%d hrs)', (power_on_hours / 24.0).round, power_on_hours) if years < 1
57
+
58
+ format('%.1f yrs (%d hrs)', years, power_on_hours)
59
+ end
60
+ end
61
+
62
+ # scope 'tree' is a folder copied with everything under it. scope 'root'
63
+ # is a share's loose top-level files only (folder_path is the share name);
64
+ # the share's subfolders are their own 'tree' folders, possibly elsewhere.
65
+ Folder = Struct.new(:folder_path, :drive_serial, :size_bytes, :assigned_at,
66
+ :last_synced_at, :last_sync_status, :scope, keyword_init: true) do
67
+ def root? = scope == 'root'
68
+ def share = folder_path.split('/').first
69
+ end
70
+
71
+ # One folder held back (or let through with --accept-changes) by the
72
+ # tripwire in one sync run. +samples+ is a few of its changed paths.
73
+ TripRecord = Struct.new(:id, :run_started_at, :folder_path, :scope, :replaced, :missing, :files_on_drive,
74
+ :samples, :accepted_at, keyword_init: true) do
75
+ def changed = replaced.to_i + missing.to_i
76
+ def accepted? = !accepted_at.nil?
77
+ end
78
+
79
+ HistoryEntry = Struct.new(:id, :folder_path, :drive_serial, :event, :recorded_at, :note,
80
+ keyword_init: true)
81
+
82
+ SyncRun = Struct.new(:id, :folder_path, :drive_serial, :started_at, :finished_at, :exit_status,
83
+ :bytes_transferred, :total_size_bytes, keyword_init: true)
84
+
85
+ # One whole `sync` run, summed over its folders (Manifest#run_summaries).
86
+ # copied counts folders that transferred any bytes; the rest only
87
+ # confirmed nothing had changed.
88
+ RunSummary = Struct.new(:run_started_at, :last_finished_at, :folders, :copied, :bytes_transferred, :failed,
89
+ keyword_init: true)
90
+
91
+ # A path on a drive that is a candidate for deletion, either because
92
+ # rsync reported it gone from the NAS (cause 'missing_on_nas';
93
+ # relative_path is '' with kind 'folder' when the whole folder is gone)
94
+ # or because the folder was reassigned off +drive_serial+ (cause
95
+ # 'reassigned', always whole-folder; relative_path there holds the old
96
+ # drive's serial instead of a real path, only so a second reassignment of
97
+ # the same folder before the first cleanup runs gets its own row rather
98
+ # than colliding on the folder_path+relative_path uniqueness).
99
+ PendingDeletion = Struct.new(:id, :folder_path, :relative_path, :kind, :drive_serial, :cause, :first_missing_at,
100
+ :last_missing_at, :missing_runs, keyword_init: true) do
101
+ def whole_folder? = kind == 'folder'
102
+ def reassigned? = cause == 'reassigned'
103
+
104
+ def expires_at(grace_days)
105
+ Time.parse(first_missing_at) + (grace_days * 86_400)
106
+ end
107
+
108
+ def expired?(now:, grace_days:, grace_runs:)
109
+ missing_runs >= grace_runs && expires_at(grace_days) <= now
110
+ end
111
+ end
112
+
113
+ Deletion = Struct.new(:id, :folder_path, :relative_path, :kind, :drive_serial, :first_missing_at, :deleted_at,
114
+ keyword_init: true)
115
+
116
+ # One tracked file on one drive, for `scrub` (see docs/integrity-scan.md).
117
+ # digest is the SHA-256 baseline (nil until the first hash); status is
118
+ # 'ok', 'corrupt' (hash mismatch), 'unreadable' (a read error), or
119
+ # 'unresolved' (still bad after sync refetched it - never refetched
120
+ # again automatically).
121
+ FileChecksum = Struct.new(:drive_serial, :folder_path, :relative_path, :size_bytes, :mtime, :digest,
122
+ :verified_at, :status, :failed_at, :refetched_at, keyword_init: true) do
123
+ def ok? = status == 'ok'
124
+ def flagged? = %w[corrupt unreadable].include?(status)
125
+ def unresolved? = status == 'unresolved'
126
+ def refetched? = !refetched_at.nil?
127
+ def label = "#{folder_path}/#{relative_path}"
128
+ end
129
+
130
+ # One `benchmark` run on one drive (see Jbod::Benchmarker). MB/s is
131
+ # MiB/s, the same unit scrub reports. used_bytes is how full the drive was
132
+ # at the time: a fuller drive writes to slower inner tracks, so a gradual
133
+ # decline that tracks it is expected, not a warning sign.
134
+ DriveBenchmark = Struct.new(:id, :drive_serial, :run_at, :bytes, :write_mb_s, :read_mb_s, :used_bytes,
135
+ keyword_init: true)
136
+
137
+ # One row per folder seen on the NAS at the last completed placement
138
+ # pass: placed (assigned to a drive, synced or queued), unplaced (no drive
139
+ # has room, or nothing is mounted) or empty (no real files).
140
+ SourceEntry = Struct.new(:folder_path, :size_bytes, :state, :detail, :seen_at, keyword_init: true) do
141
+ def share = folder_path.split('/').first
142
+ end
143
+
144
+ # SMART health as last read from the drive. status is one of
145
+ # 'ok' (self-assessment passed, no bad-sector counters), 'warning' (passed
146
+ # but reallocated/pending/uncorrectable sectors or an NVMe critical flag:
147
+ # the drive is starting to fail), 'failing' (self-assessment FAILED), or
148
+ # 'unknown' (SMART not exposed by the enclosure, smartctl missing, etc.).
149
+ # reallocated_sector_ct is the raw counter, tracked separately over time
150
+ # (see Manifest#record_smart_check) so a drive whose count is old and
151
+ # unchanging can be told apart from one that's actively climbing.
152
+ # other_bad is true when pending/uncorrectable sectors, media errors, or
153
+ # an NVMe critical-warning flag also contributed to a 'warning' status -
154
+ # those are never downgraded by reallocated-count history.
155
+ Health = Struct.new(:status, :detail, :source, :power_on_hours, :reallocated_sector_ct, :other_bad,
156
+ keyword_init: true)
157
+
158
+ # A registered drive that is currently mounted, with live usage numbers.
159
+ MountedDrive = Struct.new(:drive, :mount_point, :capacity_bytes, :used_bytes, :free_bytes,
160
+ keyword_init: true) do
161
+ def serial_number = drive.serial_number
162
+ def friendly_name = drive.friendly_name
163
+ end
164
+ end
165
+ end
@@ -0,0 +1,56 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'fiddle'
4
+
5
+ module EasySync
6
+ module Jbod
7
+ # Drops one file's pages from the macOS unified buffer cache, so the read
8
+ # that follows comes off the drive. F_NOCACHE alone is not enough: it
9
+ # stops a read from *adding* pages to the cache, but pages already there
10
+ # (a file sync just wrote, or anything read recently) are still served
11
+ # from RAM. Measured on jbod-test-1 (USB): a just-synced 100 MB file read
12
+ # at 12,037 MB/s with F_NOCACHE, and at 156 MB/s after this. msync with
13
+ # MS_INVALIDATE is the BSD/macOS stand-in for Linux's
14
+ # posix_fadvise(DONTNEED) (vmtouch uses it the same way); MS_SYNC first
15
+ # flushes any pending writes, so nothing unwritten is ever discarded.
16
+ # Best effort: any failure just leaves the cache as it was.
17
+ module PageCache
18
+ PROT_READ = 0x01
19
+ MAP_SHARED = 0x01
20
+ MS_SYNC = 0x10
21
+ MS_INVALIDATE = 0x02
22
+
23
+ module_function
24
+
25
+ def evict(io)
26
+ len = io.size
27
+ return false if len.zero?
28
+
29
+ addr = functions[:mmap].call(nil, len, PROT_READ, MAP_SHARED, io.fileno, 0)
30
+ return false if [-1, (2**64) - 1].include?(addr.to_i)
31
+
32
+ begin
33
+ functions[:msync].call(addr, len, MS_SYNC | MS_INVALIDATE).zero?
34
+ ensure
35
+ functions[:munmap].call(addr, len)
36
+ end
37
+ rescue StandardError, Fiddle::DLError
38
+ false
39
+ end
40
+
41
+ def functions
42
+ @functions ||= begin
43
+ libc = Fiddle.dlopen(nil)
44
+ {
45
+ mmap: Fiddle::Function.new(libc['mmap'], [Fiddle::TYPE_VOIDP, Fiddle::TYPE_SIZE_T, Fiddle::TYPE_INT,
46
+ Fiddle::TYPE_INT, Fiddle::TYPE_INT, Fiddle::TYPE_LONG_LONG],
47
+ Fiddle::TYPE_VOIDP),
48
+ msync: Fiddle::Function.new(libc['msync'], [Fiddle::TYPE_VOIDP, Fiddle::TYPE_SIZE_T, Fiddle::TYPE_INT],
49
+ Fiddle::TYPE_INT),
50
+ munmap: Fiddle::Function.new(libc['munmap'], [Fiddle::TYPE_VOIDP, Fiddle::TYPE_SIZE_T], Fiddle::TYPE_INT)
51
+ }
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,71 @@
1
+ # frozen_string_literal: true
2
+
3
+ module EasySync
4
+ module Jbod
5
+ # Decides where a brand-new folder goes. Pure logic, no I/O.
6
+ #
7
+ # Rule: a drive in +prefer+ (one already holding part of the same share)
8
+ # wins if the folder fits there, so a share stays on one drive while it
9
+ # can; otherwise the mounted drive with the most free space wins. A folder
10
+ # is never placed on a drive it would not fit on, and existing folders are
11
+ # never moved (that is what makes the JBOD layout browsable by hand).
12
+ module Placement
13
+ class NoMountedDrives < Error; end
14
+ class DoesNotFit < Error; end
15
+
16
+ # +candidates+ are MountedDrive structs. +size_bytes+ is the folder size
17
+ # (nil when unknown, in which case only the free-space ordering applies).
18
+ # +reserve_bytes+ is headroom to leave on the drive after placement.
19
+ def self.choose(candidates, size_bytes: nil, reserve_bytes: 0, prefer: [])
20
+ raise NoMountedDrives, 'no registered drives are mounted' if candidates.empty?
21
+
22
+ preferred = candidates.select do |c|
23
+ prefer.include?(c.serial_number) && (size_bytes.nil? || c.free_bytes - reserve_bytes >= size_bytes)
24
+ end
25
+ pool = preferred.empty? ? candidates : preferred
26
+ best = pool.max_by { |c| [c.free_bytes, c.friendly_name.to_s] }
27
+ if size_bytes && (best.free_bytes - reserve_bytes) < size_bytes
28
+ raise DoesNotFit,
29
+ "#{format_bytes(size_bytes)} does not fit on #{best.friendly_name} " \
30
+ "(#{format_bytes(best.free_bytes)} free, #{format_bytes(reserve_bytes)} reserved)"
31
+ end
32
+ best
33
+ end
34
+
35
+ # "2gb", "500 MB", 2147483648 -> bytes. Used for config values and CLI flags.
36
+ def self.parse_size(value)
37
+ return value.to_i if value.is_a?(Numeric)
38
+
39
+ m = value.to_s.strip.match(/\A([\d.]+)\s*(tb|gb|mb|kb|b)?\z/i) or raise Error, "cannot parse size #{value.inspect} (try 2gb)"
40
+ (m[1].to_f * { nil => 1, 'b' => 1, 'kb' => 1024, 'mb' => 1024**2, 'gb' => 1024**3, 'tb' => 1024**4 }[m[2]&.downcase]).to_i
41
+ end
42
+
43
+ UNITS = %w[B KB MB GB TB PB].freeze
44
+
45
+ def self.format_bytes(bytes)
46
+ return '—' if bytes.nil?
47
+
48
+ value = bytes.to_f
49
+ unit = 0
50
+ while value >= 1024 && unit < UNITS.size - 1
51
+ value /= 1024
52
+ unit += 1
53
+ end
54
+ unit.zero? ? "#{bytes} B" : format('%.1f %s', value, UNITS[unit])
55
+ end
56
+
57
+ # "3d 4h", "2h 34m", "45m", or "12s". Shared by `status` and the
58
+ # dashboard so an elapsed/remaining time reads the same everywhere.
59
+ def self.format_duration(seconds)
60
+ days, rem = seconds.to_i.divmod(86_400)
61
+ hours, rem = rem.divmod(3600)
62
+ minutes, secs = rem.divmod(60)
63
+ return "#{days}d #{hours}h" if days.positive?
64
+ return "#{hours}h #{minutes}m" if hours.positive?
65
+ return "#{minutes}m #{secs}s" if minutes.positive?
66
+
67
+ "#{secs}s"
68
+ end
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ module EasySync
4
+ module Jbod
5
+ # `easy_sync plan`: measures each configured share and checks that every
6
+ # top-level folder (each is placed on its own) fits on the largest drive
7
+ # in the fleet. Reads only; never places or copies anything.
8
+ class Planner
9
+ Row = Struct.new(:source, :mounted, :size_bytes, :subfolders, :largest_subfolder, :largest_name, :loose_files,
10
+ :reason, :fits, keyword_init: true)
11
+
12
+ def initialize(settings, shell: Shell.new, largest_drive_bytes: nil)
13
+ @settings = settings
14
+ @shell = shell
15
+ @largest = largest_drive_bytes
16
+ end
17
+
18
+ # +only+ restricts to sources whose basename or full path is in the list
19
+ # (e.g. easy_sync plan pro, matching /Volumes/pro). nil measures all.
20
+ def rows(only: nil)
21
+ sources = Array(@settings[:sources]).map { |e| Runner::Source.from_config(e) }
22
+ sources = sources.select { |s| only.include?(s.name) || only.include?(s.path) } if only
23
+ sources.map { |src| measure(src) }
24
+ end
25
+
26
+ private
27
+
28
+ def measure(source)
29
+ row = Row.new(source: source, mounted: Dir.exist?(source.path) && !Dir.empty?(source.path), loose_files: 0,
30
+ subfolders: 0)
31
+ return row.tap { |r| r.reason = 'not mounted (or empty)' } unless row.mounted
32
+
33
+ excluded = Array(@settings[:exclude_folders])
34
+ children = Dir.children(source.path).sort.reject { |n| excluded.include?(n) || n.start_with?('.') }
35
+ dirs = children.select { |n| File.directory?(File.join(source.path, n)) }
36
+ row.loose_files = children.count { |n| File.file?(File.join(source.path, n)) }
37
+ row.subfolders = dirs.size
38
+
39
+ sizes = du(dirs.map { |n| File.join(source.path, n) })
40
+ row.size_bytes = sizes.values.sum
41
+ row.largest_name, row.largest_subfolder = sizes.max_by { |_, v| v } || [nil, 0]
42
+ check_fit(row)
43
+ end
44
+
45
+ # One `du -sk` over every subfolder; the tool's own `du` cost was
46
+ # measured at ~6 s for 2,380 movie folders over SMB.
47
+ def du(paths)
48
+ return {} if paths.empty?
49
+
50
+ result = @shell.capture(['du', '-sk', *paths])
51
+ result.output.each_line.with_object({}) do |line, h|
52
+ kb, path = line.chomp.split("\t", 2)
53
+ next unless kb && path
54
+
55
+ h[File.basename(path.chomp('/'))] = kb.to_i * 1024
56
+ end
57
+ end
58
+
59
+ def check_fit(row)
60
+ largest = @largest
61
+ if largest.nil?
62
+ row.reason = 'register a drive (or pass --largest-drive) to check whether every folder fits'
63
+ elsif row.largest_subfolder.to_i > largest
64
+ row.fits = false
65
+ row.reason = "WARNING: #{row.largest_name} alone is #{Placement.format_bytes(row.largest_subfolder)}, " \
66
+ "bigger than the largest drive currently registered (#{Placement.format_bytes(largest)}); " \
67
+ 'it cannot be placed on this fleet today, but will fit once you add a bigger drive'
68
+ else
69
+ row.fits = true
70
+ row.reason = 'every folder fits on the largest drive'
71
+ end
72
+ row
73
+ end
74
+ end
75
+ end
76
+ end