mclone 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/mclone.rb +745 -745
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 10e8e59126b5586e4ed7d68af24aa1c9de5eebfc2e9d0e41df63735577de1a3f
|
|
4
|
+
data.tar.gz: 10b171b2c477c07993641e46bcb27e6c3d3766217d02683e009fbb85a5798cb8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9a14df6196d48aa0c438c1e3659aa25dc8193e77bdf487f34577c24cfd2e4d9ab41ab1a5fcf31ce0b021404118446467bfbe778d0e82c2ec65a8832f622eb35b
|
|
7
|
+
data.tar.gz: 0fae9170543c940e4089613ec2bf4f1b412ceedacadb214209ae1975369cc695a82c8eafad2660d81e19ed8144870210d037564fb9d03d16f2ed5c6aeafca306
|
data/lib/mclone.rb
CHANGED
|
@@ -1,746 +1,746 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
require 'date'
|
|
5
|
-
require 'json'
|
|
6
|
-
require 'open3'
|
|
7
|
-
require 'fileutils'
|
|
8
|
-
require 'shellwords'
|
|
9
|
-
require 'securerandom'
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
#
|
|
13
|
-
module Mclone
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
VERSION = '0.2.
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
#
|
|
20
|
-
class Error < StandardError; end
|
|
21
|
-
|
|
22
|
-
#
|
|
23
|
-
module Refinements
|
|
24
|
-
|
|
25
|
-
refine ::Hash do
|
|
26
|
-
# Same as #dig but raises KeyError exception on any non-existent key
|
|
27
|
-
def extract(*args)
|
|
28
|
-
case args.size
|
|
29
|
-
when 0 then raise(KeyError, 'non-empty key sequence expected')
|
|
30
|
-
when 1 then fetch(args.first)
|
|
31
|
-
else fetch(args.shift).extract(*args)
|
|
32
|
-
end
|
|
33
|
-
end
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
refine ::Array do
|
|
37
|
-
# Return a list of items which fully or partially match the specified pattern
|
|
38
|
-
def resolve(partial)
|
|
39
|
-
rx = Regexp.new(partial)
|
|
40
|
-
collect { |item| rx.match?(item.to_s) ? item : nil }.compact
|
|
41
|
-
end
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
refine ::String do
|
|
45
|
-
def escape
|
|
46
|
-
Mclone.windows? && %r![^\w\-\=\\\/:]!.match?(self) ? %("#{self}") : shellescape
|
|
47
|
-
end
|
|
48
|
-
end
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
using Refinements
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
# Two-way mapping between an object and its ID
|
|
56
|
-
class ObjectSet
|
|
57
|
-
|
|
58
|
-
include Enumerable
|
|
59
|
-
|
|
60
|
-
#
|
|
61
|
-
def each_id(&code)
|
|
62
|
-
@ids.each_key(&code)
|
|
63
|
-
end
|
|
64
|
-
|
|
65
|
-
#
|
|
66
|
-
def each(&code)
|
|
67
|
-
@objects.each_value(&code)
|
|
68
|
-
end
|
|
69
|
-
|
|
70
|
-
#
|
|
71
|
-
def empty?
|
|
72
|
-
@objects.empty?
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
#
|
|
76
|
-
def size
|
|
77
|
-
@objects.size
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
def initialize
|
|
81
|
-
@ids = {} # { id => object }
|
|
82
|
-
@objects = {} # { object => object }
|
|
83
|
-
@modified = false
|
|
84
|
-
end
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
#
|
|
89
|
-
def hash
|
|
90
|
-
@objects.hash
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
#
|
|
94
|
-
def eql?(other)
|
|
95
|
-
equal?(other) || objects == other.objects
|
|
96
|
-
end
|
|
97
|
-
|
|
98
|
-
alias == eql?
|
|
99
|
-
|
|
100
|
-
# Return ID of the object considered equal to the specified obj or nil
|
|
101
|
-
def id(obj)
|
|
102
|
-
@objects[obj]&.id
|
|
103
|
-
end
|
|
104
|
-
|
|
105
|
-
# Return object with specified ID or nil
|
|
106
|
-
def object(id)
|
|
107
|
-
@ids[id]
|
|
108
|
-
end
|
|
109
|
-
|
|
110
|
-
# Return object considered equal to obj or nil
|
|
111
|
-
def [](obj)
|
|
112
|
-
@objects[obj]
|
|
113
|
-
end
|
|
114
|
-
|
|
115
|
-
#
|
|
116
|
-
def modified?
|
|
117
|
-
@modified
|
|
118
|
-
end
|
|
119
|
-
|
|
120
|
-
def commit!
|
|
121
|
-
@modified = false
|
|
122
|
-
self
|
|
123
|
-
end
|
|
124
|
-
|
|
125
|
-
# Unregister an object considered equal to the specified obj and return true if object has been actually removed
|
|
126
|
-
private def forget(obj)
|
|
127
|
-
!@ids.delete(@objects.delete(obj)&.id).nil?
|
|
128
|
-
end
|
|
129
|
-
|
|
130
|
-
# Return a list of registered IDs (fully or partially) matching the specified pattern
|
|
131
|
-
def resolve(pattern)
|
|
132
|
-
each_id.to_a.resolve(pattern)
|
|
133
|
-
end
|
|
134
|
-
|
|
135
|
-
# Either add brand new object or replace existing one equal to the specified object
|
|
136
|
-
def <<(obj)
|
|
137
|
-
forget(obj)
|
|
138
|
-
@objects[obj] = @ids[obj.id] = obj
|
|
139
|
-
@modified = true
|
|
140
|
-
obj
|
|
141
|
-
end
|
|
142
|
-
|
|
143
|
-
# Remove object considered equal to the specified obj
|
|
144
|
-
def >>(obj)
|
|
145
|
-
@modified = true if (status = forget(obj))
|
|
146
|
-
status
|
|
147
|
-
end
|
|
148
|
-
|
|
149
|
-
# Add all tasks from enumerable
|
|
150
|
-
def merge!(objs)
|
|
151
|
-
objs.each { |x| self << x }
|
|
152
|
-
self
|
|
153
|
-
end
|
|
154
|
-
|
|
155
|
-
end
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
#
|
|
159
|
-
class Task
|
|
160
|
-
|
|
161
|
-
#
|
|
162
|
-
class Error < Mclone::Error
|
|
163
|
-
end
|
|
164
|
-
|
|
165
|
-
#
|
|
166
|
-
attr_reader :id
|
|
167
|
-
|
|
168
|
-
#
|
|
169
|
-
attr_reader :source_id, :destination_id
|
|
170
|
-
|
|
171
|
-
#
|
|
172
|
-
attr_reader :source_root, :destination_root
|
|
173
|
-
|
|
174
|
-
#
|
|
175
|
-
attr_reader :mtime
|
|
176
|
-
|
|
177
|
-
#
|
|
178
|
-
attr_reader :mode
|
|
179
|
-
|
|
180
|
-
#
|
|
181
|
-
attr_reader :include, :exclude
|
|
182
|
-
|
|
183
|
-
#
|
|
184
|
-
attr_reader :crypter_mode
|
|
185
|
-
|
|
186
|
-
def hash
|
|
187
|
-
@hash ||= source_id.hash ^ destination_id.hash ^ source_root.hash ^ destination_root.hash
|
|
188
|
-
end
|
|
189
|
-
|
|
190
|
-
def eql?(other)
|
|
191
|
-
equal?(other) || (
|
|
192
|
-
source_id == other.source_id &&
|
|
193
|
-
destination_id == other.destination_id &&
|
|
194
|
-
source_root == other.source_root &&
|
|
195
|
-
destination_root == other.destination_root
|
|
196
|
-
)
|
|
197
|
-
end
|
|
198
|
-
|
|
199
|
-
alias == eql?
|
|
200
|
-
|
|
201
|
-
#
|
|
202
|
-
def initialize(session, mode, source_id, source_root, destination_id, destination_root, include: nil, exclude: nil, crypter_mode: nil, crypter_token: nil, crypter_password: nil)
|
|
203
|
-
@touch = false # Indicates that the time stamp should be updated whenever state of self is altered
|
|
204
|
-
@session = session
|
|
205
|
-
@id = SecureRandom.hex(4)
|
|
206
|
-
@source_id = source_id
|
|
207
|
-
@destination_id = destination_id
|
|
208
|
-
@source_root = source_root
|
|
209
|
-
@destination_root = destination_root
|
|
210
|
-
self.mode = mode
|
|
211
|
-
self.include = include
|
|
212
|
-
self.exclude = exclude
|
|
213
|
-
set_crypter_mode crypter_mode
|
|
214
|
-
unless crypter_mode.nil?
|
|
215
|
-
raise(Task::Error, %(either Rclone crypt token or plain text password is expected, not both)) if !crypter_token.nil? && !crypter_password.nil?
|
|
216
|
-
@assigned_token = register_crypter_token crypter_token
|
|
217
|
-
@assigned_password = crypter_password
|
|
218
|
-
end
|
|
219
|
-
ensure
|
|
220
|
-
@touch = true
|
|
221
|
-
touch!
|
|
222
|
-
end
|
|
223
|
-
|
|
224
|
-
CRYPTER_MODES = %i[encrypt decrypt].freeze
|
|
225
|
-
|
|
226
|
-
@@crypter_tokens = {}
|
|
227
|
-
|
|
228
|
-
private def set_crypter_mode(mode)
|
|
229
|
-
@crypter_mode = mode.nil? ? nil : (CRYPTER_MODES.include?(mode = mode.intern) ? mode : raise(Task::Error, %(unknown crypt mode "#{mode}")))
|
|
230
|
-
end
|
|
231
|
-
|
|
232
|
-
private def register_crypter_token(token)
|
|
233
|
-
unless token.nil?
|
|
234
|
-
@@crypter_tokens[id] = @@crypter_tokens[id].nil? ? token : raise(Task::Error, %(attempt to re-register token for task "#{id}"))
|
|
235
|
-
end
|
|
236
|
-
token
|
|
237
|
-
end
|
|
238
|
-
|
|
239
|
-
# Lazily determine the crypt token from either assigned values or the token repository
|
|
240
|
-
def crypter_token
|
|
241
|
-
# Locally assigned token takes precedence over the repository's
|
|
242
|
-
unless @assigned_token.nil?
|
|
243
|
-
@@crypter_tokens[id] = @assigned_token unless @@crypter_tokens[id].nil? # Assign repository entry with this local token if not yet assigned
|
|
244
|
-
@assigned_token
|
|
245
|
-
else
|
|
246
|
-
if @@crypter_tokens[id].nil?
|
|
247
|
-
# If token is neither locally assigned nor in repository, try to construct it from the user-supplied password
|
|
248
|
-
# If a user-supplied password is omitted, create a new randomly generated password
|
|
249
|
-
args = [Mclone.rclone, 'obscure', @assigned_password.nil? ? SecureRandom.alphanumeric(16) : @assigned_password]
|
|
250
|
-
$stdout << args.collect(&:escape).join(' ') << "\n" if @session.verbose?
|
|
251
|
-
stdout, status = Open3.capture2(*args)
|
|
252
|
-
raise(Task::Error, %(Rclone execution failure)) unless status.success?
|
|
253
|
-
@@crypter_tokens[id] = stdout.strip
|
|
254
|
-
else
|
|
255
|
-
@@crypter_tokens[id]
|
|
256
|
-
end
|
|
257
|
-
end
|
|
258
|
-
end
|
|
259
|
-
|
|
260
|
-
#
|
|
261
|
-
MODES = %i[update synchronize copy move].freeze
|
|
262
|
-
|
|
263
|
-
#
|
|
264
|
-
def mode=(mode)
|
|
265
|
-
@mode = MODES.include?(mode = mode.intern) ? mode : raise(Task::Error, %(unknown mode "#{mode}"))
|
|
266
|
-
touch!
|
|
267
|
-
mode
|
|
268
|
-
end
|
|
269
|
-
|
|
270
|
-
#
|
|
271
|
-
def include=(mask)
|
|
272
|
-
@include = mask.nil? || mask == '-' ? nil : mask # TODO: verify mask
|
|
273
|
-
touch!
|
|
274
|
-
mask
|
|
275
|
-
end
|
|
276
|
-
|
|
277
|
-
#
|
|
278
|
-
def exclude=(mask)
|
|
279
|
-
@exclude = mask.nil? || mask == '-' ? nil : mask # TODO: verify mask
|
|
280
|
-
touch!
|
|
281
|
-
mask
|
|
282
|
-
end
|
|
283
|
-
|
|
284
|
-
#
|
|
285
|
-
def self.restore(session, hash)
|
|
286
|
-
obj = allocate
|
|
287
|
-
obj.send(:from_h, session, hash)
|
|
288
|
-
obj
|
|
289
|
-
end
|
|
290
|
-
|
|
291
|
-
#
|
|
292
|
-
private def from_h(session, hash)
|
|
293
|
-
@session = session
|
|
294
|
-
@touch = false
|
|
295
|
-
@id = hash.extract(:task)
|
|
296
|
-
@mtime = DateTime.parse(hash.extract(:mtime)) rescue DateTime.now # Deleting mtime entry from json can be used to modify data out of mclone
|
|
297
|
-
@source_id = hash.extract(:source, :volume)
|
|
298
|
-
@destination_id = hash.extract(:destination, :volume)
|
|
299
|
-
@source_root = hash.dig(:source, :root)
|
|
300
|
-
@destination_root = hash.dig(:destination, :root)
|
|
301
|
-
self.mode = hash.extract(:mode)
|
|
302
|
-
self.include = hash.dig(:include)
|
|
303
|
-
self.exclude = hash.dig(:exclude)
|
|
304
|
-
set_crypter_mode hash.dig(:crypter, :mode)
|
|
305
|
-
@assigned_token = register_crypter_token(hash.dig(:crypter, :token)) unless crypter_mode.nil?
|
|
306
|
-
ensure
|
|
307
|
-
@touch = true
|
|
308
|
-
end
|
|
309
|
-
|
|
310
|
-
#
|
|
311
|
-
def to_h(volume)
|
|
312
|
-
hash = {
|
|
313
|
-
task: id,
|
|
314
|
-
mode: mode,
|
|
315
|
-
mtime: mtime,
|
|
316
|
-
source: { volume: source_id },
|
|
317
|
-
destination: { volume: destination_id }
|
|
318
|
-
}
|
|
319
|
-
hash[:source][:root] = source_root unless source_root.nil? || source_root.empty?
|
|
320
|
-
hash[:destination][:root] = destination_root unless destination_root.nil? || destination_root.empty?
|
|
321
|
-
hash[:include] = include unless include.nil?
|
|
322
|
-
hash[:exclude] = exclude unless exclude.nil?
|
|
323
|
-
unless crypter_mode.nil?
|
|
324
|
-
crypter = hash[:crypter] = { mode: crypter_mode }
|
|
325
|
-
# Make sure the token won't get into the encrypted volume's task
|
|
326
|
-
crypter[:token] = crypter_token if (crypter_mode == :encrypt && source_id == volume.id) || (crypter_mode == :decrypt && destination_id == volume.id)
|
|
327
|
-
end
|
|
328
|
-
hash
|
|
329
|
-
end
|
|
330
|
-
|
|
331
|
-
#
|
|
332
|
-
def touch!
|
|
333
|
-
@mtime = DateTime.now if @touch
|
|
334
|
-
self
|
|
335
|
-
end
|
|
336
|
-
end
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
#
|
|
340
|
-
class TaskSet < ObjectSet
|
|
341
|
-
|
|
342
|
-
alias task object
|
|
343
|
-
|
|
344
|
-
# Add new task or replace existing one with outdated timestamp
|
|
345
|
-
def <<(task)
|
|
346
|
-
t = self[task]
|
|
347
|
-
super if t.nil? || (!t.nil? && t.mtime < task.mtime)
|
|
348
|
-
task
|
|
349
|
-
end
|
|
350
|
-
|
|
351
|
-
#
|
|
352
|
-
def resolve(id)
|
|
353
|
-
case (ids = super).size
|
|
354
|
-
when 0 then raise(Task::Error, %(no task matching "#{id}" pattern found))
|
|
355
|
-
when 1 then ids.first
|
|
356
|
-
else raise(Task::Error, %(ambiguous "#{id}" pattern: two or more tasks match))
|
|
357
|
-
end
|
|
358
|
-
end
|
|
359
|
-
|
|
360
|
-
end
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
#
|
|
364
|
-
class Volume
|
|
365
|
-
|
|
366
|
-
#
|
|
367
|
-
class Error < Mclone::Error
|
|
368
|
-
|
|
369
|
-
end
|
|
370
|
-
|
|
371
|
-
#
|
|
372
|
-
VERSION = 0
|
|
373
|
-
|
|
374
|
-
#
|
|
375
|
-
FILE = '.mclone'
|
|
376
|
-
|
|
377
|
-
#
|
|
378
|
-
attr_reader :id
|
|
379
|
-
|
|
380
|
-
#
|
|
381
|
-
attr_reader :file
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
#
|
|
385
|
-
def root
|
|
386
|
-
@root ||= File.realpath(File.dirname(file))
|
|
387
|
-
end
|
|
388
|
-
|
|
389
|
-
#
|
|
390
|
-
def initialize(session, file)
|
|
391
|
-
@loaded_tasks = ObjectSet.new
|
|
392
|
-
@id = SecureRandom.hex(4)
|
|
393
|
-
@session = session
|
|
394
|
-
@file = file
|
|
395
|
-
end
|
|
396
|
-
|
|
397
|
-
#
|
|
398
|
-
def self.restore(session, file)
|
|
399
|
-
obj = allocate
|
|
400
|
-
obj.send(:from_file, session, file)
|
|
401
|
-
obj
|
|
402
|
-
end
|
|
403
|
-
|
|
404
|
-
#
|
|
405
|
-
private def from_file(session, file)
|
|
406
|
-
hash = JSON.parse(IO.read(file), symbolize_names: true)
|
|
407
|
-
@loaded_tasks = ObjectSet.new
|
|
408
|
-
@id = hash.extract(:volume)
|
|
409
|
-
@session = session
|
|
410
|
-
@file = file
|
|
411
|
-
raise(Volume::Error, %(unsupported Mclone volume format version "#{version}")) unless hash.extract(:mclone) == VERSION
|
|
412
|
-
hash.dig(:tasks)&.each { |h| session.tasks << (@loaded_tasks << Task.restore(@session, h)) }
|
|
413
|
-
self
|
|
414
|
-
end
|
|
415
|
-
|
|
416
|
-
#
|
|
417
|
-
def hash
|
|
418
|
-
id.hash
|
|
419
|
-
end
|
|
420
|
-
|
|
421
|
-
#
|
|
422
|
-
def eql?(other)
|
|
423
|
-
equal?(other) || id == other.id
|
|
424
|
-
end
|
|
425
|
-
|
|
426
|
-
#
|
|
427
|
-
def modified?
|
|
428
|
-
# Comparison against the original loaded tasks set allows to account for task removals
|
|
429
|
-
(ts = tasks).modified? || ts != @loaded_tasks
|
|
430
|
-
end
|
|
431
|
-
|
|
432
|
-
#
|
|
433
|
-
def commit!(force = false)
|
|
434
|
-
if force || @session.force? || modified?
|
|
435
|
-
# As a safeguard against malformed volume files generation, first write to a new file
|
|
436
|
-
# and rename it to a real volume file only in case of normal turn of events
|
|
437
|
-
_file = "#{file}~"
|
|
438
|
-
begin
|
|
439
|
-
open(_file, 'w') do |stream|
|
|
440
|
-
stream << JSON.pretty_generate(to_h)
|
|
441
|
-
tasks.commit!
|
|
442
|
-
end
|
|
443
|
-
FileUtils.mv(_file, file, force: true)
|
|
444
|
-
ensure
|
|
445
|
-
FileUtils.rm_f(_file)
|
|
446
|
-
end
|
|
447
|
-
end
|
|
448
|
-
self
|
|
449
|
-
end
|
|
450
|
-
|
|
451
|
-
#
|
|
452
|
-
def tasks
|
|
453
|
-
TaskSet.new(self).merge!(@session.tasks)
|
|
454
|
-
end
|
|
455
|
-
|
|
456
|
-
#
|
|
457
|
-
def to_h
|
|
458
|
-
{ mclone: VERSION, volume: id, tasks: tasks.collect { |task| task.to_h(self) } }
|
|
459
|
-
end
|
|
460
|
-
|
|
461
|
-
# Volume-bound set of tasks belonging to the specific volume
|
|
462
|
-
class TaskSet < Mclone::TaskSet
|
|
463
|
-
|
|
464
|
-
def initialize(volume)
|
|
465
|
-
@volume = volume
|
|
466
|
-
super()
|
|
467
|
-
end
|
|
468
|
-
|
|
469
|
-
# Accept only the tasks referencing the volume as either source or destination
|
|
470
|
-
def <<(task)
|
|
471
|
-
task.source_id == @volume.id || task.destination_id == @volume.id ? super : task
|
|
472
|
-
end
|
|
473
|
-
|
|
474
|
-
end
|
|
475
|
-
end
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
#
|
|
479
|
-
class VolumeSet < ObjectSet
|
|
480
|
-
|
|
481
|
-
alias volume object
|
|
482
|
-
|
|
483
|
-
#
|
|
484
|
-
def resolve(id)
|
|
485
|
-
case (ids = super).size
|
|
486
|
-
when 0 then raise(Volume::Error, %(no volume matching "#{id}" pattern found))
|
|
487
|
-
when 1 then ids.first
|
|
488
|
-
else raise(Volume::Error, %(ambiguous "#{id}" pattern: two or more volumes match))
|
|
489
|
-
end
|
|
490
|
-
end
|
|
491
|
-
|
|
492
|
-
end
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
#
|
|
496
|
-
class Session
|
|
497
|
-
|
|
498
|
-
#
|
|
499
|
-
class Error < Mclone::Error
|
|
500
|
-
|
|
501
|
-
end
|
|
502
|
-
|
|
503
|
-
#
|
|
504
|
-
attr_reader :volumes
|
|
505
|
-
|
|
506
|
-
#
|
|
507
|
-
def simulate?
|
|
508
|
-
@simulate == true
|
|
509
|
-
end
|
|
510
|
-
|
|
511
|
-
#
|
|
512
|
-
def verbose?
|
|
513
|
-
@verbose == true
|
|
514
|
-
end
|
|
515
|
-
|
|
516
|
-
#
|
|
517
|
-
def force?
|
|
518
|
-
@force == true
|
|
519
|
-
end
|
|
520
|
-
|
|
521
|
-
#
|
|
522
|
-
attr_writer :simulate, :verbose, :force
|
|
523
|
-
|
|
524
|
-
#
|
|
525
|
-
attr_reader :tasks
|
|
526
|
-
|
|
527
|
-
#
|
|
528
|
-
def initialize
|
|
529
|
-
@volumes = VolumeSet.new
|
|
530
|
-
@tasks = SessionTaskSet.new(self)
|
|
531
|
-
end
|
|
532
|
-
|
|
533
|
-
#
|
|
534
|
-
def format_volume!(dir)
|
|
535
|
-
mclone = File.join(dir, Volume::FILE)
|
|
536
|
-
raise(Session::Error, %(refuse to overwrite existing Mclone volume file "#{mclone}")) if File.exist?(mclone) && !force?
|
|
537
|
-
volumes << (volume = Volume.new(self, mclone))
|
|
538
|
-
volume.commit!(true) unless simulate? # Force creation of a new (empty) volume
|
|
539
|
-
self
|
|
540
|
-
end
|
|
541
|
-
|
|
542
|
-
#
|
|
543
|
-
def restore_volume!(dir)
|
|
544
|
-
volumes << Volume.restore(self, File.join(dir, Volume::FILE))
|
|
545
|
-
self
|
|
546
|
-
end
|
|
547
|
-
|
|
548
|
-
#
|
|
549
|
-
def restore_volumes!
|
|
550
|
-
(Mclone.environment_mounts + Mclone.system_mounts + [ENV['HOME']]).each { |dir| restore_volume!(dir) rescue Errno::ENOENT }
|
|
551
|
-
self
|
|
552
|
-
end
|
|
553
|
-
|
|
554
|
-
#
|
|
555
|
-
def delete_volume!(id)
|
|
556
|
-
volume = volumes.volume(id = volumes.resolve(id))
|
|
557
|
-
raise(Session::Error, %(refuse to delete non-empty Mclone volume file "#{volume.file}")) unless volume.tasks.empty? || force?
|
|
558
|
-
volumes >> volume
|
|
559
|
-
FileUtils.rm_f(volume.file) unless simulate?
|
|
560
|
-
self
|
|
561
|
-
end
|
|
562
|
-
|
|
563
|
-
#
|
|
564
|
-
def create_task!(mode, source, destination, **kws)
|
|
565
|
-
task = Task.new(self, mode, *locate(source), *locate(destination), **kws)
|
|
566
|
-
_task = tasks[task]
|
|
567
|
-
raise(Session::Error, %(refuse to overwrite existing task "#{_task.id}")) unless _task.nil? || force?
|
|
568
|
-
tasks << task
|
|
569
|
-
self
|
|
570
|
-
end
|
|
571
|
-
|
|
572
|
-
#
|
|
573
|
-
def modify_task!(id, mode: nil, include: nil, exclude: nil)
|
|
574
|
-
ts = tasks
|
|
575
|
-
task = ts.task(ts.resolve(id)).clone
|
|
576
|
-
task.mode = mode unless mode.nil?
|
|
577
|
-
task.include = include unless include.nil?
|
|
578
|
-
task.exclude = exclude unless exclude.nil?
|
|
579
|
-
tasks << task
|
|
580
|
-
self
|
|
581
|
-
end
|
|
582
|
-
|
|
583
|
-
#
|
|
584
|
-
def delete_task!(id)
|
|
585
|
-
tasks >> tasks.task(tasks.resolve(id))
|
|
586
|
-
self
|
|
587
|
-
end
|
|
588
|
-
|
|
589
|
-
#
|
|
590
|
-
def process_tasks!(*ids)
|
|
591
|
-
failed = false
|
|
592
|
-
intacts = intact_tasks
|
|
593
|
-
ids = intacts.collect(&:id) if ids.empty?
|
|
594
|
-
ids.collect { |id| intacts.task(intacts.resolve(id)) }.each do |task|
|
|
595
|
-
source_path = File.join(volumes.volume(task.source_id).root, task.source_root.nil? || task.source_root.empty? ? '' : task.source_root)
|
|
596
|
-
destination_path = File.join(volumes.volume(task.destination_id).root, task.destination_root.nil? || task.destination_root.empty? ? '' : task.destination_root)
|
|
597
|
-
args = [Mclone.rclone]
|
|
598
|
-
opts = [
|
|
599
|
-
'--config', Mclone.windows? ? 'NUL' : '/dev/null',
|
|
600
|
-
simulate? ? '--dry-run' : nil,
|
|
601
|
-
verbose? ? '--verbose' : nil,
|
|
602
|
-
verbose? ? '--progress' : nil
|
|
603
|
-
].compact
|
|
604
|
-
opts.append('--crypt-password', task.crypter_token) unless task.crypter_mode.nil?
|
|
605
|
-
case task.crypter_mode
|
|
606
|
-
when :encrypt then opts.append('--crypt-remote', destination_path)
|
|
607
|
-
when :decrypt then opts.append('--crypt-remote', source_path)
|
|
608
|
-
end
|
|
609
|
-
case task.mode
|
|
610
|
-
when :update then args.push('copy', '--update')
|
|
611
|
-
when :synchronize then args << 'sync'
|
|
612
|
-
when :copy then args << 'copy'
|
|
613
|
-
when :move then args << 'move'
|
|
614
|
-
end
|
|
615
|
-
opts.append('--filter', "- /#{Volume::FILE}")
|
|
616
|
-
opts.append('--filter', "- #{task.exclude}") unless task.exclude.nil? || task.exclude.empty?
|
|
617
|
-
opts.append('--filter', "+ #{task.include}") unless task.include.nil? || task.include.empty?
|
|
618
|
-
args.concat(opts)
|
|
619
|
-
case task.crypter_mode
|
|
620
|
-
when nil then args.append(source_path, destination_path)
|
|
621
|
-
when :encrypt then args.append(source_path, ':crypt:')
|
|
622
|
-
when :decrypt then args.append(':crypt:', destination_path)
|
|
623
|
-
end
|
|
624
|
-
$stdout << args.collect(&:escape).join(' ') << "\n" if verbose?
|
|
625
|
-
case system(*args)
|
|
626
|
-
when nil
|
|
627
|
-
$stderr << %(failed to execute "#{args.first}") << "\n" if verbose?
|
|
628
|
-
failed = true
|
|
629
|
-
when false
|
|
630
|
-
$stderr << %(Rclone exited with status #{$?.to_i}) << "\n" if verbose?
|
|
631
|
-
failed = true
|
|
632
|
-
end
|
|
633
|
-
end
|
|
634
|
-
raise(Session::Error, "Rclone execution failure(s)") if failed
|
|
635
|
-
self
|
|
636
|
-
end
|
|
637
|
-
|
|
638
|
-
# Collect all tasks from all loaded volumes which are ready to be executed
|
|
639
|
-
def intact_tasks
|
|
640
|
-
IntactTaskSet.new(self).merge!(tasks)
|
|
641
|
-
end
|
|
642
|
-
|
|
643
|
-
#
|
|
644
|
-
private def locate(path)
|
|
645
|
-
path = File.realpath(path)
|
|
646
|
-
x = volumes.each.collect { |v| Regexp.new(%!^#{v.root}/?(.*)!, Mclone.windows? ? Regexp::IGNORECASE : nil) =~ path ? [v.root, v.id, $1] : nil }.compact
|
|
647
|
-
if x.empty?
|
|
648
|
-
raise(Session::Error, %(path "#{path}" does not belong to a loaded Mclone volume))
|
|
649
|
-
else
|
|
650
|
-
root, volume, path = x.sort { |a,b| a.first.size <=> b.first.size}.last
|
|
651
|
-
[volume, path]
|
|
652
|
-
end
|
|
653
|
-
end
|
|
654
|
-
|
|
655
|
-
#
|
|
656
|
-
def commit!
|
|
657
|
-
volumes.each { |v| v.commit!(force?) } unless simulate?
|
|
658
|
-
self
|
|
659
|
-
end
|
|
660
|
-
|
|
661
|
-
#
|
|
662
|
-
class SessionTaskSet < Mclone::TaskSet
|
|
663
|
-
|
|
664
|
-
def initialize(session)
|
|
665
|
-
@session = session
|
|
666
|
-
super()
|
|
667
|
-
end
|
|
668
|
-
|
|
669
|
-
end
|
|
670
|
-
|
|
671
|
-
# Session-bound set of intact tasks for which both source and destination volumes are loaded
|
|
672
|
-
class IntactTaskSet < SessionTaskSet
|
|
673
|
-
|
|
674
|
-
# Accept only intact tasks for which both source and destination volumes are loaded
|
|
675
|
-
def <<(task)
|
|
676
|
-
@session.volumes.volume(task.source_id).nil? || @session.volumes.volume(task.destination_id).nil? ? task : super
|
|
677
|
-
end
|
|
678
|
-
|
|
679
|
-
end
|
|
680
|
-
|
|
681
|
-
end
|
|
682
|
-
|
|
683
|
-
#
|
|
684
|
-
def self.rclone
|
|
685
|
-
@@rclone ||= (rclone = ENV['RCLONE']).nil? ? 'rclone' : rclone
|
|
686
|
-
end
|
|
687
|
-
|
|
688
|
-
# Return true if run in the real Windows environment (e.g. not in real *NIX or various emulation layers such as MSYS, Cygwin etc.)
|
|
689
|
-
def self.windows?
|
|
690
|
-
@@windows ||= /^(mingw)/.match?(RbConfig::CONFIG['target_os']) # RubyInstaller's MRI, other MinGW-build MRI
|
|
691
|
-
end
|
|
692
|
-
|
|
693
|
-
# Match OS-specific system mount points (/dev /proc etc.) which normally should be omitted when scanning for Mclone voulmes
|
|
694
|
-
UNIX_SYSTEM_MOUNTS = %r!^/(dev|sys|proc
|
|
695
|
-
|
|
696
|
-
# TODO handle Windows variants
|
|
697
|
-
# Specify OS-specific path name list separator (such as in the $PATH environment variable)
|
|
698
|
-
PATH_LIST_SEPARATOR = windows? ? ';' : ':'
|
|
699
|
-
|
|
700
|
-
# Return list of live user-provided mounts (mount points on *NIX and disk drives on Windows) which may contain Mclone volumes
|
|
701
|
-
# Look for the $MCLONE_PATH environment variable
|
|
702
|
-
def self.environment_mounts
|
|
703
|
-
ENV['MCLONE_PATH'].split(PATH_LIST_SEPARATOR).collect { |path| File.directory?(path) ? path : nil }.compact rescue []
|
|
704
|
-
end
|
|
705
|
-
# Return list of live system-managed mounts (mount points on *NIX and disk drives on Windows) which may contain Mclone volumes
|
|
706
|
-
case RbConfig::CONFIG['target_os']
|
|
707
|
-
when 'linux'
|
|
708
|
-
# Linux OS
|
|
709
|
-
def self.system_mounts
|
|
710
|
-
# Query /proc for currently mounted file systems
|
|
711
|
-
IO.readlines('/proc/self/mountstats').collect do |line|
|
|
712
|
-
mount = line.split[4]
|
|
713
|
-
UNIX_SYSTEM_MOUNTS.match?(mount) || !File.directory?(mount) ? nil : mount
|
|
714
|
-
end.compact
|
|
715
|
-
end
|
|
716
|
-
# TODO handle Windows variants
|
|
717
|
-
when /^mingw/ # RubyInstaller's MRI
|
|
718
|
-
module Kernel32
|
|
719
|
-
require 'fiddle'
|
|
720
|
-
require 'fiddle/types'
|
|
721
|
-
require 'fiddle/import'
|
|
722
|
-
extend Fiddle::Importer
|
|
723
|
-
dlload('kernel32')
|
|
724
|
-
include Fiddle::Win32Types
|
|
725
|
-
extern 'DWORD WINAPI GetLogicalDrives()'
|
|
726
|
-
end
|
|
727
|
-
def self.system_mounts
|
|
728
|
-
mounts = []
|
|
729
|
-
mask = Kernel32.GetLogicalDrives
|
|
730
|
-
('A'..'Z').each do |x|
|
|
731
|
-
mounts << "#{x}:" if mask & 1 == 1
|
|
732
|
-
mask >>= 1
|
|
733
|
-
end
|
|
734
|
-
mounts
|
|
735
|
-
end
|
|
736
|
-
else
|
|
737
|
-
# Generic *NIX-like OS, including Cygwin & MSYS(2)
|
|
738
|
-
def self.system_mounts
|
|
739
|
-
# Use $(mount) system utility to obtain currently mounted file systems
|
|
740
|
-
%x(mount).split("\n").collect do |line|
|
|
741
|
-
mount = line.split[2]
|
|
742
|
-
UNIX_SYSTEM_MOUNTS.match?(mount) || !File.directory?(mount) ? nil : mount
|
|
743
|
-
end.compact
|
|
744
|
-
end
|
|
745
|
-
end
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
require 'date'
|
|
5
|
+
require 'json'
|
|
6
|
+
require 'open3'
|
|
7
|
+
require 'fileutils'
|
|
8
|
+
require 'shellwords'
|
|
9
|
+
require 'securerandom'
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
#
|
|
13
|
+
module Mclone
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
VERSION = '0.2.1'
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
#
|
|
20
|
+
class Error < StandardError; end
|
|
21
|
+
|
|
22
|
+
#
|
|
23
|
+
module Refinements
|
|
24
|
+
|
|
25
|
+
refine ::Hash do
|
|
26
|
+
# Same as #dig but raises KeyError exception on any non-existent key
|
|
27
|
+
def extract(*args)
|
|
28
|
+
case args.size
|
|
29
|
+
when 0 then raise(KeyError, 'non-empty key sequence expected')
|
|
30
|
+
when 1 then fetch(args.first)
|
|
31
|
+
else fetch(args.shift).extract(*args)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
refine ::Array do
|
|
37
|
+
# Return a list of items which fully or partially match the specified pattern
|
|
38
|
+
def resolve(partial)
|
|
39
|
+
rx = Regexp.new(partial)
|
|
40
|
+
collect { |item| rx.match?(item.to_s) ? item : nil }.compact
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
refine ::String do
|
|
45
|
+
def escape
|
|
46
|
+
Mclone.windows? && %r![^\w\-\=\\\/:]!.match?(self) ? %("#{self}") : shellescape
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
using Refinements
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# Two-way mapping between an object and its ID
|
|
56
|
+
class ObjectSet
|
|
57
|
+
|
|
58
|
+
include Enumerable
|
|
59
|
+
|
|
60
|
+
#
|
|
61
|
+
def each_id(&code)
|
|
62
|
+
@ids.each_key(&code)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
#
|
|
66
|
+
def each(&code)
|
|
67
|
+
@objects.each_value(&code)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
#
|
|
71
|
+
def empty?
|
|
72
|
+
@objects.empty?
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
#
|
|
76
|
+
def size
|
|
77
|
+
@objects.size
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def initialize
|
|
81
|
+
@ids = {} # { id => object }
|
|
82
|
+
@objects = {} # { object => object }
|
|
83
|
+
@modified = false
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
attr_reader :objects; protected :objects
|
|
87
|
+
|
|
88
|
+
#
|
|
89
|
+
def hash
|
|
90
|
+
@objects.hash
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
#
|
|
94
|
+
def eql?(other)
|
|
95
|
+
equal?(other) || objects == other.objects
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
alias == eql?
|
|
99
|
+
|
|
100
|
+
# Return ID of the object considered equal to the specified obj or nil
|
|
101
|
+
def id(obj)
|
|
102
|
+
@objects[obj]&.id
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Return object with specified ID or nil
|
|
106
|
+
def object(id)
|
|
107
|
+
@ids[id]
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Return object considered equal to obj or nil
|
|
111
|
+
def [](obj)
|
|
112
|
+
@objects[obj]
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
#
|
|
116
|
+
def modified?
|
|
117
|
+
@modified
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def commit!
|
|
121
|
+
@modified = false
|
|
122
|
+
self
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Unregister an object considered equal to the specified obj and return true if object has been actually removed
|
|
126
|
+
private def forget(obj)
|
|
127
|
+
!@ids.delete(@objects.delete(obj)&.id).nil?
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# Return a list of registered IDs (fully or partially) matching the specified pattern
|
|
131
|
+
def resolve(pattern)
|
|
132
|
+
each_id.to_a.resolve(pattern)
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Either add brand new object or replace existing one equal to the specified object
|
|
136
|
+
def <<(obj)
|
|
137
|
+
forget(obj)
|
|
138
|
+
@objects[obj] = @ids[obj.id] = obj
|
|
139
|
+
@modified = true
|
|
140
|
+
obj
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Remove object considered equal to the specified obj
|
|
144
|
+
def >>(obj)
|
|
145
|
+
@modified = true if (status = forget(obj))
|
|
146
|
+
status
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# Add all tasks from enumerable
|
|
150
|
+
def merge!(objs)
|
|
151
|
+
objs.each { |x| self << x }
|
|
152
|
+
self
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
#
|
|
159
|
+
class Task
|
|
160
|
+
|
|
161
|
+
#
|
|
162
|
+
class Error < Mclone::Error
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
#
|
|
166
|
+
attr_reader :id
|
|
167
|
+
|
|
168
|
+
#
|
|
169
|
+
attr_reader :source_id, :destination_id
|
|
170
|
+
|
|
171
|
+
#
|
|
172
|
+
attr_reader :source_root, :destination_root
|
|
173
|
+
|
|
174
|
+
#
|
|
175
|
+
attr_reader :mtime
|
|
176
|
+
|
|
177
|
+
#
|
|
178
|
+
attr_reader :mode
|
|
179
|
+
|
|
180
|
+
#
|
|
181
|
+
attr_reader :include, :exclude
|
|
182
|
+
|
|
183
|
+
#
|
|
184
|
+
attr_reader :crypter_mode
|
|
185
|
+
|
|
186
|
+
def hash
|
|
187
|
+
@hash ||= source_id.hash ^ destination_id.hash ^ source_root.hash ^ destination_root.hash
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def eql?(other)
|
|
191
|
+
equal?(other) || (
|
|
192
|
+
source_id == other.source_id &&
|
|
193
|
+
destination_id == other.destination_id &&
|
|
194
|
+
source_root == other.source_root &&
|
|
195
|
+
destination_root == other.destination_root
|
|
196
|
+
)
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
alias == eql?
|
|
200
|
+
|
|
201
|
+
#
|
|
202
|
+
def initialize(session, mode, source_id, source_root, destination_id, destination_root, include: nil, exclude: nil, crypter_mode: nil, crypter_token: nil, crypter_password: nil)
|
|
203
|
+
@touch = false # Indicates that the time stamp should be updated whenever state of self is altered
|
|
204
|
+
@session = session
|
|
205
|
+
@id = SecureRandom.hex(4)
|
|
206
|
+
@source_id = source_id
|
|
207
|
+
@destination_id = destination_id
|
|
208
|
+
@source_root = source_root
|
|
209
|
+
@destination_root = destination_root
|
|
210
|
+
self.mode = mode
|
|
211
|
+
self.include = include
|
|
212
|
+
self.exclude = exclude
|
|
213
|
+
set_crypter_mode crypter_mode
|
|
214
|
+
unless crypter_mode.nil?
|
|
215
|
+
raise(Task::Error, %(either Rclone crypt token or plain text password is expected, not both)) if !crypter_token.nil? && !crypter_password.nil?
|
|
216
|
+
@assigned_token = register_crypter_token crypter_token
|
|
217
|
+
@assigned_password = crypter_password
|
|
218
|
+
end
|
|
219
|
+
ensure
|
|
220
|
+
@touch = true
|
|
221
|
+
touch!
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
CRYPTER_MODES = %i[encrypt decrypt].freeze
|
|
225
|
+
|
|
226
|
+
@@crypter_tokens = {}
|
|
227
|
+
|
|
228
|
+
private def set_crypter_mode(mode)
|
|
229
|
+
@crypter_mode = mode.nil? ? nil : (CRYPTER_MODES.include?(mode = mode.intern) ? mode : raise(Task::Error, %(unknown crypt mode "#{mode}")))
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
private def register_crypter_token(token)
|
|
233
|
+
unless token.nil?
|
|
234
|
+
@@crypter_tokens[id] = @@crypter_tokens[id].nil? ? token : raise(Task::Error, %(attempt to re-register token for task "#{id}"))
|
|
235
|
+
end
|
|
236
|
+
token
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
# Lazily determine the crypt token from either assigned values or the token repository
|
|
240
|
+
def crypter_token
|
|
241
|
+
# Locally assigned token takes precedence over the repository's
|
|
242
|
+
unless @assigned_token.nil?
|
|
243
|
+
@@crypter_tokens[id] = @assigned_token unless @@crypter_tokens[id].nil? # Assign repository entry with this local token if not yet assigned
|
|
244
|
+
@assigned_token
|
|
245
|
+
else
|
|
246
|
+
if @@crypter_tokens[id].nil?
|
|
247
|
+
# If token is neither locally assigned nor in repository, try to construct it from the user-supplied password
|
|
248
|
+
# If a user-supplied password is omitted, create a new randomly generated password
|
|
249
|
+
args = [Mclone.rclone, 'obscure', @assigned_password.nil? ? SecureRandom.alphanumeric(16) : @assigned_password]
|
|
250
|
+
$stdout << args.collect(&:escape).join(' ') << "\n" if @session.verbose?
|
|
251
|
+
stdout, status = Open3.capture2(*args)
|
|
252
|
+
raise(Task::Error, %(Rclone execution failure)) unless status.success?
|
|
253
|
+
@@crypter_tokens[id] = stdout.strip
|
|
254
|
+
else
|
|
255
|
+
@@crypter_tokens[id]
|
|
256
|
+
end
|
|
257
|
+
end
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
#
|
|
261
|
+
MODES = %i[update synchronize copy move].freeze
|
|
262
|
+
|
|
263
|
+
#
|
|
264
|
+
def mode=(mode)
|
|
265
|
+
@mode = MODES.include?(mode = mode.intern) ? mode : raise(Task::Error, %(unknown mode "#{mode}"))
|
|
266
|
+
touch!
|
|
267
|
+
mode
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
#
|
|
271
|
+
def include=(mask)
|
|
272
|
+
@include = mask.nil? || mask == '-' ? nil : mask # TODO: verify mask
|
|
273
|
+
touch!
|
|
274
|
+
mask
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
#
|
|
278
|
+
def exclude=(mask)
|
|
279
|
+
@exclude = mask.nil? || mask == '-' ? nil : mask # TODO: verify mask
|
|
280
|
+
touch!
|
|
281
|
+
mask
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
#
|
|
285
|
+
def self.restore(session, hash)
|
|
286
|
+
obj = allocate
|
|
287
|
+
obj.send(:from_h, session, hash)
|
|
288
|
+
obj
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
#
|
|
292
|
+
private def from_h(session, hash)
|
|
293
|
+
@session = session
|
|
294
|
+
@touch = false
|
|
295
|
+
@id = hash.extract(:task)
|
|
296
|
+
@mtime = DateTime.parse(hash.extract(:mtime)) rescue DateTime.now # Deleting mtime entry from json can be used to modify data out of mclone
|
|
297
|
+
@source_id = hash.extract(:source, :volume)
|
|
298
|
+
@destination_id = hash.extract(:destination, :volume)
|
|
299
|
+
@source_root = hash.dig(:source, :root)
|
|
300
|
+
@destination_root = hash.dig(:destination, :root)
|
|
301
|
+
self.mode = hash.extract(:mode)
|
|
302
|
+
self.include = hash.dig(:include)
|
|
303
|
+
self.exclude = hash.dig(:exclude)
|
|
304
|
+
set_crypter_mode hash.dig(:crypter, :mode)
|
|
305
|
+
@assigned_token = register_crypter_token(hash.dig(:crypter, :token)) unless crypter_mode.nil?
|
|
306
|
+
ensure
|
|
307
|
+
@touch = true
|
|
308
|
+
end
|
|
309
|
+
|
|
310
|
+
#
|
|
311
|
+
def to_h(volume)
|
|
312
|
+
hash = {
|
|
313
|
+
task: id,
|
|
314
|
+
mode: mode,
|
|
315
|
+
mtime: mtime,
|
|
316
|
+
source: { volume: source_id },
|
|
317
|
+
destination: { volume: destination_id }
|
|
318
|
+
}
|
|
319
|
+
hash[:source][:root] = source_root unless source_root.nil? || source_root.empty?
|
|
320
|
+
hash[:destination][:root] = destination_root unless destination_root.nil? || destination_root.empty?
|
|
321
|
+
hash[:include] = include unless include.nil?
|
|
322
|
+
hash[:exclude] = exclude unless exclude.nil?
|
|
323
|
+
unless crypter_mode.nil?
|
|
324
|
+
crypter = hash[:crypter] = { mode: crypter_mode }
|
|
325
|
+
# Make sure the token won't get into the encrypted volume's task
|
|
326
|
+
crypter[:token] = crypter_token if (crypter_mode == :encrypt && source_id == volume.id) || (crypter_mode == :decrypt && destination_id == volume.id)
|
|
327
|
+
end
|
|
328
|
+
hash
|
|
329
|
+
end
|
|
330
|
+
|
|
331
|
+
#
|
|
332
|
+
def touch!
|
|
333
|
+
@mtime = DateTime.now if @touch
|
|
334
|
+
self
|
|
335
|
+
end
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
#
|
|
340
|
+
class TaskSet < ObjectSet
|
|
341
|
+
|
|
342
|
+
alias task object
|
|
343
|
+
|
|
344
|
+
# Add new task or replace existing one with outdated timestamp
|
|
345
|
+
def <<(task)
|
|
346
|
+
t = self[task]
|
|
347
|
+
super if t.nil? || (!t.nil? && t.mtime < task.mtime)
|
|
348
|
+
task
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
#
|
|
352
|
+
def resolve(id)
|
|
353
|
+
case (ids = super).size
|
|
354
|
+
when 0 then raise(Task::Error, %(no task matching "#{id}" pattern found))
|
|
355
|
+
when 1 then ids.first
|
|
356
|
+
else raise(Task::Error, %(ambiguous "#{id}" pattern: two or more tasks match))
|
|
357
|
+
end
|
|
358
|
+
end
|
|
359
|
+
|
|
360
|
+
end
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
#
|
|
364
|
+
class Volume
|
|
365
|
+
|
|
366
|
+
#
|
|
367
|
+
class Error < Mclone::Error
|
|
368
|
+
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
#
|
|
372
|
+
VERSION = 0
|
|
373
|
+
|
|
374
|
+
#
|
|
375
|
+
FILE = '.mclone'
|
|
376
|
+
|
|
377
|
+
#
|
|
378
|
+
attr_reader :id
|
|
379
|
+
|
|
380
|
+
#
|
|
381
|
+
attr_reader :file
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
#
|
|
385
|
+
def root
|
|
386
|
+
@root ||= File.realpath(File.dirname(file))
|
|
387
|
+
end
|
|
388
|
+
|
|
389
|
+
#
|
|
390
|
+
def initialize(session, file)
|
|
391
|
+
@loaded_tasks = ObjectSet.new
|
|
392
|
+
@id = SecureRandom.hex(4)
|
|
393
|
+
@session = session
|
|
394
|
+
@file = file
|
|
395
|
+
end
|
|
396
|
+
|
|
397
|
+
#
|
|
398
|
+
def self.restore(session, file)
|
|
399
|
+
obj = allocate
|
|
400
|
+
obj.send(:from_file, session, file)
|
|
401
|
+
obj
|
|
402
|
+
end
|
|
403
|
+
|
|
404
|
+
#
|
|
405
|
+
private def from_file(session, file)
|
|
406
|
+
hash = JSON.parse(IO.read(file), symbolize_names: true)
|
|
407
|
+
@loaded_tasks = ObjectSet.new
|
|
408
|
+
@id = hash.extract(:volume)
|
|
409
|
+
@session = session
|
|
410
|
+
@file = file
|
|
411
|
+
raise(Volume::Error, %(unsupported Mclone volume format version "#{version}")) unless hash.extract(:mclone) == VERSION
|
|
412
|
+
hash.dig(:tasks)&.each { |h| session.tasks << (@loaded_tasks << Task.restore(@session, h)) }
|
|
413
|
+
self
|
|
414
|
+
end
|
|
415
|
+
|
|
416
|
+
#
|
|
417
|
+
def hash
|
|
418
|
+
id.hash
|
|
419
|
+
end
|
|
420
|
+
|
|
421
|
+
#
|
|
422
|
+
def eql?(other)
|
|
423
|
+
equal?(other) || id == other.id
|
|
424
|
+
end
|
|
425
|
+
|
|
426
|
+
#
|
|
427
|
+
def modified?
|
|
428
|
+
# Comparison against the original loaded tasks set allows to account for task removals
|
|
429
|
+
(ts = tasks).modified? || ts != @loaded_tasks
|
|
430
|
+
end
|
|
431
|
+
|
|
432
|
+
#
|
|
433
|
+
def commit!(force = false)
|
|
434
|
+
if force || @session.force? || modified?
|
|
435
|
+
# As a safeguard against malformed volume files generation, first write to a new file
|
|
436
|
+
# and rename it to a real volume file only in case of normal turn of events
|
|
437
|
+
_file = "#{file}~"
|
|
438
|
+
begin
|
|
439
|
+
open(_file, 'w') do |stream|
|
|
440
|
+
stream << JSON.pretty_generate(to_h)
|
|
441
|
+
tasks.commit!
|
|
442
|
+
end
|
|
443
|
+
FileUtils.mv(_file, file, force: true)
|
|
444
|
+
ensure
|
|
445
|
+
FileUtils.rm_f(_file)
|
|
446
|
+
end
|
|
447
|
+
end
|
|
448
|
+
self
|
|
449
|
+
end
|
|
450
|
+
|
|
451
|
+
#
|
|
452
|
+
def tasks
|
|
453
|
+
TaskSet.new(self).merge!(@session.tasks)
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
#
|
|
457
|
+
def to_h
|
|
458
|
+
{ mclone: VERSION, volume: id, tasks: tasks.collect { |task| task.to_h(self) } }
|
|
459
|
+
end
|
|
460
|
+
|
|
461
|
+
# Volume-bound set of tasks belonging to the specific volume
|
|
462
|
+
class TaskSet < Mclone::TaskSet
|
|
463
|
+
|
|
464
|
+
def initialize(volume)
|
|
465
|
+
@volume = volume
|
|
466
|
+
super()
|
|
467
|
+
end
|
|
468
|
+
|
|
469
|
+
# Accept only the tasks referencing the volume as either source or destination
|
|
470
|
+
def <<(task)
|
|
471
|
+
task.source_id == @volume.id || task.destination_id == @volume.id ? super : task
|
|
472
|
+
end
|
|
473
|
+
|
|
474
|
+
end
|
|
475
|
+
end
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
#
|
|
479
|
+
class VolumeSet < ObjectSet
|
|
480
|
+
|
|
481
|
+
alias volume object
|
|
482
|
+
|
|
483
|
+
#
|
|
484
|
+
def resolve(id)
|
|
485
|
+
case (ids = super).size
|
|
486
|
+
when 0 then raise(Volume::Error, %(no volume matching "#{id}" pattern found))
|
|
487
|
+
when 1 then ids.first
|
|
488
|
+
else raise(Volume::Error, %(ambiguous "#{id}" pattern: two or more volumes match))
|
|
489
|
+
end
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
end
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
#
|
|
496
|
+
class Session
|
|
497
|
+
|
|
498
|
+
#
|
|
499
|
+
class Error < Mclone::Error
|
|
500
|
+
|
|
501
|
+
end
|
|
502
|
+
|
|
503
|
+
#
|
|
504
|
+
attr_reader :volumes
|
|
505
|
+
|
|
506
|
+
#
|
|
507
|
+
def simulate?
|
|
508
|
+
@simulate == true
|
|
509
|
+
end
|
|
510
|
+
|
|
511
|
+
#
|
|
512
|
+
def verbose?
|
|
513
|
+
@verbose == true
|
|
514
|
+
end
|
|
515
|
+
|
|
516
|
+
#
|
|
517
|
+
def force?
|
|
518
|
+
@force == true
|
|
519
|
+
end
|
|
520
|
+
|
|
521
|
+
#
|
|
522
|
+
attr_writer :simulate, :verbose, :force
|
|
523
|
+
|
|
524
|
+
#
|
|
525
|
+
attr_reader :tasks
|
|
526
|
+
|
|
527
|
+
#
|
|
528
|
+
def initialize
|
|
529
|
+
@volumes = VolumeSet.new
|
|
530
|
+
@tasks = SessionTaskSet.new(self)
|
|
531
|
+
end
|
|
532
|
+
|
|
533
|
+
#
|
|
534
|
+
def format_volume!(dir)
|
|
535
|
+
mclone = File.join(dir, Volume::FILE)
|
|
536
|
+
raise(Session::Error, %(refuse to overwrite existing Mclone volume file "#{mclone}")) if File.exist?(mclone) && !force?
|
|
537
|
+
volumes << (volume = Volume.new(self, mclone))
|
|
538
|
+
volume.commit!(true) unless simulate? # Force creation of a new (empty) volume
|
|
539
|
+
self
|
|
540
|
+
end
|
|
541
|
+
|
|
542
|
+
#
|
|
543
|
+
def restore_volume!(dir)
|
|
544
|
+
volumes << Volume.restore(self, File.join(dir, Volume::FILE))
|
|
545
|
+
self
|
|
546
|
+
end
|
|
547
|
+
|
|
548
|
+
#
|
|
549
|
+
def restore_volumes!
|
|
550
|
+
(Mclone.environment_mounts + Mclone.system_mounts + [ENV['HOME']]).each { |dir| restore_volume!(dir) rescue Errno::ENOENT }
|
|
551
|
+
self
|
|
552
|
+
end
|
|
553
|
+
|
|
554
|
+
#
|
|
555
|
+
def delete_volume!(id)
|
|
556
|
+
volume = volumes.volume(id = volumes.resolve(id))
|
|
557
|
+
raise(Session::Error, %(refuse to delete non-empty Mclone volume file "#{volume.file}")) unless volume.tasks.empty? || force?
|
|
558
|
+
volumes >> volume
|
|
559
|
+
FileUtils.rm_f(volume.file) unless simulate?
|
|
560
|
+
self
|
|
561
|
+
end
|
|
562
|
+
|
|
563
|
+
#
|
|
564
|
+
def create_task!(mode, source, destination, **kws)
|
|
565
|
+
task = Task.new(self, mode, *locate(source), *locate(destination), **kws)
|
|
566
|
+
_task = tasks[task]
|
|
567
|
+
raise(Session::Error, %(refuse to overwrite existing task "#{_task.id}")) unless _task.nil? || force?
|
|
568
|
+
tasks << task
|
|
569
|
+
self
|
|
570
|
+
end
|
|
571
|
+
|
|
572
|
+
#
|
|
573
|
+
def modify_task!(id, mode: nil, include: nil, exclude: nil)
|
|
574
|
+
ts = tasks
|
|
575
|
+
task = ts.task(ts.resolve(id)).clone
|
|
576
|
+
task.mode = mode unless mode.nil?
|
|
577
|
+
task.include = include unless include.nil?
|
|
578
|
+
task.exclude = exclude unless exclude.nil?
|
|
579
|
+
tasks << task
|
|
580
|
+
self
|
|
581
|
+
end
|
|
582
|
+
|
|
583
|
+
#
|
|
584
|
+
def delete_task!(id)
|
|
585
|
+
tasks >> tasks.task(tasks.resolve(id))
|
|
586
|
+
self
|
|
587
|
+
end
|
|
588
|
+
|
|
589
|
+
#
|
|
590
|
+
def process_tasks!(*ids)
|
|
591
|
+
failed = false
|
|
592
|
+
intacts = intact_tasks
|
|
593
|
+
ids = intacts.collect(&:id) if ids.empty?
|
|
594
|
+
ids.collect { |id| intacts.task(intacts.resolve(id)) }.each do |task|
|
|
595
|
+
source_path = File.join(volumes.volume(task.source_id).root, task.source_root.nil? || task.source_root.empty? ? '' : task.source_root)
|
|
596
|
+
destination_path = File.join(volumes.volume(task.destination_id).root, task.destination_root.nil? || task.destination_root.empty? ? '' : task.destination_root)
|
|
597
|
+
args = [Mclone.rclone]
|
|
598
|
+
opts = [
|
|
599
|
+
'--config', Mclone.windows? ? 'NUL' : '/dev/null',
|
|
600
|
+
simulate? ? '--dry-run' : nil,
|
|
601
|
+
verbose? ? '--verbose' : nil,
|
|
602
|
+
verbose? ? '--progress' : nil
|
|
603
|
+
].compact
|
|
604
|
+
opts.append('--crypt-password', task.crypter_token) unless task.crypter_mode.nil?
|
|
605
|
+
case task.crypter_mode
|
|
606
|
+
when :encrypt then opts.append('--crypt-remote', destination_path)
|
|
607
|
+
when :decrypt then opts.append('--crypt-remote', source_path)
|
|
608
|
+
end
|
|
609
|
+
case task.mode
|
|
610
|
+
when :update then args.push('copy', '--update')
|
|
611
|
+
when :synchronize then args << 'sync'
|
|
612
|
+
when :copy then args << 'copy'
|
|
613
|
+
when :move then args << 'move'
|
|
614
|
+
end
|
|
615
|
+
opts.append('--filter', "- /#{Volume::FILE}")
|
|
616
|
+
opts.append('--filter', "- #{task.exclude}") unless task.exclude.nil? || task.exclude.empty?
|
|
617
|
+
opts.append('--filter', "+ #{task.include}") unless task.include.nil? || task.include.empty?
|
|
618
|
+
args.concat(opts)
|
|
619
|
+
case task.crypter_mode
|
|
620
|
+
when nil then args.append(source_path, destination_path)
|
|
621
|
+
when :encrypt then args.append(source_path, ':crypt:')
|
|
622
|
+
when :decrypt then args.append(':crypt:', destination_path)
|
|
623
|
+
end
|
|
624
|
+
$stdout << args.collect(&:escape).join(' ') << "\n" if verbose?
|
|
625
|
+
case system(*args)
|
|
626
|
+
when nil
|
|
627
|
+
$stderr << %(failed to execute "#{args.first}") << "\n" if verbose?
|
|
628
|
+
failed = true
|
|
629
|
+
when false
|
|
630
|
+
$stderr << %(Rclone exited with status #{$?.to_i}) << "\n" if verbose?
|
|
631
|
+
failed = true
|
|
632
|
+
end
|
|
633
|
+
end
|
|
634
|
+
raise(Session::Error, "Rclone execution failure(s)") if failed
|
|
635
|
+
self
|
|
636
|
+
end
|
|
637
|
+
|
|
638
|
+
# Collect all tasks from all loaded volumes which are ready to be executed
|
|
639
|
+
def intact_tasks
|
|
640
|
+
IntactTaskSet.new(self).merge!(tasks)
|
|
641
|
+
end
|
|
642
|
+
|
|
643
|
+
#
|
|
644
|
+
private def locate(path)
|
|
645
|
+
path = File.realpath(path)
|
|
646
|
+
x = volumes.each.collect { |v| Regexp.new(%!^#{v.root}/?(.*)!, Mclone.windows? ? Regexp::IGNORECASE : nil) =~ path ? [v.root, v.id, $1] : nil }.compact
|
|
647
|
+
if x.empty?
|
|
648
|
+
raise(Session::Error, %(path "#{path}" does not belong to a loaded Mclone volume))
|
|
649
|
+
else
|
|
650
|
+
root, volume, path = x.sort { |a,b| a.first.size <=> b.first.size}.last
|
|
651
|
+
[volume, path]
|
|
652
|
+
end
|
|
653
|
+
end
|
|
654
|
+
|
|
655
|
+
#
|
|
656
|
+
def commit!
|
|
657
|
+
volumes.each { |v| v.commit!(force?) } unless simulate?
|
|
658
|
+
self
|
|
659
|
+
end
|
|
660
|
+
|
|
661
|
+
#
|
|
662
|
+
class SessionTaskSet < Mclone::TaskSet
|
|
663
|
+
|
|
664
|
+
def initialize(session)
|
|
665
|
+
@session = session
|
|
666
|
+
super()
|
|
667
|
+
end
|
|
668
|
+
|
|
669
|
+
end
|
|
670
|
+
|
|
671
|
+
# Session-bound set of intact tasks for which both source and destination volumes are loaded
|
|
672
|
+
class IntactTaskSet < SessionTaskSet
|
|
673
|
+
|
|
674
|
+
# Accept only intact tasks for which both source and destination volumes are loaded
|
|
675
|
+
def <<(task)
|
|
676
|
+
@session.volumes.volume(task.source_id).nil? || @session.volumes.volume(task.destination_id).nil? ? task : super
|
|
677
|
+
end
|
|
678
|
+
|
|
679
|
+
end
|
|
680
|
+
|
|
681
|
+
end
|
|
682
|
+
|
|
683
|
+
#
|
|
684
|
+
def self.rclone
|
|
685
|
+
@@rclone ||= (rclone = ENV['RCLONE']).nil? ? 'rclone' : rclone
|
|
686
|
+
end
|
|
687
|
+
|
|
688
|
+
# Return true if run in the real Windows environment (e.g. not in real *NIX or various emulation layers such as MSYS, Cygwin etc.)
|
|
689
|
+
def self.windows?
|
|
690
|
+
@@windows ||= /^(mingw)/.match?(RbConfig::CONFIG['target_os']) # RubyInstaller's MRI, other MinGW-build MRI
|
|
691
|
+
end
|
|
692
|
+
|
|
693
|
+
# Match OS-specific system mount points (/dev /proc etc.) which normally should be omitted when scanning for Mclone voulmes
|
|
694
|
+
UNIX_SYSTEM_MOUNTS = %r!^/(dev|sys|proc)!
|
|
695
|
+
|
|
696
|
+
# TODO handle Windows variants
|
|
697
|
+
# Specify OS-specific path name list separator (such as in the $PATH environment variable)
|
|
698
|
+
PATH_LIST_SEPARATOR = windows? ? ';' : ':'
|
|
699
|
+
|
|
700
|
+
# Return list of live user-provided mounts (mount points on *NIX and disk drives on Windows) which may contain Mclone volumes
|
|
701
|
+
# Look for the $MCLONE_PATH environment variable
|
|
702
|
+
def self.environment_mounts
|
|
703
|
+
ENV['MCLONE_PATH'].split(PATH_LIST_SEPARATOR).collect { |path| File.directory?(path) ? path : nil }.compact rescue []
|
|
704
|
+
end
|
|
705
|
+
# Return list of live system-managed mounts (mount points on *NIX and disk drives on Windows) which may contain Mclone volumes
|
|
706
|
+
case RbConfig::CONFIG['target_os']
|
|
707
|
+
when 'linux'
|
|
708
|
+
# Linux OS
|
|
709
|
+
def self.system_mounts
|
|
710
|
+
# Query /proc for currently mounted file systems
|
|
711
|
+
IO.readlines('/proc/self/mountstats').collect do |line|
|
|
712
|
+
mount = line.split[4]
|
|
713
|
+
UNIX_SYSTEM_MOUNTS.match?(mount) || !File.directory?(mount) ? nil : mount
|
|
714
|
+
end.compact
|
|
715
|
+
end
|
|
716
|
+
# TODO handle Windows variants
|
|
717
|
+
when /^mingw/ # RubyInstaller's MRI
|
|
718
|
+
module Kernel32
|
|
719
|
+
require 'fiddle'
|
|
720
|
+
require 'fiddle/types'
|
|
721
|
+
require 'fiddle/import'
|
|
722
|
+
extend Fiddle::Importer
|
|
723
|
+
dlload('kernel32')
|
|
724
|
+
include Fiddle::Win32Types
|
|
725
|
+
extern 'DWORD WINAPI GetLogicalDrives()'
|
|
726
|
+
end
|
|
727
|
+
def self.system_mounts
|
|
728
|
+
mounts = []
|
|
729
|
+
mask = Kernel32.GetLogicalDrives
|
|
730
|
+
('A'..'Z').each do |x|
|
|
731
|
+
mounts << "#{x}:" if mask & 1 == 1
|
|
732
|
+
mask >>= 1
|
|
733
|
+
end
|
|
734
|
+
mounts
|
|
735
|
+
end
|
|
736
|
+
else
|
|
737
|
+
# Generic *NIX-like OS, including Cygwin & MSYS(2)
|
|
738
|
+
def self.system_mounts
|
|
739
|
+
# Use $(mount) system utility to obtain currently mounted file systems
|
|
740
|
+
%x(mount).split("\n").collect do |line|
|
|
741
|
+
mount = line.split[2]
|
|
742
|
+
UNIX_SYSTEM_MOUNTS.match?(mount) || !File.directory?(mount) ? nil : mount
|
|
743
|
+
end.compact
|
|
744
|
+
end
|
|
745
|
+
end
|
|
746
746
|
end
|