odisk 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +27 -0
- data/README.md +150 -0
- data/bin/odisk +184 -0
- data/lib/odisk.rb +46 -0
- data/lib/odisk/copier.rb +74 -0
- data/lib/odisk/crypter.rb +40 -0
- data/lib/odisk/diff.rb +81 -0
- data/lib/odisk/digest.rb +85 -0
- data/lib/odisk/digester.rb +33 -0
- data/lib/odisk/dir.rb +10 -0
- data/lib/odisk/dirsyncjob.rb +25 -0
- data/lib/odisk/fetcher.rb +54 -0
- data/lib/odisk/file.rb +18 -0
- data/lib/odisk/info.rb +40 -0
- data/lib/odisk/link.rb +18 -0
- data/lib/odisk/planner.rb +309 -0
- data/lib/odisk/remote.rb +45 -0
- data/lib/odisk/statfixer.rb +102 -0
- data/lib/odisk/statjob.rb +32 -0
- data/lib/odisk/syncjob.rb +25 -0
- data/lib/odisk/syncstarter.rb +37 -0
- data/lib/odisk/version.rb +5 -0
- metadata +135 -0
@@ -0,0 +1,40 @@
|
|
1
|
+
|
2
|
+
module ODisk
|
3
|
+
class Crypter < ::Opee::Actor
|
4
|
+
|
5
|
+
def initialize(options={})
|
6
|
+
super(options)
|
7
|
+
@crypt_queue.ask(:ready, self)
|
8
|
+
end
|
9
|
+
|
10
|
+
def set_options(options)
|
11
|
+
super(options)
|
12
|
+
@crypt_queue = options[:crypt_queue]
|
13
|
+
@copy_queue = options[:copy_queue]
|
14
|
+
@fixer = options[:fixer]
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def encrypt(src, dest, remote)
|
20
|
+
::Opee::Env.info("encrypt \"#{src}\" into \"#{dest}\"")
|
21
|
+
unless $dry_run
|
22
|
+
`gpg -c --batch --yes --force-mdc --passphrase-file "#{$remote.pass_file}" -o "#{dest}" "#{src}"`
|
23
|
+
@copy_queue.add_method(:upload, dest, remote, true)
|
24
|
+
end
|
25
|
+
@crypt_queue.ask(:ready, self)
|
26
|
+
end
|
27
|
+
|
28
|
+
def decrypt(src, dest)
|
29
|
+
::Opee::Env.info("decrypt \"#{src}\" into \"#{dest}\"")
|
30
|
+
unless $dry_run
|
31
|
+
`gpg -d --batch --yes -q --passphrase-file "#{$remote.pass_file}" -o "#{dest}" "#{src}"`
|
32
|
+
::File.delete(src)
|
33
|
+
::Opee::Env.warn("Downloaded and decrypted \"#{dest}\"")
|
34
|
+
end
|
35
|
+
@fixer.ask(:collect, dest, :crypter) unless @fixer.nil?
|
36
|
+
@crypt_queue.ask(:ready, self)
|
37
|
+
end
|
38
|
+
|
39
|
+
end # Crypter
|
40
|
+
end # ODisk
|
data/lib/odisk/diff.rb
ADDED
@@ -0,0 +1,81 @@
|
|
1
|
+
|
2
|
+
module ODisk
|
3
|
+
class Diff
|
4
|
+
# local Info Object
|
5
|
+
attr_accessor :local
|
6
|
+
# remote Info Object
|
7
|
+
attr_accessor :remote
|
8
|
+
|
9
|
+
attr_accessor :sub_diffs
|
10
|
+
|
11
|
+
def self.dir_diff(l, r, recursive=false)
|
12
|
+
ld = Digest.create(l, nil)
|
13
|
+
rd = Digest.create(r, nil)
|
14
|
+
diffs = digest_diff(ld, rd)
|
15
|
+
if recursive
|
16
|
+
lh = ld.entries_hash()
|
17
|
+
rh = rd.entries_hash()
|
18
|
+
keys = lh.keys | rh.keys
|
19
|
+
keys.each do |k|
|
20
|
+
next unless lh[k].is_a?(::ODisk::Dir) || rh[k].is_a?(::ODisk::Dir)
|
21
|
+
unless (sd = dir_diff(::File.join(l, k), ::File.join(r, k))).nil? || sd.empty?
|
22
|
+
if (d = diffs[k]).nil?
|
23
|
+
d = self.new(lh[k], rh[k], sd)
|
24
|
+
else
|
25
|
+
d.sub_diffs = sd
|
26
|
+
end
|
27
|
+
diffs[k] = d
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
diffs
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.digest_diff(ld, rd)
|
35
|
+
diffs = {}
|
36
|
+
lh = ld.entries_hash()
|
37
|
+
rh = rd.entries_hash()
|
38
|
+
keys = lh.keys | rh.keys
|
39
|
+
keys.each do |k|
|
40
|
+
le = lh[k]
|
41
|
+
re = rh[k]
|
42
|
+
diffs[k] = self.new(le, re) unless le == re
|
43
|
+
end
|
44
|
+
diffs
|
45
|
+
end
|
46
|
+
|
47
|
+
def initialize(local, remote, sub_diffs={})
|
48
|
+
@local = local
|
49
|
+
@remote = remote
|
50
|
+
@sub_diffs = sub_diffs
|
51
|
+
end
|
52
|
+
|
53
|
+
def fill_hash(prefix, h, sym=false)
|
54
|
+
[:class, :name, :owner, :group, :mode, :mtime, :size, :target].each do |m|
|
55
|
+
next unless @local.respond_to?(m) and @remote.respond_to?(m)
|
56
|
+
lv = @local.send(m)
|
57
|
+
rv = @remote.send(m)
|
58
|
+
if sym
|
59
|
+
k = m
|
60
|
+
else
|
61
|
+
k = @local.name + '.' + m.to_s
|
62
|
+
k = prefix + '.' + k unless prefix.nil?
|
63
|
+
end
|
64
|
+
h[k] = [lv, rv] unless lv == rv unless (:mtime == m && !@remote.is_a?(::ODisk::File))
|
65
|
+
end
|
66
|
+
pre = prefix.nil? ? @local.name : (prefix + '.' + @local.name)
|
67
|
+
@sub_diffs.each do |name,d|
|
68
|
+
d.fill_hash(pre, h)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def to_s()
|
73
|
+
s = ''
|
74
|
+
h = {}
|
75
|
+
fill_hash(nil, h)
|
76
|
+
h.each { |k,v| s << "#{k}: #{v[0]} vs #{v[1]}\n" }
|
77
|
+
s
|
78
|
+
end
|
79
|
+
|
80
|
+
end # Diff
|
81
|
+
end # ODisk
|
data/lib/odisk/digest.rb
ADDED
@@ -0,0 +1,85 @@
|
|
1
|
+
|
2
|
+
module ODisk
|
3
|
+
class Digest
|
4
|
+
|
5
|
+
attr_accessor :version
|
6
|
+
|
7
|
+
attr_accessor :entries
|
8
|
+
|
9
|
+
attr_reader :top_path
|
10
|
+
|
11
|
+
def self.create(top, rel_path)
|
12
|
+
path = (rel_path.nil? || rel_path.empty?) ? top : ::File.join(top, rel_path)
|
13
|
+
raise "#{path} is not a directory" unless ::File.directory?(path)
|
14
|
+
d = self.new(rel_path)
|
15
|
+
::Dir.foreach(path) do |filename|
|
16
|
+
next if filename.start_with?('.')
|
17
|
+
child_path = ::File.join(path, filename)
|
18
|
+
c = self.create_info(child_path, filename, top)
|
19
|
+
d.entries << c
|
20
|
+
end
|
21
|
+
d
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.create_info(path, filename=nil, top=nil)
|
25
|
+
top = $local_top if top.nil?
|
26
|
+
filename = ::File.basename(path) if filename.nil?
|
27
|
+
stat = ::File.lstat(path)
|
28
|
+
if stat.directory?
|
29
|
+
c = ::ODisk::Dir.new(filename)
|
30
|
+
elsif stat.symlink?
|
31
|
+
c = ::ODisk::Link.new(filename)
|
32
|
+
c.target = ::File.readlink(path)
|
33
|
+
c.target = c.target[top.size + 1..-1] if c.target.start_with?(top)
|
34
|
+
elsif stat.file?
|
35
|
+
c = ::ODisk::File.new(filename)
|
36
|
+
c.size = stat.size()
|
37
|
+
else
|
38
|
+
raise "file type of #{job.path} is not supported"
|
39
|
+
end
|
40
|
+
c.mtime = stat.mtime()
|
41
|
+
c.mode = stat.mode & 0777
|
42
|
+
begin
|
43
|
+
c.owner = Etc.getpwuid(stat.uid).name
|
44
|
+
rescue
|
45
|
+
c.owner = stat.uid
|
46
|
+
end
|
47
|
+
begin
|
48
|
+
c.group = Etc.getgrgid(stat.gid).name
|
49
|
+
rescue
|
50
|
+
c.group = stat.gid
|
51
|
+
end
|
52
|
+
c
|
53
|
+
end
|
54
|
+
|
55
|
+
def initialize(top_path)
|
56
|
+
@top_path = top_path
|
57
|
+
@version = 0
|
58
|
+
@entries = []
|
59
|
+
end
|
60
|
+
|
61
|
+
def [](name)
|
62
|
+
@entries.each { |e| return e if name == e.name }
|
63
|
+
nil
|
64
|
+
end
|
65
|
+
|
66
|
+
def empty?()
|
67
|
+
@entries.empty?
|
68
|
+
end
|
69
|
+
|
70
|
+
def delete(name)
|
71
|
+
@entries.delete_if { |e| name == e.name }
|
72
|
+
end
|
73
|
+
|
74
|
+
def sub_dirs()
|
75
|
+
@entries.select { |e| e.is_a?(::ODisk::Dir) }
|
76
|
+
end
|
77
|
+
|
78
|
+
def entries_hash()
|
79
|
+
h = {}
|
80
|
+
@entries.each { |e| h[e.name] = e }
|
81
|
+
h
|
82
|
+
end
|
83
|
+
|
84
|
+
end # Digest
|
85
|
+
end # ODisk
|
@@ -0,0 +1,33 @@
|
|
1
|
+
|
2
|
+
require 'etc'
|
3
|
+
|
4
|
+
module ODisk
|
5
|
+
class Digester < ::Opee::Actor
|
6
|
+
|
7
|
+
def initialize(options={})
|
8
|
+
super(options)
|
9
|
+
end
|
10
|
+
|
11
|
+
def set_options(options)
|
12
|
+
super(options)
|
13
|
+
@collector = options[:collector]
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
def create(job)
|
19
|
+
path = ::File.join($local_top, job.path)
|
20
|
+
if ::File.directory?(path)
|
21
|
+
::Opee::Env.info("create digest for #{path}")
|
22
|
+
d = ::ODisk::Digest.create($local_top, job.path)
|
23
|
+
else
|
24
|
+
::Opee::Env.info("#{path} does not exist, no digest")
|
25
|
+
d = nil
|
26
|
+
end
|
27
|
+
job.current_digest = d
|
28
|
+
@collector.ask(:collect, job, :digester) unless @collector.nil?
|
29
|
+
::Opee::Env.debug("#{Oj.dump(d, indent: 2)})")
|
30
|
+
end
|
31
|
+
|
32
|
+
end # Digester
|
33
|
+
end # ODisk
|
data/lib/odisk/dir.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
|
2
|
+
module ODisk
|
3
|
+
class DirSyncJob < ::Opee::Job
|
4
|
+
|
5
|
+
# relative path from top
|
6
|
+
attr_reader :path
|
7
|
+
# only set by the Digester
|
8
|
+
attr_accessor :current_digest
|
9
|
+
# only set by the SyncStarter
|
10
|
+
attr_accessor :previous_digest
|
11
|
+
# only set by the Fetcher
|
12
|
+
attr_accessor :remote_digest
|
13
|
+
# new digest to be set locally and remotely
|
14
|
+
attr_accessor :new_digest
|
15
|
+
|
16
|
+
def initialize(path)
|
17
|
+
@path = path
|
18
|
+
@current_digest = nil
|
19
|
+
@previous_digest = nil
|
20
|
+
@remote_digest = nil
|
21
|
+
@new_digest = nil
|
22
|
+
end
|
23
|
+
|
24
|
+
end # DirSyncJob
|
25
|
+
end # ODisk
|
@@ -0,0 +1,54 @@
|
|
1
|
+
|
2
|
+
begin
|
3
|
+
v = $VERBOSE
|
4
|
+
$VERBOSE = false
|
5
|
+
require 'net/ssh'
|
6
|
+
require 'net/sftp'
|
7
|
+
$VERBOSE = v
|
8
|
+
end
|
9
|
+
require 'oj'
|
10
|
+
|
11
|
+
module ODisk
|
12
|
+
class Fetcher < ::Opee::Actor
|
13
|
+
|
14
|
+
def initialize(options={})
|
15
|
+
@ftp = nil
|
16
|
+
super(options)
|
17
|
+
end
|
18
|
+
|
19
|
+
def set_options(options)
|
20
|
+
super(options)
|
21
|
+
@collector = options[:collector]
|
22
|
+
end
|
23
|
+
|
24
|
+
def close()
|
25
|
+
@ftp.close_channel() unless @ftp.nil?
|
26
|
+
@ftp = nil
|
27
|
+
super()
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def fetch(job)
|
33
|
+
top = (job.path.nil? || job.path.empty?) ? $remote.dir : ::File.join($remote.dir, job.path)
|
34
|
+
path = ::File.join(top, '.odisk', 'digest.json')
|
35
|
+
::Opee::Env.info("fetch digest \"#{path}\"")
|
36
|
+
@ftp = Net::SFTP.start($remote.host, $remote.user) if @ftp.nil?
|
37
|
+
begin
|
38
|
+
json = @ftp.download!(path)
|
39
|
+
files = @ftp.dir.entries(top).map {|e| e.name }
|
40
|
+
job.remote_digest = Oj.load(json, mode: :object)
|
41
|
+
missing = []
|
42
|
+
job.remote_digest.entries.each { |e| missing << e.name unless files.include?(e.name + '.gpg') || files.include?(e.name) }
|
43
|
+
unless ::ODisk::Planner::Step::REMOTE == $master
|
44
|
+
missing.each { |name| job.remote_digest.delete(name) }
|
45
|
+
end
|
46
|
+
rescue Exception
|
47
|
+
job.remote_digest = nil
|
48
|
+
end
|
49
|
+
@collector.ask(:collect, job, :fetcher) unless @collector.nil?
|
50
|
+
::Opee::Env.debug("#{Oj.dump(job.remote_digest, indent: 2)})")
|
51
|
+
end
|
52
|
+
|
53
|
+
end # Fetcher
|
54
|
+
end # ODisk
|
data/lib/odisk/file.rb
ADDED
data/lib/odisk/info.rb
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
|
2
|
+
module ODisk
|
3
|
+
class Info
|
4
|
+
|
5
|
+
NORMAL = 0 # or nil
|
6
|
+
ERROR = 1
|
7
|
+
REMOTE = 2
|
8
|
+
LOCAL = 3
|
9
|
+
|
10
|
+
attr_accessor :name
|
11
|
+
attr_accessor :owner
|
12
|
+
attr_accessor :group
|
13
|
+
attr_accessor :mode
|
14
|
+
attr_accessor :mtime
|
15
|
+
attr_accessor :master
|
16
|
+
attr_accessor :removed
|
17
|
+
|
18
|
+
def initialize(name)
|
19
|
+
@name = name
|
20
|
+
@owner = nil
|
21
|
+
@group = nil
|
22
|
+
@mode = 0
|
23
|
+
@mtime = nil
|
24
|
+
@master = nil
|
25
|
+
@removed = false
|
26
|
+
end
|
27
|
+
|
28
|
+
def eql?(o)
|
29
|
+
return false unless o.class == self.class
|
30
|
+
# don't check master flag
|
31
|
+
(@name == o.name &&
|
32
|
+
@owner == o.owner &&
|
33
|
+
@group == o.group &&
|
34
|
+
@mode == o.mode &&
|
35
|
+
@removed == o.removed)
|
36
|
+
end
|
37
|
+
alias == eql?
|
38
|
+
|
39
|
+
end # Info
|
40
|
+
end # ODisk
|
data/lib/odisk/link.rb
ADDED
@@ -0,0 +1,309 @@
|
|
1
|
+
|
2
|
+
require 'oj'
|
3
|
+
|
4
|
+
module ODisk
|
5
|
+
class Planner < ::Opee::Collector
|
6
|
+
|
7
|
+
def initialize(options={})
|
8
|
+
super(options)
|
9
|
+
end
|
10
|
+
|
11
|
+
def set_options(options)
|
12
|
+
super(options)
|
13
|
+
@dir_queue = options[:dir_queue]
|
14
|
+
@copy_queue = options[:copy_queue]
|
15
|
+
@crypt_queue = options[:crypt_queue]
|
16
|
+
@inputs = options[:inputs]
|
17
|
+
@fixer = options[:fixer]
|
18
|
+
end
|
19
|
+
|
20
|
+
# master can be Step::LOCAL or Step::REMOTE and forces direction
|
21
|
+
def self.sync_steps(pd, ld, rd, master=nil)
|
22
|
+
steps = {}
|
23
|
+
lh = {}
|
24
|
+
rh = {}
|
25
|
+
ph = {}
|
26
|
+
ld.entries.each { |e| lh[e.name] = e }
|
27
|
+
rd.entries.each { |e| rh[e.name] = e }
|
28
|
+
pd.entries.each { |e| ph[e.name] = e } unless pd.nil?
|
29
|
+
keys = lh.keys | rh.keys
|
30
|
+
keys.each do |name|
|
31
|
+
le = lh[name]
|
32
|
+
re = rh[name]
|
33
|
+
if re.nil?
|
34
|
+
if Step::REMOTE == master
|
35
|
+
steps[name] = Step.new(name, Step::LOCAL, Step::REMOVE)
|
36
|
+
elsif le.is_a?(::ODisk::File)
|
37
|
+
steps[name] = Step.new(name, Step::LOCAL, Step::COPY)
|
38
|
+
elsif le.is_a?(::ODisk::Dir)
|
39
|
+
steps[name] = Step.new(name, Step::LOCAL, Step::DIGEST)
|
40
|
+
end
|
41
|
+
elsif le.nil?
|
42
|
+
if Step::LOCAL == master
|
43
|
+
steps[name] = Step.new(name, Step::REMOTE, Step::REMOVE)
|
44
|
+
elsif re.is_a?(::ODisk::File)
|
45
|
+
steps[name] = Step.new(name, Step::REMOTE, Step::COPY)
|
46
|
+
elsif re.is_a?(::ODisk::Dir)
|
47
|
+
steps[name] = Step.new(name, Step::REMOTE, Step::DIGEST)
|
48
|
+
end
|
49
|
+
elsif le != re # both exist but are different
|
50
|
+
if le.class != re.class
|
51
|
+
::Opee::Env.error("Conflict syncing #{ld.top_path}/#{name}. Local and remote types do not match.")
|
52
|
+
steps[name] = Step.new(name, Step::LOCAL, Step::ERROR)
|
53
|
+
elsif le.is_a?(::ODisk::File) || le.is_a?(::ODisk::Link)
|
54
|
+
op = le.is_a?(::ODisk::File) ? Step::COPY : Step::LINK
|
55
|
+
if Step::LOCAL == master
|
56
|
+
steps[name] = Step.new(name, Step::LOCAL, op) if le.is_a?(::ODisk::File)
|
57
|
+
elsif Step::REMOTE == master
|
58
|
+
steps[name] = Step.new(name, Step::REMOTE, op) if re.is_a?(::ODisk::File)
|
59
|
+
elsif le.removed
|
60
|
+
# TBD
|
61
|
+
elsif re.removed
|
62
|
+
# TBD
|
63
|
+
elsif le.mtime > re.mtime
|
64
|
+
pe = ph[name]
|
65
|
+
if pe.nil? || pe.mtime == re.mtime
|
66
|
+
# Don't know if the content or the stats changed so copy it.
|
67
|
+
steps[name] = Step.new(name, Step::LOCAL, op) if le.is_a?(::ODisk::File)
|
68
|
+
else
|
69
|
+
::Opee::Env.error("Conflict syncing #{ld.top_path}/#{name}. Both local and remote have changed.")
|
70
|
+
steps[name] = Step.new(name, Step::LOCAL, Step::ERROR)
|
71
|
+
end
|
72
|
+
elsif le.mtime < re.mtime
|
73
|
+
# Don't know if the content or the stats changed so copy it.
|
74
|
+
steps[name] = Step.new(name, Step::REMOTE, op) if re.is_a?(::ODisk::File)
|
75
|
+
else # same times but different can't be good
|
76
|
+
::Opee::Env.error("Conflict syncing #{ld.top_path}/#{name}. Both local and remote have changed.")
|
77
|
+
steps[name] = Step.new(name, Step::LOCAL, Step::ERROR)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
steps.empty? ? nil : steps
|
83
|
+
end
|
84
|
+
|
85
|
+
private
|
86
|
+
|
87
|
+
def update_token(job, token, path_id)
|
88
|
+
token = [] if token.nil?
|
89
|
+
token << path_id
|
90
|
+
token
|
91
|
+
end
|
92
|
+
|
93
|
+
def complete?(job, token)
|
94
|
+
result = !token.nil? && token.size == @inputs.size && token.sort == @inputs.sort
|
95
|
+
::Opee::Env.info("complete?(#{job.key()}, #{token}) => #{result}")
|
96
|
+
#::Opee::Env.debug("#{Oj.dump(job, indent: 2)}")
|
97
|
+
result
|
98
|
+
end
|
99
|
+
|
100
|
+
def keep_going(job)
|
101
|
+
path = ::File.join($local_top, job.path)
|
102
|
+
odisk_dir = ::File.join(path, '.odisk')
|
103
|
+
`mkdir -p "#{odisk_dir}"` unless ::File.directory?(odisk_dir) && !$dry_run
|
104
|
+
if @copy_queue.nil? && @crypt_queue.nil? && !job.current_digest.nil? # digests_only
|
105
|
+
if job.previous_digest.nil?
|
106
|
+
job.current_digest.version = 1
|
107
|
+
else
|
108
|
+
job.current_digest.version = job.previous_digest.version + 1
|
109
|
+
Oj.to_file(::File.join(odisk_dir, 'digest.old.json'), job.previous_digest, indent: 2)
|
110
|
+
end
|
111
|
+
Oj.to_file(::File.join(odisk_dir, 'digest.json'), job.current_digest, indent: 2)
|
112
|
+
job.current_digest.entries.each do |e|
|
113
|
+
@dir_queue.ask(:add, job.path.empty? ? e.name : ::File.join(job.path, e.name)) if e.is_a?(::ODisk::Dir)
|
114
|
+
end
|
115
|
+
elsif job.remote_digest.nil?
|
116
|
+
process_new(job, odisk_dir)
|
117
|
+
elsif ((job.current_digest.nil? || job.current_digest.empty?) &&
|
118
|
+
(job.previous_digest.nil? || job.previous_digest.empty?))
|
119
|
+
process_down(job, odisk_dir)
|
120
|
+
else
|
121
|
+
process_sync(job, odisk_dir)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
def process_new(job, odisk_dir)
|
126
|
+
job.current_digest.version = 1
|
127
|
+
job.new_digest = job.current_digest
|
128
|
+
# determine update new_digest
|
129
|
+
# write the digest files
|
130
|
+
# TBD if they are the same then don't bother
|
131
|
+
Oj.to_file(::File.join(odisk_dir, 'digest.old.json'), job.previous_digest, indent: 2) unless job.previous_digest.nil?
|
132
|
+
Oj.to_file(::File.join(odisk_dir, 'digest.json'), job.new_digest, indent: 2)
|
133
|
+
|
134
|
+
# get the transfers going for all files
|
135
|
+
job.new_digest.entries.each do |e|
|
136
|
+
path = job.path.empty? ? e.name : ::File.join(job.path, e.name)
|
137
|
+
case e
|
138
|
+
when ::ODisk::Dir
|
139
|
+
@dir_queue.ask(:add, path)
|
140
|
+
when ::ODisk::File
|
141
|
+
local = ::File.join($local_top, path)
|
142
|
+
remote = ::File.join($remote.dir, path)
|
143
|
+
if $remote.encrypt?
|
144
|
+
encrypt_path = (job.path.empty? ?
|
145
|
+
::File.join($local_top, '.odisk', e.name + '.gpg') :
|
146
|
+
::File.join($local_top, job.path, '.odisk', e.name + '.gpg'))
|
147
|
+
@crypt_queue.add_method(:encrypt, local, encrypt_path, remote + '.gpg')
|
148
|
+
else
|
149
|
+
@copy_queue.add_method(:upload, local, remote)
|
150
|
+
end
|
151
|
+
when ::ODisk::Link
|
152
|
+
# nothing to do
|
153
|
+
end
|
154
|
+
end
|
155
|
+
path = job.path.empty? ? ::File.join('.odisk', 'digest.json') : ::File.join(job.path, '.odisk', 'digest.json')
|
156
|
+
local = ::File.join($local_top, path)
|
157
|
+
remote = ::File.join($remote.dir, path)
|
158
|
+
@copy_queue.add_method(:upload, local, remote)
|
159
|
+
end
|
160
|
+
|
161
|
+
def process_down(job, odisk_dir)
|
162
|
+
job.new_digest = job.remote_digest
|
163
|
+
Oj.to_file(::File.join(odisk_dir, 'digest.json'), job.new_digest, indent: 2)
|
164
|
+
full_job_path = job.path.empty? ? $local_top : ::File.join($local_top, job.path)
|
165
|
+
stat_job = StatJob.new(full_job_path, job.new_digest)
|
166
|
+
job.new_digest.entries.each do |e|
|
167
|
+
path = job.path.empty? ? e.name : ::File.join(job.path, e.name)
|
168
|
+
local = ::File.join($local_top, path)
|
169
|
+
case e
|
170
|
+
when ::ODisk::Dir
|
171
|
+
::Dir.mkdir(local) unless $dry_run || ::File.directory?(local)
|
172
|
+
@dir_queue.ask(:add, path)
|
173
|
+
when ::ODisk::File
|
174
|
+
remote = ::File.join($remote.dir, path)
|
175
|
+
stat_job.add_mod(e.name)
|
176
|
+
if $remote.encrypt?
|
177
|
+
encrypt_path = (job.path.empty? ?
|
178
|
+
::File.join($local_top, '.odisk', e.name + '.gpg') :
|
179
|
+
::File.join($local_top, job.path, '.odisk', e.name + '.gpg'))
|
180
|
+
@copy_queue.add_method(:download, remote + '.gpg', encrypt_path, local)
|
181
|
+
else
|
182
|
+
@copy_queue.add_method(:download, remote, local, nil)
|
183
|
+
end
|
184
|
+
when ::ODisk::Link
|
185
|
+
target = e.target
|
186
|
+
target = ::File.join($local_top, e.target) unless '/' == target[0]
|
187
|
+
::Opee::Env.info("symlink \"#{local}\" -> \"#{target}\"}")
|
188
|
+
::File.symlink(target, local) unless $dry_run || ::File.exists?(local)
|
189
|
+
end
|
190
|
+
end
|
191
|
+
@fixer.ask(:collect, stat_job, :planner) unless @fixer.nil?
|
192
|
+
end
|
193
|
+
|
194
|
+
def process_sync(job, odisk_dir)
|
195
|
+
dirs = []
|
196
|
+
job.current_digest.entries.each { |e| dirs << e.name unless !e.is_a?(::ODisk::Dir) || dirs.include?(e.name) }
|
197
|
+
job.remote_digest.entries.each { |e| dirs << e.name unless !e.is_a?(::ODisk::Dir) || dirs.include?(e.name) }
|
198
|
+
dirs.each do |dir|
|
199
|
+
path = job.path.empty? ? dir : ::File.join(job.path, dir)
|
200
|
+
local = ::File.join($local_top, path)
|
201
|
+
::Dir.mkdir(local) unless $dry_run || ::File.directory?(local)
|
202
|
+
@dir_queue.ask(:add, path)
|
203
|
+
end
|
204
|
+
steps = self.class.sync_steps(job.previous_digest, job.current_digest, job.remote_digest, $master)
|
205
|
+
#puts "*** steps for #{job.path}: #{steps}"
|
206
|
+
return if steps.nil?
|
207
|
+
|
208
|
+
Oj.to_file(::File.join(odisk_dir, 'digest.old.json'), job.previous_digest, indent: 2) unless job.previous_digest.nil?
|
209
|
+
nrh = {} # fill with new digest entries for remote
|
210
|
+
nlh = {} # fill with new digest entries for local
|
211
|
+
full_job_path = job.path.empty? ? $local_top : ::File.join($local_top, job.path)
|
212
|
+
stat_job = StatJob.new(full_job_path, nil)
|
213
|
+
job.remote_digest.entries.each { |e| nrh[e.name] = e }
|
214
|
+
job.current_digest.entries.each { |e| nlh[e.name] = e }
|
215
|
+
=begin
|
216
|
+
if job.previous_digest.nil?
|
217
|
+
job.current_digest.entries.each { |e| nlh[e.name] = e }
|
218
|
+
else
|
219
|
+
job.previous_digest.entries.each { |e| nlh[e.name] = e }
|
220
|
+
end
|
221
|
+
=end
|
222
|
+
steps.values.each do |s|
|
223
|
+
stat_job.add_mod(s.name) unless Step::REMOVE == s.op || Step::DIGEST == s.op
|
224
|
+
case s.op
|
225
|
+
when Step::COPY
|
226
|
+
path = job.path.empty? ? s.name : ::File.join(job.path, s.name)
|
227
|
+
local = ::File.join($local_top, path)
|
228
|
+
remote = ::File.join($remote.dir, path)
|
229
|
+
encrypt_path = (job.path.empty? ?
|
230
|
+
::File.join($local_top, '.odisk', s.name + '.gpg') :
|
231
|
+
::File.join($local_top, job.path, '.odisk', s.name + '.gpg'))
|
232
|
+
if Step::REMOTE == s.master
|
233
|
+
if $remote.encrypt?
|
234
|
+
@copy_queue.add_method(:download, remote + '.gpg', encrypt_path, local)
|
235
|
+
else
|
236
|
+
@copy_queue.add_method(:download, remote, local, nil)
|
237
|
+
end
|
238
|
+
e = job.remote_digest[s.name]
|
239
|
+
else
|
240
|
+
if $remote.encrypt?
|
241
|
+
@crypt_queue.add_method(:encrypt, local, encrypt_path, remote + '.gpg')
|
242
|
+
else
|
243
|
+
@copy_queue.add_method(:upload, local, remote)
|
244
|
+
end
|
245
|
+
e = job.current_digest[s.name]
|
246
|
+
end
|
247
|
+
nrh[e.name] = e
|
248
|
+
nlh[e.name] = e
|
249
|
+
when Step::REMOVE
|
250
|
+
# TBD
|
251
|
+
when Step::DIGEST
|
252
|
+
e = (Step::REMOTE == s.master) ? job.remote_digest[s.name] : job.current_digest[s.name]
|
253
|
+
nrh[e.name] = e
|
254
|
+
nlh[e.name] = e
|
255
|
+
when Step::LINK
|
256
|
+
if Step::REMOTE == s.master
|
257
|
+
#TBD create link
|
258
|
+
end
|
259
|
+
when Step::ERROR
|
260
|
+
# TBD
|
261
|
+
end
|
262
|
+
end
|
263
|
+
nrd = Digest.new(job.remote_digest.top_path)
|
264
|
+
nld = Digest.new(job.current_digest.top_path)
|
265
|
+
# fill in digest entries from hashs
|
266
|
+
nrd.entries = nrh.values
|
267
|
+
nld.entries = nlh.values
|
268
|
+
|
269
|
+
job.new_digest = nrd
|
270
|
+
remote_digest_path = ::File.join(odisk_dir, 'digest.remote.json')
|
271
|
+
Oj.to_file(::File.join(odisk_dir, 'digest.json'), nld, indent: 2)
|
272
|
+
Oj.to_file(remote_digest_path, job.new_digest, indent: 2)
|
273
|
+
path = job.path.empty? ? ::File.join('.odisk', 'digest.json') : ::File.join(job.path, '.odisk', 'digest.json')
|
274
|
+
@copy_queue.add_method(:upload, remote_digest_path, ::File.join($remote.dir, path)) unless Step::REMOTE == $master
|
275
|
+
|
276
|
+
stat_job.digest = nld
|
277
|
+
@fixer.ask(:collect, stat_job, :planner) unless @fixer.nil?
|
278
|
+
end
|
279
|
+
|
280
|
+
class Step
|
281
|
+
# op values
|
282
|
+
STATS = 0
|
283
|
+
REMOVE = 1
|
284
|
+
COPY = 2
|
285
|
+
LINK = 3
|
286
|
+
DIGEST = 4
|
287
|
+
ERROR = 5
|
288
|
+
|
289
|
+
# location of master
|
290
|
+
LOCAL = true
|
291
|
+
REMOTE = false
|
292
|
+
|
293
|
+
attr_reader :name
|
294
|
+
attr_reader :master
|
295
|
+
attr_reader :op
|
296
|
+
|
297
|
+
def initialize(name, master, op)
|
298
|
+
@name = name
|
299
|
+
@master = master
|
300
|
+
@op = op
|
301
|
+
end
|
302
|
+
|
303
|
+
def to_s()
|
304
|
+
"<Step name=#{@name} master=#{@master ? 'LOCAL' : 'REMOTE'} op=#{['STATS', 'REMOVE', 'COPY', 'LINK', 'ERROR'][@op]}>"
|
305
|
+
end
|
306
|
+
end # Step
|
307
|
+
|
308
|
+
end # Planner
|
309
|
+
end # ODisk
|