odisk 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +27 -0
- data/README.md +150 -0
- data/bin/odisk +184 -0
- data/lib/odisk.rb +46 -0
- data/lib/odisk/copier.rb +74 -0
- data/lib/odisk/crypter.rb +40 -0
- data/lib/odisk/diff.rb +81 -0
- data/lib/odisk/digest.rb +85 -0
- data/lib/odisk/digester.rb +33 -0
- data/lib/odisk/dir.rb +10 -0
- data/lib/odisk/dirsyncjob.rb +25 -0
- data/lib/odisk/fetcher.rb +54 -0
- data/lib/odisk/file.rb +18 -0
- data/lib/odisk/info.rb +40 -0
- data/lib/odisk/link.rb +18 -0
- data/lib/odisk/planner.rb +309 -0
- data/lib/odisk/remote.rb +45 -0
- data/lib/odisk/statfixer.rb +102 -0
- data/lib/odisk/statjob.rb +32 -0
- data/lib/odisk/syncjob.rb +25 -0
- data/lib/odisk/syncstarter.rb +37 -0
- data/lib/odisk/version.rb +5 -0
- metadata +135 -0
@@ -0,0 +1,40 @@
|
|
1
|
+
|
2
|
+
module ODisk
|
3
|
+
class Crypter < ::Opee::Actor
|
4
|
+
|
5
|
+
def initialize(options={})
|
6
|
+
super(options)
|
7
|
+
@crypt_queue.ask(:ready, self)
|
8
|
+
end
|
9
|
+
|
10
|
+
def set_options(options)
|
11
|
+
super(options)
|
12
|
+
@crypt_queue = options[:crypt_queue]
|
13
|
+
@copy_queue = options[:copy_queue]
|
14
|
+
@fixer = options[:fixer]
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def encrypt(src, dest, remote)
|
20
|
+
::Opee::Env.info("encrypt \"#{src}\" into \"#{dest}\"")
|
21
|
+
unless $dry_run
|
22
|
+
`gpg -c --batch --yes --force-mdc --passphrase-file "#{$remote.pass_file}" -o "#{dest}" "#{src}"`
|
23
|
+
@copy_queue.add_method(:upload, dest, remote, true)
|
24
|
+
end
|
25
|
+
@crypt_queue.ask(:ready, self)
|
26
|
+
end
|
27
|
+
|
28
|
+
def decrypt(src, dest)
|
29
|
+
::Opee::Env.info("decrypt \"#{src}\" into \"#{dest}\"")
|
30
|
+
unless $dry_run
|
31
|
+
`gpg -d --batch --yes -q --passphrase-file "#{$remote.pass_file}" -o "#{dest}" "#{src}"`
|
32
|
+
::File.delete(src)
|
33
|
+
::Opee::Env.warn("Downloaded and decrypted \"#{dest}\"")
|
34
|
+
end
|
35
|
+
@fixer.ask(:collect, dest, :crypter) unless @fixer.nil?
|
36
|
+
@crypt_queue.ask(:ready, self)
|
37
|
+
end
|
38
|
+
|
39
|
+
end # Crypter
|
40
|
+
end # ODisk
|
data/lib/odisk/diff.rb
ADDED
@@ -0,0 +1,81 @@
|
|
1
|
+
|
2
|
+
module ODisk
|
3
|
+
class Diff
|
4
|
+
# local Info Object
|
5
|
+
attr_accessor :local
|
6
|
+
# remote Info Object
|
7
|
+
attr_accessor :remote
|
8
|
+
|
9
|
+
attr_accessor :sub_diffs
|
10
|
+
|
11
|
+
def self.dir_diff(l, r, recursive=false)
|
12
|
+
ld = Digest.create(l, nil)
|
13
|
+
rd = Digest.create(r, nil)
|
14
|
+
diffs = digest_diff(ld, rd)
|
15
|
+
if recursive
|
16
|
+
lh = ld.entries_hash()
|
17
|
+
rh = rd.entries_hash()
|
18
|
+
keys = lh.keys | rh.keys
|
19
|
+
keys.each do |k|
|
20
|
+
next unless lh[k].is_a?(::ODisk::Dir) || rh[k].is_a?(::ODisk::Dir)
|
21
|
+
unless (sd = dir_diff(::File.join(l, k), ::File.join(r, k))).nil? || sd.empty?
|
22
|
+
if (d = diffs[k]).nil?
|
23
|
+
d = self.new(lh[k], rh[k], sd)
|
24
|
+
else
|
25
|
+
d.sub_diffs = sd
|
26
|
+
end
|
27
|
+
diffs[k] = d
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
diffs
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.digest_diff(ld, rd)
|
35
|
+
diffs = {}
|
36
|
+
lh = ld.entries_hash()
|
37
|
+
rh = rd.entries_hash()
|
38
|
+
keys = lh.keys | rh.keys
|
39
|
+
keys.each do |k|
|
40
|
+
le = lh[k]
|
41
|
+
re = rh[k]
|
42
|
+
diffs[k] = self.new(le, re) unless le == re
|
43
|
+
end
|
44
|
+
diffs
|
45
|
+
end
|
46
|
+
|
47
|
+
def initialize(local, remote, sub_diffs={})
|
48
|
+
@local = local
|
49
|
+
@remote = remote
|
50
|
+
@sub_diffs = sub_diffs
|
51
|
+
end
|
52
|
+
|
53
|
+
def fill_hash(prefix, h, sym=false)
|
54
|
+
[:class, :name, :owner, :group, :mode, :mtime, :size, :target].each do |m|
|
55
|
+
next unless @local.respond_to?(m) and @remote.respond_to?(m)
|
56
|
+
lv = @local.send(m)
|
57
|
+
rv = @remote.send(m)
|
58
|
+
if sym
|
59
|
+
k = m
|
60
|
+
else
|
61
|
+
k = @local.name + '.' + m.to_s
|
62
|
+
k = prefix + '.' + k unless prefix.nil?
|
63
|
+
end
|
64
|
+
h[k] = [lv, rv] unless lv == rv unless (:mtime == m && !@remote.is_a?(::ODisk::File))
|
65
|
+
end
|
66
|
+
pre = prefix.nil? ? @local.name : (prefix + '.' + @local.name)
|
67
|
+
@sub_diffs.each do |name,d|
|
68
|
+
d.fill_hash(pre, h)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def to_s()
|
73
|
+
s = ''
|
74
|
+
h = {}
|
75
|
+
fill_hash(nil, h)
|
76
|
+
h.each { |k,v| s << "#{k}: #{v[0]} vs #{v[1]}\n" }
|
77
|
+
s
|
78
|
+
end
|
79
|
+
|
80
|
+
end # Diff
|
81
|
+
end # ODisk
|
data/lib/odisk/digest.rb
ADDED
@@ -0,0 +1,85 @@
|
|
1
|
+
|
2
|
+
module ODisk
|
3
|
+
class Digest
|
4
|
+
|
5
|
+
attr_accessor :version
|
6
|
+
|
7
|
+
attr_accessor :entries
|
8
|
+
|
9
|
+
attr_reader :top_path
|
10
|
+
|
11
|
+
def self.create(top, rel_path)
|
12
|
+
path = (rel_path.nil? || rel_path.empty?) ? top : ::File.join(top, rel_path)
|
13
|
+
raise "#{path} is not a directory" unless ::File.directory?(path)
|
14
|
+
d = self.new(rel_path)
|
15
|
+
::Dir.foreach(path) do |filename|
|
16
|
+
next if filename.start_with?('.')
|
17
|
+
child_path = ::File.join(path, filename)
|
18
|
+
c = self.create_info(child_path, filename, top)
|
19
|
+
d.entries << c
|
20
|
+
end
|
21
|
+
d
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.create_info(path, filename=nil, top=nil)
|
25
|
+
top = $local_top if top.nil?
|
26
|
+
filename = ::File.basename(path) if filename.nil?
|
27
|
+
stat = ::File.lstat(path)
|
28
|
+
if stat.directory?
|
29
|
+
c = ::ODisk::Dir.new(filename)
|
30
|
+
elsif stat.symlink?
|
31
|
+
c = ::ODisk::Link.new(filename)
|
32
|
+
c.target = ::File.readlink(path)
|
33
|
+
c.target = c.target[top.size + 1..-1] if c.target.start_with?(top)
|
34
|
+
elsif stat.file?
|
35
|
+
c = ::ODisk::File.new(filename)
|
36
|
+
c.size = stat.size()
|
37
|
+
else
|
38
|
+
raise "file type of #{job.path} is not supported"
|
39
|
+
end
|
40
|
+
c.mtime = stat.mtime()
|
41
|
+
c.mode = stat.mode & 0777
|
42
|
+
begin
|
43
|
+
c.owner = Etc.getpwuid(stat.uid).name
|
44
|
+
rescue
|
45
|
+
c.owner = stat.uid
|
46
|
+
end
|
47
|
+
begin
|
48
|
+
c.group = Etc.getgrgid(stat.gid).name
|
49
|
+
rescue
|
50
|
+
c.group = stat.gid
|
51
|
+
end
|
52
|
+
c
|
53
|
+
end
|
54
|
+
|
55
|
+
def initialize(top_path)
|
56
|
+
@top_path = top_path
|
57
|
+
@version = 0
|
58
|
+
@entries = []
|
59
|
+
end
|
60
|
+
|
61
|
+
def [](name)
|
62
|
+
@entries.each { |e| return e if name == e.name }
|
63
|
+
nil
|
64
|
+
end
|
65
|
+
|
66
|
+
def empty?()
|
67
|
+
@entries.empty?
|
68
|
+
end
|
69
|
+
|
70
|
+
def delete(name)
|
71
|
+
@entries.delete_if { |e| name == e.name }
|
72
|
+
end
|
73
|
+
|
74
|
+
def sub_dirs()
|
75
|
+
@entries.select { |e| e.is_a?(::ODisk::Dir) }
|
76
|
+
end
|
77
|
+
|
78
|
+
def entries_hash()
|
79
|
+
h = {}
|
80
|
+
@entries.each { |e| h[e.name] = e }
|
81
|
+
h
|
82
|
+
end
|
83
|
+
|
84
|
+
end # Digest
|
85
|
+
end # ODisk
|
@@ -0,0 +1,33 @@
|
|
1
|
+
|
2
|
+
require 'etc'
|
3
|
+
|
4
|
+
module ODisk
|
5
|
+
class Digester < ::Opee::Actor
|
6
|
+
|
7
|
+
def initialize(options={})
|
8
|
+
super(options)
|
9
|
+
end
|
10
|
+
|
11
|
+
def set_options(options)
|
12
|
+
super(options)
|
13
|
+
@collector = options[:collector]
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
def create(job)
|
19
|
+
path = ::File.join($local_top, job.path)
|
20
|
+
if ::File.directory?(path)
|
21
|
+
::Opee::Env.info("create digest for #{path}")
|
22
|
+
d = ::ODisk::Digest.create($local_top, job.path)
|
23
|
+
else
|
24
|
+
::Opee::Env.info("#{path} does not exist, no digest")
|
25
|
+
d = nil
|
26
|
+
end
|
27
|
+
job.current_digest = d
|
28
|
+
@collector.ask(:collect, job, :digester) unless @collector.nil?
|
29
|
+
::Opee::Env.debug("#{Oj.dump(d, indent: 2)})")
|
30
|
+
end
|
31
|
+
|
32
|
+
end # Digester
|
33
|
+
end # ODisk
|
data/lib/odisk/dir.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
|
2
|
+
module ODisk
|
3
|
+
class DirSyncJob < ::Opee::Job
|
4
|
+
|
5
|
+
# relative path from top
|
6
|
+
attr_reader :path
|
7
|
+
# only set by the Digester
|
8
|
+
attr_accessor :current_digest
|
9
|
+
# only set by the SyncStarter
|
10
|
+
attr_accessor :previous_digest
|
11
|
+
# only set by the Fetcher
|
12
|
+
attr_accessor :remote_digest
|
13
|
+
# new digest to be set locally and remotely
|
14
|
+
attr_accessor :new_digest
|
15
|
+
|
16
|
+
def initialize(path)
|
17
|
+
@path = path
|
18
|
+
@current_digest = nil
|
19
|
+
@previous_digest = nil
|
20
|
+
@remote_digest = nil
|
21
|
+
@new_digest = nil
|
22
|
+
end
|
23
|
+
|
24
|
+
end # DirSyncJob
|
25
|
+
end # ODisk
|
@@ -0,0 +1,54 @@
|
|
1
|
+
|
2
|
+
begin
|
3
|
+
v = $VERBOSE
|
4
|
+
$VERBOSE = false
|
5
|
+
require 'net/ssh'
|
6
|
+
require 'net/sftp'
|
7
|
+
$VERBOSE = v
|
8
|
+
end
|
9
|
+
require 'oj'
|
10
|
+
|
11
|
+
module ODisk
|
12
|
+
class Fetcher < ::Opee::Actor
|
13
|
+
|
14
|
+
def initialize(options={})
|
15
|
+
@ftp = nil
|
16
|
+
super(options)
|
17
|
+
end
|
18
|
+
|
19
|
+
def set_options(options)
|
20
|
+
super(options)
|
21
|
+
@collector = options[:collector]
|
22
|
+
end
|
23
|
+
|
24
|
+
def close()
|
25
|
+
@ftp.close_channel() unless @ftp.nil?
|
26
|
+
@ftp = nil
|
27
|
+
super()
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def fetch(job)
|
33
|
+
top = (job.path.nil? || job.path.empty?) ? $remote.dir : ::File.join($remote.dir, job.path)
|
34
|
+
path = ::File.join(top, '.odisk', 'digest.json')
|
35
|
+
::Opee::Env.info("fetch digest \"#{path}\"")
|
36
|
+
@ftp = Net::SFTP.start($remote.host, $remote.user) if @ftp.nil?
|
37
|
+
begin
|
38
|
+
json = @ftp.download!(path)
|
39
|
+
files = @ftp.dir.entries(top).map {|e| e.name }
|
40
|
+
job.remote_digest = Oj.load(json, mode: :object)
|
41
|
+
missing = []
|
42
|
+
job.remote_digest.entries.each { |e| missing << e.name unless files.include?(e.name + '.gpg') || files.include?(e.name) }
|
43
|
+
unless ::ODisk::Planner::Step::REMOTE == $master
|
44
|
+
missing.each { |name| job.remote_digest.delete(name) }
|
45
|
+
end
|
46
|
+
rescue Exception
|
47
|
+
job.remote_digest = nil
|
48
|
+
end
|
49
|
+
@collector.ask(:collect, job, :fetcher) unless @collector.nil?
|
50
|
+
::Opee::Env.debug("#{Oj.dump(job.remote_digest, indent: 2)})")
|
51
|
+
end
|
52
|
+
|
53
|
+
end # Fetcher
|
54
|
+
end # ODisk
|
data/lib/odisk/file.rb
ADDED
data/lib/odisk/info.rb
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
|
2
|
+
module ODisk
|
3
|
+
class Info
|
4
|
+
|
5
|
+
NORMAL = 0 # or nil
|
6
|
+
ERROR = 1
|
7
|
+
REMOTE = 2
|
8
|
+
LOCAL = 3
|
9
|
+
|
10
|
+
attr_accessor :name
|
11
|
+
attr_accessor :owner
|
12
|
+
attr_accessor :group
|
13
|
+
attr_accessor :mode
|
14
|
+
attr_accessor :mtime
|
15
|
+
attr_accessor :master
|
16
|
+
attr_accessor :removed
|
17
|
+
|
18
|
+
def initialize(name)
|
19
|
+
@name = name
|
20
|
+
@owner = nil
|
21
|
+
@group = nil
|
22
|
+
@mode = 0
|
23
|
+
@mtime = nil
|
24
|
+
@master = nil
|
25
|
+
@removed = false
|
26
|
+
end
|
27
|
+
|
28
|
+
def eql?(o)
|
29
|
+
return false unless o.class == self.class
|
30
|
+
# don't check master flag
|
31
|
+
(@name == o.name &&
|
32
|
+
@owner == o.owner &&
|
33
|
+
@group == o.group &&
|
34
|
+
@mode == o.mode &&
|
35
|
+
@removed == o.removed)
|
36
|
+
end
|
37
|
+
alias == eql?
|
38
|
+
|
39
|
+
end # Info
|
40
|
+
end # ODisk
|
data/lib/odisk/link.rb
ADDED
@@ -0,0 +1,309 @@
|
|
1
|
+
|
2
|
+
require 'oj'
|
3
|
+
|
4
|
+
module ODisk
|
5
|
+
class Planner < ::Opee::Collector
|
6
|
+
|
7
|
+
def initialize(options={})
|
8
|
+
super(options)
|
9
|
+
end
|
10
|
+
|
11
|
+
def set_options(options)
|
12
|
+
super(options)
|
13
|
+
@dir_queue = options[:dir_queue]
|
14
|
+
@copy_queue = options[:copy_queue]
|
15
|
+
@crypt_queue = options[:crypt_queue]
|
16
|
+
@inputs = options[:inputs]
|
17
|
+
@fixer = options[:fixer]
|
18
|
+
end
|
19
|
+
|
20
|
+
# master can be Step::LOCAL or Step::REMOTE and forces direction
|
21
|
+
def self.sync_steps(pd, ld, rd, master=nil)
|
22
|
+
steps = {}
|
23
|
+
lh = {}
|
24
|
+
rh = {}
|
25
|
+
ph = {}
|
26
|
+
ld.entries.each { |e| lh[e.name] = e }
|
27
|
+
rd.entries.each { |e| rh[e.name] = e }
|
28
|
+
pd.entries.each { |e| ph[e.name] = e } unless pd.nil?
|
29
|
+
keys = lh.keys | rh.keys
|
30
|
+
keys.each do |name|
|
31
|
+
le = lh[name]
|
32
|
+
re = rh[name]
|
33
|
+
if re.nil?
|
34
|
+
if Step::REMOTE == master
|
35
|
+
steps[name] = Step.new(name, Step::LOCAL, Step::REMOVE)
|
36
|
+
elsif le.is_a?(::ODisk::File)
|
37
|
+
steps[name] = Step.new(name, Step::LOCAL, Step::COPY)
|
38
|
+
elsif le.is_a?(::ODisk::Dir)
|
39
|
+
steps[name] = Step.new(name, Step::LOCAL, Step::DIGEST)
|
40
|
+
end
|
41
|
+
elsif le.nil?
|
42
|
+
if Step::LOCAL == master
|
43
|
+
steps[name] = Step.new(name, Step::REMOTE, Step::REMOVE)
|
44
|
+
elsif re.is_a?(::ODisk::File)
|
45
|
+
steps[name] = Step.new(name, Step::REMOTE, Step::COPY)
|
46
|
+
elsif re.is_a?(::ODisk::Dir)
|
47
|
+
steps[name] = Step.new(name, Step::REMOTE, Step::DIGEST)
|
48
|
+
end
|
49
|
+
elsif le != re # both exist but are different
|
50
|
+
if le.class != re.class
|
51
|
+
::Opee::Env.error("Conflict syncing #{ld.top_path}/#{name}. Local and remote types do not match.")
|
52
|
+
steps[name] = Step.new(name, Step::LOCAL, Step::ERROR)
|
53
|
+
elsif le.is_a?(::ODisk::File) || le.is_a?(::ODisk::Link)
|
54
|
+
op = le.is_a?(::ODisk::File) ? Step::COPY : Step::LINK
|
55
|
+
if Step::LOCAL == master
|
56
|
+
steps[name] = Step.new(name, Step::LOCAL, op) if le.is_a?(::ODisk::File)
|
57
|
+
elsif Step::REMOTE == master
|
58
|
+
steps[name] = Step.new(name, Step::REMOTE, op) if re.is_a?(::ODisk::File)
|
59
|
+
elsif le.removed
|
60
|
+
# TBD
|
61
|
+
elsif re.removed
|
62
|
+
# TBD
|
63
|
+
elsif le.mtime > re.mtime
|
64
|
+
pe = ph[name]
|
65
|
+
if pe.nil? || pe.mtime == re.mtime
|
66
|
+
# Don't know if the content or the stats changed so copy it.
|
67
|
+
steps[name] = Step.new(name, Step::LOCAL, op) if le.is_a?(::ODisk::File)
|
68
|
+
else
|
69
|
+
::Opee::Env.error("Conflict syncing #{ld.top_path}/#{name}. Both local and remote have changed.")
|
70
|
+
steps[name] = Step.new(name, Step::LOCAL, Step::ERROR)
|
71
|
+
end
|
72
|
+
elsif le.mtime < re.mtime
|
73
|
+
# Don't know if the content or the stats changed so copy it.
|
74
|
+
steps[name] = Step.new(name, Step::REMOTE, op) if re.is_a?(::ODisk::File)
|
75
|
+
else # same times but different can't be good
|
76
|
+
::Opee::Env.error("Conflict syncing #{ld.top_path}/#{name}. Both local and remote have changed.")
|
77
|
+
steps[name] = Step.new(name, Step::LOCAL, Step::ERROR)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
steps.empty? ? nil : steps
|
83
|
+
end
|
84
|
+
|
85
|
+
private
|
86
|
+
|
87
|
+
def update_token(job, token, path_id)
|
88
|
+
token = [] if token.nil?
|
89
|
+
token << path_id
|
90
|
+
token
|
91
|
+
end
|
92
|
+
|
93
|
+
def complete?(job, token)
|
94
|
+
result = !token.nil? && token.size == @inputs.size && token.sort == @inputs.sort
|
95
|
+
::Opee::Env.info("complete?(#{job.key()}, #{token}) => #{result}")
|
96
|
+
#::Opee::Env.debug("#{Oj.dump(job, indent: 2)}")
|
97
|
+
result
|
98
|
+
end
|
99
|
+
|
100
|
+
def keep_going(job)
|
101
|
+
path = ::File.join($local_top, job.path)
|
102
|
+
odisk_dir = ::File.join(path, '.odisk')
|
103
|
+
`mkdir -p "#{odisk_dir}"` unless ::File.directory?(odisk_dir) && !$dry_run
|
104
|
+
if @copy_queue.nil? && @crypt_queue.nil? && !job.current_digest.nil? # digests_only
|
105
|
+
if job.previous_digest.nil?
|
106
|
+
job.current_digest.version = 1
|
107
|
+
else
|
108
|
+
job.current_digest.version = job.previous_digest.version + 1
|
109
|
+
Oj.to_file(::File.join(odisk_dir, 'digest.old.json'), job.previous_digest, indent: 2)
|
110
|
+
end
|
111
|
+
Oj.to_file(::File.join(odisk_dir, 'digest.json'), job.current_digest, indent: 2)
|
112
|
+
job.current_digest.entries.each do |e|
|
113
|
+
@dir_queue.ask(:add, job.path.empty? ? e.name : ::File.join(job.path, e.name)) if e.is_a?(::ODisk::Dir)
|
114
|
+
end
|
115
|
+
elsif job.remote_digest.nil?
|
116
|
+
process_new(job, odisk_dir)
|
117
|
+
elsif ((job.current_digest.nil? || job.current_digest.empty?) &&
|
118
|
+
(job.previous_digest.nil? || job.previous_digest.empty?))
|
119
|
+
process_down(job, odisk_dir)
|
120
|
+
else
|
121
|
+
process_sync(job, odisk_dir)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
def process_new(job, odisk_dir)
|
126
|
+
job.current_digest.version = 1
|
127
|
+
job.new_digest = job.current_digest
|
128
|
+
# determine update new_digest
|
129
|
+
# write the digest files
|
130
|
+
# TBD if they are the same then don't bother
|
131
|
+
Oj.to_file(::File.join(odisk_dir, 'digest.old.json'), job.previous_digest, indent: 2) unless job.previous_digest.nil?
|
132
|
+
Oj.to_file(::File.join(odisk_dir, 'digest.json'), job.new_digest, indent: 2)
|
133
|
+
|
134
|
+
# get the transfers going for all files
|
135
|
+
job.new_digest.entries.each do |e|
|
136
|
+
path = job.path.empty? ? e.name : ::File.join(job.path, e.name)
|
137
|
+
case e
|
138
|
+
when ::ODisk::Dir
|
139
|
+
@dir_queue.ask(:add, path)
|
140
|
+
when ::ODisk::File
|
141
|
+
local = ::File.join($local_top, path)
|
142
|
+
remote = ::File.join($remote.dir, path)
|
143
|
+
if $remote.encrypt?
|
144
|
+
encrypt_path = (job.path.empty? ?
|
145
|
+
::File.join($local_top, '.odisk', e.name + '.gpg') :
|
146
|
+
::File.join($local_top, job.path, '.odisk', e.name + '.gpg'))
|
147
|
+
@crypt_queue.add_method(:encrypt, local, encrypt_path, remote + '.gpg')
|
148
|
+
else
|
149
|
+
@copy_queue.add_method(:upload, local, remote)
|
150
|
+
end
|
151
|
+
when ::ODisk::Link
|
152
|
+
# nothing to do
|
153
|
+
end
|
154
|
+
end
|
155
|
+
path = job.path.empty? ? ::File.join('.odisk', 'digest.json') : ::File.join(job.path, '.odisk', 'digest.json')
|
156
|
+
local = ::File.join($local_top, path)
|
157
|
+
remote = ::File.join($remote.dir, path)
|
158
|
+
@copy_queue.add_method(:upload, local, remote)
|
159
|
+
end
|
160
|
+
|
161
|
+
def process_down(job, odisk_dir)
|
162
|
+
job.new_digest = job.remote_digest
|
163
|
+
Oj.to_file(::File.join(odisk_dir, 'digest.json'), job.new_digest, indent: 2)
|
164
|
+
full_job_path = job.path.empty? ? $local_top : ::File.join($local_top, job.path)
|
165
|
+
stat_job = StatJob.new(full_job_path, job.new_digest)
|
166
|
+
job.new_digest.entries.each do |e|
|
167
|
+
path = job.path.empty? ? e.name : ::File.join(job.path, e.name)
|
168
|
+
local = ::File.join($local_top, path)
|
169
|
+
case e
|
170
|
+
when ::ODisk::Dir
|
171
|
+
::Dir.mkdir(local) unless $dry_run || ::File.directory?(local)
|
172
|
+
@dir_queue.ask(:add, path)
|
173
|
+
when ::ODisk::File
|
174
|
+
remote = ::File.join($remote.dir, path)
|
175
|
+
stat_job.add_mod(e.name)
|
176
|
+
if $remote.encrypt?
|
177
|
+
encrypt_path = (job.path.empty? ?
|
178
|
+
::File.join($local_top, '.odisk', e.name + '.gpg') :
|
179
|
+
::File.join($local_top, job.path, '.odisk', e.name + '.gpg'))
|
180
|
+
@copy_queue.add_method(:download, remote + '.gpg', encrypt_path, local)
|
181
|
+
else
|
182
|
+
@copy_queue.add_method(:download, remote, local, nil)
|
183
|
+
end
|
184
|
+
when ::ODisk::Link
|
185
|
+
target = e.target
|
186
|
+
target = ::File.join($local_top, e.target) unless '/' == target[0]
|
187
|
+
::Opee::Env.info("symlink \"#{local}\" -> \"#{target}\"}")
|
188
|
+
::File.symlink(target, local) unless $dry_run || ::File.exists?(local)
|
189
|
+
end
|
190
|
+
end
|
191
|
+
@fixer.ask(:collect, stat_job, :planner) unless @fixer.nil?
|
192
|
+
end
|
193
|
+
|
194
|
+
def process_sync(job, odisk_dir)
|
195
|
+
dirs = []
|
196
|
+
job.current_digest.entries.each { |e| dirs << e.name unless !e.is_a?(::ODisk::Dir) || dirs.include?(e.name) }
|
197
|
+
job.remote_digest.entries.each { |e| dirs << e.name unless !e.is_a?(::ODisk::Dir) || dirs.include?(e.name) }
|
198
|
+
dirs.each do |dir|
|
199
|
+
path = job.path.empty? ? dir : ::File.join(job.path, dir)
|
200
|
+
local = ::File.join($local_top, path)
|
201
|
+
::Dir.mkdir(local) unless $dry_run || ::File.directory?(local)
|
202
|
+
@dir_queue.ask(:add, path)
|
203
|
+
end
|
204
|
+
steps = self.class.sync_steps(job.previous_digest, job.current_digest, job.remote_digest, $master)
|
205
|
+
#puts "*** steps for #{job.path}: #{steps}"
|
206
|
+
return if steps.nil?
|
207
|
+
|
208
|
+
Oj.to_file(::File.join(odisk_dir, 'digest.old.json'), job.previous_digest, indent: 2) unless job.previous_digest.nil?
|
209
|
+
nrh = {} # fill with new digest entries for remote
|
210
|
+
nlh = {} # fill with new digest entries for local
|
211
|
+
full_job_path = job.path.empty? ? $local_top : ::File.join($local_top, job.path)
|
212
|
+
stat_job = StatJob.new(full_job_path, nil)
|
213
|
+
job.remote_digest.entries.each { |e| nrh[e.name] = e }
|
214
|
+
job.current_digest.entries.each { |e| nlh[e.name] = e }
|
215
|
+
=begin
|
216
|
+
if job.previous_digest.nil?
|
217
|
+
job.current_digest.entries.each { |e| nlh[e.name] = e }
|
218
|
+
else
|
219
|
+
job.previous_digest.entries.each { |e| nlh[e.name] = e }
|
220
|
+
end
|
221
|
+
=end
|
222
|
+
steps.values.each do |s|
|
223
|
+
stat_job.add_mod(s.name) unless Step::REMOVE == s.op || Step::DIGEST == s.op
|
224
|
+
case s.op
|
225
|
+
when Step::COPY
|
226
|
+
path = job.path.empty? ? s.name : ::File.join(job.path, s.name)
|
227
|
+
local = ::File.join($local_top, path)
|
228
|
+
remote = ::File.join($remote.dir, path)
|
229
|
+
encrypt_path = (job.path.empty? ?
|
230
|
+
::File.join($local_top, '.odisk', s.name + '.gpg') :
|
231
|
+
::File.join($local_top, job.path, '.odisk', s.name + '.gpg'))
|
232
|
+
if Step::REMOTE == s.master
|
233
|
+
if $remote.encrypt?
|
234
|
+
@copy_queue.add_method(:download, remote + '.gpg', encrypt_path, local)
|
235
|
+
else
|
236
|
+
@copy_queue.add_method(:download, remote, local, nil)
|
237
|
+
end
|
238
|
+
e = job.remote_digest[s.name]
|
239
|
+
else
|
240
|
+
if $remote.encrypt?
|
241
|
+
@crypt_queue.add_method(:encrypt, local, encrypt_path, remote + '.gpg')
|
242
|
+
else
|
243
|
+
@copy_queue.add_method(:upload, local, remote)
|
244
|
+
end
|
245
|
+
e = job.current_digest[s.name]
|
246
|
+
end
|
247
|
+
nrh[e.name] = e
|
248
|
+
nlh[e.name] = e
|
249
|
+
when Step::REMOVE
|
250
|
+
# TBD
|
251
|
+
when Step::DIGEST
|
252
|
+
e = (Step::REMOTE == s.master) ? job.remote_digest[s.name] : job.current_digest[s.name]
|
253
|
+
nrh[e.name] = e
|
254
|
+
nlh[e.name] = e
|
255
|
+
when Step::LINK
|
256
|
+
if Step::REMOTE == s.master
|
257
|
+
#TBD create link
|
258
|
+
end
|
259
|
+
when Step::ERROR
|
260
|
+
# TBD
|
261
|
+
end
|
262
|
+
end
|
263
|
+
nrd = Digest.new(job.remote_digest.top_path)
|
264
|
+
nld = Digest.new(job.current_digest.top_path)
|
265
|
+
# fill in digest entries from hashs
|
266
|
+
nrd.entries = nrh.values
|
267
|
+
nld.entries = nlh.values
|
268
|
+
|
269
|
+
job.new_digest = nrd
|
270
|
+
remote_digest_path = ::File.join(odisk_dir, 'digest.remote.json')
|
271
|
+
Oj.to_file(::File.join(odisk_dir, 'digest.json'), nld, indent: 2)
|
272
|
+
Oj.to_file(remote_digest_path, job.new_digest, indent: 2)
|
273
|
+
path = job.path.empty? ? ::File.join('.odisk', 'digest.json') : ::File.join(job.path, '.odisk', 'digest.json')
|
274
|
+
@copy_queue.add_method(:upload, remote_digest_path, ::File.join($remote.dir, path)) unless Step::REMOTE == $master
|
275
|
+
|
276
|
+
stat_job.digest = nld
|
277
|
+
@fixer.ask(:collect, stat_job, :planner) unless @fixer.nil?
|
278
|
+
end
|
279
|
+
|
280
|
+
class Step
|
281
|
+
# op values
|
282
|
+
STATS = 0
|
283
|
+
REMOVE = 1
|
284
|
+
COPY = 2
|
285
|
+
LINK = 3
|
286
|
+
DIGEST = 4
|
287
|
+
ERROR = 5
|
288
|
+
|
289
|
+
# location of master
|
290
|
+
LOCAL = true
|
291
|
+
REMOTE = false
|
292
|
+
|
293
|
+
attr_reader :name
|
294
|
+
attr_reader :master
|
295
|
+
attr_reader :op
|
296
|
+
|
297
|
+
def initialize(name, master, op)
|
298
|
+
@name = name
|
299
|
+
@master = master
|
300
|
+
@op = op
|
301
|
+
end
|
302
|
+
|
303
|
+
def to_s()
|
304
|
+
"<Step name=#{@name} master=#{@master ? 'LOCAL' : 'REMOTE'} op=#{['STATS', 'REMOVE', 'COPY', 'LINK', 'ERROR'][@op]}>"
|
305
|
+
end
|
306
|
+
end # Step
|
307
|
+
|
308
|
+
end # Planner
|
309
|
+
end # ODisk
|