odisk 0.2.2 → 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +4 -8
- data/bin/odisk +7 -1
- data/bin/odisk_forget +3 -10
- data/bin/odisk_remove +122 -0
- data/lib/odisk/copier.rb +32 -5
- data/lib/odisk/fetcher.rb +4 -1
- data/lib/odisk/planner.rb +29 -7
- data/lib/odisk/version.rb +1 -1
- metadata +10 -5
data/README.md
CHANGED
@@ -25,9 +25,11 @@ which utilizes an alternative approach to dealing with multiple threads.
|
|
25
25
|
|
26
26
|
## <a name="release">Release Notes</a>
|
27
27
|
|
28
|
-
### Release 0.2.
|
28
|
+
### Release 0.2.3
|
29
29
|
|
30
|
-
- Added odisk_forget which
|
30
|
+
- Added odisk_forget which clears the local and remote digest of a specific file or directory.
|
31
|
+
|
32
|
+
- Added odisk_remove which marks a file or directory as removed so that all file or directory is removed from all local copies.
|
31
33
|
|
32
34
|
# Plans and Notes
|
33
35
|
|
@@ -46,12 +48,6 @@ know when you run into bugs.
|
|
46
48
|
- pass array of ignores to digest creation
|
47
49
|
- loosen up current restriction on any file that begins with .
|
48
50
|
|
49
|
-
- Support file removal
|
50
|
-
- Detect file removal based on previous digest or by calling a remove script
|
51
|
-
- Keep record of removals in digest
|
52
|
-
- Note conflicts if modifications are more recent that removal
|
53
|
-
- Use a script to pick remove or keep
|
54
|
-
|
55
51
|
- Handle changes in mode, owner, and group
|
56
52
|
- Compare to previous digest to detect changes
|
57
53
|
- File modification times are not changed by mode, owner, or group changes
|
data/bin/odisk
CHANGED
@@ -33,12 +33,18 @@ $plain = false
|
|
33
33
|
$remote = ::ODisk::Remote.new()
|
34
34
|
$crypter_count = 2
|
35
35
|
$copier_count = 4
|
36
|
+
$careful = false
|
36
37
|
|
37
|
-
opts = OptionParser.new(
|
38
|
+
opts = OptionParser.new(%{Usage: odisk [options] <local_directory>
|
39
|
+
|
40
|
+
Synchronizes a local directory with a remote directory. The remote directory
|
41
|
+
is encrypted and compressed.
|
42
|
+
})
|
38
43
|
opts.on('-s', 'decrease verbosity') { $verbose += 1 unless 5 == $verbose }
|
39
44
|
opts.on('-v', 'increase verbosity') { $verbose -= 1 unless 0 == $verbose }
|
40
45
|
opts.on('-d', 'generate digests only') { $digests_only = true }
|
41
46
|
opts.on('-u', 'unencrypted or plain remote files') { $plain = true }
|
47
|
+
opts.on('-c', 'careful mode, no local removes') { $careful = true }
|
42
48
|
opts.on('-n', 'dry run / no modifications') { $dry_run = true }
|
43
49
|
opts.on('-r', '--remote [user@host:dir:pass_file]',
|
44
50
|
String, 'remote user, host, directory, passphrase file for gpg') { |r|
|
data/bin/odisk_forget
CHANGED
@@ -59,7 +59,7 @@ unless ODisk.gather_remote_info($local_top, $remote)
|
|
59
59
|
end
|
60
60
|
$remote.pass_file = nil if $plain
|
61
61
|
|
62
|
-
Thread.current[:name] = '
|
62
|
+
Thread.current[:name] = 'forget'
|
63
63
|
::Opee::Env.logger.formatter = proc { |s,t,p,m|
|
64
64
|
s = '' if s.nil?
|
65
65
|
"#{s[0]} [#{t.strftime('%Y-%m-%dT%H:%M:%S.%6N')} ##{p}]: #{m}\n"
|
@@ -67,13 +67,7 @@ Thread.current[:name] = 'main'
|
|
67
67
|
::Opee::Env.logger.severity = $verbose
|
68
68
|
|
69
69
|
if Logger::INFO >= $verbose
|
70
|
-
|
71
|
-
::Opee::Env.info(%{
|
72
|
-
Generate Local Digests
|
73
|
-
local directory: #{::File.expand_path($local_top)}
|
74
|
-
})
|
75
|
-
else
|
76
|
-
::Opee::Env.info(%{
|
70
|
+
::Opee::Env.info(%{
|
77
71
|
Forget "#{$forget_me}"
|
78
72
|
remote host: #{$remote.host}
|
79
73
|
remote user: #{$remote.user}
|
@@ -81,10 +75,9 @@ if Logger::INFO >= $verbose
|
|
81
75
|
local: #{::File.expand_path($local_top)}
|
82
76
|
dry run: #{$dry_run}
|
83
77
|
})
|
84
|
-
end
|
85
78
|
end
|
86
79
|
|
87
|
-
# If $local_top/.odisk/remote does not exist or is different
|
80
|
+
# If $local_top/.odisk/remote does not exist or is different than what is in $remote, replace it.
|
88
81
|
remote_str = $remote.to_s
|
89
82
|
top_remote_path = ::File.join($local_top, '.odisk', 'remote')
|
90
83
|
if !::File.file?(top_remote_path) || ::File.read(top_remote_path).strip() != remote_str
|
data/bin/odisk_remove
ADDED
@@ -0,0 +1,122 @@
|
|
1
|
+
#!/usr/bin/env ruby -wW1
|
2
|
+
# encoding: UTF-8
|
3
|
+
|
4
|
+
while (i = ARGV.index('-I'))
|
5
|
+
x,path = ARGV.slice!(i, 2)
|
6
|
+
$: << path
|
7
|
+
end
|
8
|
+
|
9
|
+
# TBD tmp for testing
|
10
|
+
$: << ::File.join(::File.dirname(__FILE__), "../../oj/lib")
|
11
|
+
$: << ::File.join(::File.dirname(__FILE__), "../../oj/ext")
|
12
|
+
$: << ::File.join(::File.dirname(__FILE__), "../../opee/lib")
|
13
|
+
$: << ::File.join(::File.dirname(__FILE__), "../lib")
|
14
|
+
|
15
|
+
require 'optparse'
|
16
|
+
begin
|
17
|
+
v = $VERBOSE
|
18
|
+
$VERBOSE = false
|
19
|
+
require 'net/ssh'
|
20
|
+
require 'net/sftp'
|
21
|
+
$VERBOSE = v
|
22
|
+
end
|
23
|
+
require 'opee'
|
24
|
+
require 'oj'
|
25
|
+
require 'odisk'
|
26
|
+
|
27
|
+
$verbose = Logger::WARN
|
28
|
+
$dry_run = false
|
29
|
+
$dir = '.'
|
30
|
+
$master = nil
|
31
|
+
$remote = ::ODisk::Remote.new()
|
32
|
+
|
33
|
+
opts = OptionParser.new(%{Usage: odisk_remove [options] <local_directory> <relative_remove_path>
|
34
|
+
|
35
|
+
Will remove data for the file or directory specified from the local file system as well as on the
|
36
|
+
remote server. The path will also be marked indication that the file or directory should be removed
|
37
|
+
from other local directories if that directory is synced with the remote site. To get rid of the
|
38
|
+
remove flag for the path use the oDisk_cleanup command or sync with a local as master.
|
39
|
+
})
|
40
|
+
opts.on('-s', 'decrease verbosity') { $verbose += 1 unless 5 == $verbose }
|
41
|
+
opts.on('-v', 'increase verbosity') { $verbose -= 1 unless 0 == $verbose }
|
42
|
+
opts.on('-n', 'dry run / no modifications') { $dry_run = true }
|
43
|
+
opts.on('-r', '--remote [user@host:dir:pass_file]',
|
44
|
+
String, 'remote user, host, directory, passphrase file for gpg') { |r|
|
45
|
+
$remote.update(r)
|
46
|
+
}
|
47
|
+
opts.on('-h', '--help', 'Show this display') { puts opts.help; Process.exit!(0) }
|
48
|
+
dirs = opts.parse(ARGV)
|
49
|
+
|
50
|
+
if 2 != dirs.size
|
51
|
+
puts opts.help
|
52
|
+
Process.exit!(0)
|
53
|
+
end
|
54
|
+
$local_top = ::File.expand_path(dirs[0])
|
55
|
+
$remove_me = dirs[1] # relative path from top
|
56
|
+
|
57
|
+
# TBD move this to odisk.rb
|
58
|
+
unless ODisk.gather_remote_info($local_top, $remote)
|
59
|
+
puts opts.help
|
60
|
+
Process.exit!(0)
|
61
|
+
end
|
62
|
+
$remote.pass_file = nil if $plain
|
63
|
+
|
64
|
+
Thread.current[:name] = 'remove'
|
65
|
+
::Opee::Env.logger.formatter = proc { |s,t,p,m|
|
66
|
+
s = '' if s.nil?
|
67
|
+
"#{s[0]} [#{t.strftime('%Y-%m-%dT%H:%M:%S.%6N')} ##{p}]: #{m}\n"
|
68
|
+
}
|
69
|
+
::Opee::Env.logger.severity = $verbose
|
70
|
+
|
71
|
+
if Logger::INFO >= $verbose
|
72
|
+
::Opee::Env.info(%{
|
73
|
+
Remove "#{$remove_me}"
|
74
|
+
remote host: #{$remote.host}
|
75
|
+
remote user: #{$remote.user}
|
76
|
+
remote directory: #{$remote.dir}
|
77
|
+
local: #{::File.expand_path($local_top)}
|
78
|
+
dry run: #{$dry_run}
|
79
|
+
})
|
80
|
+
end
|
81
|
+
|
82
|
+
# If $local_top/.odisk/remote does not exist or is different than what is in $remote, replace it.
|
83
|
+
remote_str = $remote.to_s
|
84
|
+
top_remote_path = ::File.join($local_top, '.odisk', 'remote')
|
85
|
+
if !::File.file?(top_remote_path) || ::File.read(top_remote_path).strip() != remote_str
|
86
|
+
::Opee::Env.info("Writing #{top_remote_path}")
|
87
|
+
unless $dry_run
|
88
|
+
`mkdir -p #{::File.join($local_top, '.odisk')}`
|
89
|
+
::File.open(top_remote_path, 'w') { |f| f.write(remote_str + "\n") }
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
path = ::File.join($local_top, $remove_me)
|
94
|
+
local_digest_path = ::File.join(::File.dirname(path), '.odisk', 'digest.json')
|
95
|
+
remove_name = ::File.basename(path)
|
96
|
+
remote_path = ::File.join($remote.dir, $remove_me)
|
97
|
+
remote_digest_path = ::File.join(::File.dirname(remote_path), '.odisk', 'digest.json')
|
98
|
+
|
99
|
+
# Removed the local file.
|
100
|
+
`rm -rf "#{path}"`
|
101
|
+
|
102
|
+
# Remove remove_me from local digest.
|
103
|
+
digest = Oj.load_file(local_digest_path, mode: :object)
|
104
|
+
digest.delete(remove_name)
|
105
|
+
Oj.to_file(local_digest_path, digest, indent: 2) unless $dry_run
|
106
|
+
|
107
|
+
# Removed the remote file or directory.
|
108
|
+
Net::SSH.start($remote.host, $remote.user) do |ssh|
|
109
|
+
ssh.exec!(%{rm -rf "#{remote_path}.gpg"})
|
110
|
+
ssh.exec!(%{rm -rf "#{remote_path}"})
|
111
|
+
end
|
112
|
+
|
113
|
+
# Set the removed flag on the remove_me entry in the remote digest.
|
114
|
+
Net::SFTP.start($remote.host, $remote.user) do |ftp|
|
115
|
+
json = ftp.download!(remote_digest_path)
|
116
|
+
digest = Oj.load(json, mode: :object)
|
117
|
+
unless (entry = digest[remove_name]).nil?
|
118
|
+
entry.removed = true
|
119
|
+
end
|
120
|
+
json = Oj.dump(digest, indent: 2)
|
121
|
+
ftp.open!(remote_digest_path, 'w') { |res| ftp.write!(res[:handle], 0, json) }
|
122
|
+
end
|
data/lib/odisk/copier.rb
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
begin
|
2
|
+
v = $VERBOSE
|
3
|
+
$VERBOSE = false
|
4
|
+
require 'net/ssh'
|
5
|
+
require 'net/sftp'
|
6
|
+
$VERBOSE = v
|
7
|
+
end
|
1
8
|
|
2
9
|
module ODisk
|
3
10
|
# Provides upload and download functionality using sftp and ssh on a single connection.
|
@@ -36,8 +43,12 @@ module ODisk
|
|
36
43
|
::Opee::Env.warn("Uploaded \"#{local}\"")
|
37
44
|
rescue Net::SFTP::StatusException => e
|
38
45
|
if Net::SFTP::Constants::StatusCodes::FX_NO_SUCH_FILE == e.code
|
39
|
-
|
40
|
-
|
46
|
+
begin
|
47
|
+
assure_dirs_exist(::File.dirname(remote))
|
48
|
+
retry
|
49
|
+
rescue Exception => e2
|
50
|
+
::Opee::Env.error("Upload of \"#{local}\" failed: #{e2.class}: #{e2.message}")
|
51
|
+
end
|
41
52
|
else
|
42
53
|
::Opee::Env.error("Upload of \"#{local}\" failed: #{e.class}: (#{e.code}) #{e.description}\n #{e.text}\n #{e.response}")
|
43
54
|
end
|
@@ -64,11 +75,27 @@ module ODisk
|
|
64
75
|
@copy_queue.ask(:ready, self)
|
65
76
|
end
|
66
77
|
|
78
|
+
def remove_local(path)
|
79
|
+
`rm -rf "#{path}"` unless $dry_run
|
80
|
+
::Opee::Env.warn("Removed local \"#{path}\"")
|
81
|
+
end
|
82
|
+
|
83
|
+
def remove_remote(path)
|
84
|
+
unless $dry_runb
|
85
|
+
@ssh = Net::SSH.start($remote.host, $remote.user) if @ssh.nil?
|
86
|
+
out = @ssh.exec!(%{rm -rf "#{path}" "#{path}.gpg"})
|
87
|
+
raise out unless out.nil? || out.strip().empty?
|
88
|
+
end
|
89
|
+
::Opee::Env.warn("Removed remote \"#{path}\"")
|
90
|
+
end
|
91
|
+
|
67
92
|
def assure_dirs_exist(dir)
|
68
93
|
::Opee::Env.info("creating remote dir \"#{dir}\"")
|
69
|
-
|
70
|
-
|
71
|
-
|
94
|
+
unless $dry_run
|
95
|
+
@ssh = Net::SSH.start($remote.host, $remote.user) if @ssh.nil?
|
96
|
+
out = @ssh.exec!(%{mkdir -p "#{dir}"})
|
97
|
+
raise 'Remote ' + out unless out.nil? || out.strip().empty?
|
98
|
+
end
|
72
99
|
end
|
73
100
|
|
74
101
|
end # Copier
|
data/lib/odisk/fetcher.rb
CHANGED
@@ -42,7 +42,10 @@ module ODisk
|
|
42
42
|
files = @ftp.dir.entries(top).map {|e| e.name }
|
43
43
|
job.remote_digest = Oj.load(json, mode: :object)
|
44
44
|
missing = []
|
45
|
-
job.remote_digest.entries.each { |e| missing << e.name unless files.include?(e.name + '.gpg') ||
|
45
|
+
job.remote_digest.entries.each { |e| missing << e.name unless (files.include?(e.name + '.gpg') ||
|
46
|
+
files.include?(e.name) ||
|
47
|
+
e.is_a?(::ODisk::Link) ||
|
48
|
+
e.removed) }
|
46
49
|
unless ::ODisk::Planner::Step::REMOTE == $master
|
47
50
|
missing.each { |name| job.remote_digest.delete(name) }
|
48
51
|
end
|
data/lib/odisk/planner.rb
CHANGED
@@ -54,16 +54,16 @@ module ODisk
|
|
54
54
|
if le.class != re.class
|
55
55
|
::Opee::Env.error("Conflict syncing #{ld.top_path}/#{name}. Local and remote types do not match.")
|
56
56
|
steps[name] = Step.new(name, Step::LOCAL, Step::ERROR)
|
57
|
+
elsif le.removed
|
58
|
+
::Opee::Env.error("Unexpected digest entry for #{ld.top_path}/#{name}. The removed flag is sent in the local digest.")
|
59
|
+
elsif re.removed
|
60
|
+
steps[name] = Step.new(name, Step::LOCAL, Step::REMOVE)
|
57
61
|
elsif le.is_a?(::ODisk::File) || le.is_a?(::ODisk::Link)
|
58
62
|
op = le.is_a?(::ODisk::File) ? Step::COPY : Step::LINK
|
59
63
|
if Step::LOCAL == master
|
60
64
|
steps[name] = Step.new(name, Step::LOCAL, op) if le.is_a?(::ODisk::File)
|
61
65
|
elsif Step::REMOTE == master
|
62
66
|
steps[name] = Step.new(name, Step::REMOTE, op) if re.is_a?(::ODisk::File)
|
63
|
-
elsif le.removed
|
64
|
-
# TBD
|
65
|
-
elsif re.removed
|
66
|
-
# TBD
|
67
67
|
elsif le.mtime > re.mtime
|
68
68
|
pe = ph[name]
|
69
69
|
if pe.nil? || pe.mtime == re.mtime
|
@@ -196,8 +196,21 @@ module ODisk
|
|
196
196
|
|
197
197
|
def process_sync(job, odisk_dir)
|
198
198
|
dirs = []
|
199
|
-
|
200
|
-
|
199
|
+
if Step::REMOTE == $master
|
200
|
+
job.remote_digest.entries.each { |e| dirs << e.name unless !e.is_a?(::ODisk::Dir) || dirs.include?(e.name) }
|
201
|
+
elsif Step::LOCAL == $master
|
202
|
+
job.current_digest.entries.each { |e| dirs << e.name unless !e.is_a?(::ODisk::Dir) || dirs.include?(e.name) }
|
203
|
+
else
|
204
|
+
job.current_digest.entries.each { |e| dirs << e.name unless !e.is_a?(::ODisk::Dir) || dirs.include?(e.name) }
|
205
|
+
job.remote_digest.entries.each do |e|
|
206
|
+
next unless e.is_a?(::ODisk::Dir)
|
207
|
+
if e.removed
|
208
|
+
dirs.delete(e.name)
|
209
|
+
else
|
210
|
+
dirs << e.name unless dirs.include?(e.name)
|
211
|
+
end
|
212
|
+
end
|
213
|
+
end
|
201
214
|
dirs.each do |dir|
|
202
215
|
path = job.path.empty? ? dir : ::File.join(job.path, dir)
|
203
216
|
local = ::File.join($local_top, path)
|
@@ -250,7 +263,16 @@ module ODisk
|
|
250
263
|
nrh[e.name] = e
|
251
264
|
nlh[e.name] = e
|
252
265
|
when Step::REMOVE
|
253
|
-
|
266
|
+
path = job.path.empty? ? s.name : ::File.join(job.path, s.name)
|
267
|
+
if Step::REMOTE == s.master
|
268
|
+
remote = ::File.join($remote.dir, path)
|
269
|
+
@copy_queue.add_method(:remove_remote, remote)
|
270
|
+
nrh.delete(s.name)
|
271
|
+
else
|
272
|
+
local = ::File.join($local_top, path)
|
273
|
+
@copy_queue.add_method(:remove_local, local)
|
274
|
+
nlh.delete(s.name)
|
275
|
+
end
|
254
276
|
when Step::DIGEST
|
255
277
|
e = (Step::REMOTE == s.master) ? job.remote_digest[s.name] : job.current_digest[s.name]
|
256
278
|
nrh[e.name] = e
|
data/lib/odisk/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: odisk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-09-14 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: opee
|
@@ -80,6 +80,7 @@ email: peter@ohler.com
|
|
80
80
|
executables:
|
81
81
|
- odisk
|
82
82
|
- odisk_forget
|
83
|
+
- odisk_remove
|
83
84
|
extensions: []
|
84
85
|
extra_rdoc_files:
|
85
86
|
- README.md
|
@@ -105,8 +106,12 @@ files:
|
|
105
106
|
- lib/odisk.rb
|
106
107
|
- LICENSE
|
107
108
|
- README.md
|
108
|
-
-
|
109
|
-
|
109
|
+
- !binary |-
|
110
|
+
YmluL29kaXNr
|
111
|
+
- !binary |-
|
112
|
+
YmluL29kaXNrX2ZvcmdldA==
|
113
|
+
- !binary |-
|
114
|
+
YmluL29kaXNrX3JlbW92ZQ==
|
110
115
|
homepage: http://www.ohler.com/odisk
|
111
116
|
licenses: []
|
112
117
|
post_install_message:
|
@@ -129,7 +134,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
129
134
|
version: '0'
|
130
135
|
requirements: []
|
131
136
|
rubyforge_project: odisk
|
132
|
-
rubygems_version: 1.8.
|
137
|
+
rubygems_version: 1.8.21
|
133
138
|
signing_key:
|
134
139
|
specification_version: 3
|
135
140
|
summary: Remote Encrypted File Synchronization, oDisk
|