odisk 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +27 -0
- data/README.md +150 -0
- data/bin/odisk +184 -0
- data/lib/odisk.rb +46 -0
- data/lib/odisk/copier.rb +74 -0
- data/lib/odisk/crypter.rb +40 -0
- data/lib/odisk/diff.rb +81 -0
- data/lib/odisk/digest.rb +85 -0
- data/lib/odisk/digester.rb +33 -0
- data/lib/odisk/dir.rb +10 -0
- data/lib/odisk/dirsyncjob.rb +25 -0
- data/lib/odisk/fetcher.rb +54 -0
- data/lib/odisk/file.rb +18 -0
- data/lib/odisk/info.rb +40 -0
- data/lib/odisk/link.rb +18 -0
- data/lib/odisk/planner.rb +309 -0
- data/lib/odisk/remote.rb +45 -0
- data/lib/odisk/statfixer.rb +102 -0
- data/lib/odisk/statjob.rb +32 -0
- data/lib/odisk/syncjob.rb +25 -0
- data/lib/odisk/syncstarter.rb +37 -0
- data/lib/odisk/version.rb +5 -0
- metadata +135 -0
data/LICENSE
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
Copyright (c) 2012, Peter Ohler
|
2
|
+
All rights reserved.
|
3
|
+
|
4
|
+
Redistribution and use in source and binary forms, with or without
|
5
|
+
modification, are permitted provided that the following conditions are met:
|
6
|
+
|
7
|
+
- Redistributions of source code must retain the above copyright notice, this
|
8
|
+
list of conditions and the following disclaimer.
|
9
|
+
|
10
|
+
- Redistributions in binary form must reproduce the above copyright notice,
|
11
|
+
this list of conditions and the following disclaimer in the documentation
|
12
|
+
and/or other materials provided with the distribution.
|
13
|
+
|
14
|
+
- Neither the name of Peter Ohler nor the names of its contributors may be
|
15
|
+
used to endorse or promote products derived from this software without
|
16
|
+
specific prior written permission.
|
17
|
+
|
18
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
19
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
20
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
21
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
22
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
23
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
24
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
25
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
26
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
27
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
data/README.md
ADDED
@@ -0,0 +1,150 @@
|
|
1
|
+
oDisk
|
2
|
+
=====
|
3
|
+
|
4
|
+
Remote Encrypted File Synchronization, oDisk
|
5
|
+
|
6
|
+
oDisk is a file synchronization application. The need for oDisk came from the
|
7
|
+
discontinuation of Apple's iDisk. oDisk provides a means to backup to a remote
|
8
|
+
host. oDisk encrypts and compresses backups to save space and provide a layer
|
9
|
+
of security not found in iDisk or other backup schemes. Backups are stored as
|
10
|
+
individual files to provide finer granularity in recover if everything else
|
11
|
+
falls apart.
|
12
|
+
|
13
|
+
oDisk is also a demonstration of the [Opee](http://www.ohler.com/opee) gem
|
14
|
+
which utilizes an alternative approach to dealing with multiple threads.
|
15
|
+
|
16
|
+
## <a name="source">Source</a>
|
17
|
+
|
18
|
+
*GitHub* *repo*: https://github.com/ohler55/odisk
|
19
|
+
|
20
|
+
*RubyGems* *repo*: https://rubygems.org/gems/odisk
|
21
|
+
|
22
|
+
## <a name="links">Links of Interest</a>
|
23
|
+
|
24
|
+
[Object-based Parallel Evaluation Environment](http://www.ohler.com/opee) the gem oDisk is built on.
|
25
|
+
|
26
|
+
## <a name="release">Release Notes</a>
|
27
|
+
|
28
|
+
### Release 0.2.0
|
29
|
+
|
30
|
+
- Renamed Orefs to oDisk.
|
31
|
+
|
32
|
+
# Plans and Notes
|
33
|
+
|
34
|
+
oDisk has only recently been opened up for public viewing. It is barely ready
|
35
|
+
for use. I am using it to backup some financial records but I am also keeping
|
36
|
+
copies on more than one computer in case something fails. So far the basic
|
37
|
+
backup functionality works with backups encrypted and compressed on a remote
|
38
|
+
server. Adding new files and making modifications works just fine. Removing
|
39
|
+
files has not been implemented yet and changing ownership or mode only take
|
40
|
+
effect if the file is touched as well. Feel free to give it a try and let me
|
41
|
+
know when you run into bugs.
|
42
|
+
|
43
|
+
- Support exclusion of file and directories
|
44
|
+
- specify on command line and create .odisk/exclude file
|
45
|
+
- use ::File.fnmatch(pattern, path, ::File::FNM_DOTMATCH)
|
46
|
+
- pass array of excludes to digest creation
|
47
|
+
- loosen up current restriction on any file that begins with .
|
48
|
+
|
49
|
+
- Support file removal
|
50
|
+
- Detect file removal based on previous digest or by calling a remove script
|
51
|
+
- Keep record of removals in digest
|
52
|
+
- Note conflicts if modifications are more recent that removal
|
53
|
+
- Use a script to pick remove or keep
|
54
|
+
|
55
|
+
- Handle changes in mode, owner, and group
|
56
|
+
- Compare to previous digest to detect changes
|
57
|
+
- File modification times are not changes by mode, owner, or group changes
|
58
|
+
- Note conflicts if modifications are more recent than remote
|
59
|
+
- Use a script to pick change or keep local version
|
60
|
+
|
61
|
+
- Add progress tracker
|
62
|
+
- Have component pass info to progress actor which will update progress
|
63
|
+
- Planner sends info on all changes along with sizes and comp/crypt flags to progress actor
|
64
|
+
- come up with algoritm for estimating time based on size and flags
|
65
|
+
- comp, crypt, and transfer send status to progress actor
|
66
|
+
- Optional terminal display
|
67
|
+
|
68
|
+
- Support backgroup application with web front end (much later)
|
69
|
+
|
70
|
+
## Installation
|
71
|
+
|
72
|
+
Installation requires Ruby 1.9.3. After that, install the odisk gem.
|
73
|
+
|
74
|
+
gem install odisk
|
75
|
+
|
76
|
+
The net-ssh and net-sftp gems are also needed as are the oj and opee gems.
|
77
|
+
|
78
|
+
gem install net-ssh
|
79
|
+
gem install net-sftp
|
80
|
+
gem install oj
|
81
|
+
gem install opee
|
82
|
+
|
83
|
+
GnuPG must be installed. It can be down loaded from
|
84
|
+
[GnuPG.org](http://www.gnupg.org). Follow the instruction on
|
85
|
+
[GnuPG.org](http://www.gnupg.org) site for installation.
|
86
|
+
|
87
|
+
oDisk is now ready to use. ssh and sftp must be running on the remote site and
|
88
|
+
credentials must be installed so that the user is not prompted for a password
|
89
|
+
when using ssh or sftp.
|
90
|
+
|
91
|
+
## Usage
|
92
|
+
|
93
|
+
After a directory has been selected for backing up *odisk* can be run to
|
94
|
+
copy the directory to a remote server. For purposes of this description the
|
95
|
+
directory to be backed up is *~/backup*.
|
96
|
+
|
97
|
+
A passphrase file will also be needed for encryption. The recommended location
|
98
|
+
is in a *~/.odisk* directory. The contents of the file will be the passphrase
|
99
|
+
for *gpg*.
|
100
|
+
|
101
|
+
Make sure you have a remote server that has an sftp and ssh daemon
|
102
|
+
running. Your credentials must be set in the authorized_keys file. If you can
|
103
|
+
login without a password it is set up correctly.
|
104
|
+
|
105
|
+
The first time *odisk* is used to backup a directory information about the
|
106
|
+
remote server must be provided. After the first time that information is not
|
107
|
+
needed again. Alternatively a *~/.odisk/remotes* file can be set up before
|
108
|
+
running *odisk*.
|
109
|
+
|
110
|
+
To backup to *my_server.remote.com* for user *me* to the *backup* directory on
|
111
|
+
the remote server with a passphrase file of *~/.odisk/backup.pass* the
|
112
|
+
following command should be executed.
|
113
|
+
|
114
|
+
odisk -r me@my_server.remote.com:~/.odisk/backup.pass ~/backup
|
115
|
+
|
116
|
+
A file named *~/backup/.odisk/remote* will be created with the connection
|
117
|
+
information for future invocations so that the next time a backup is made on
|
118
|
+
the *~/backup* directory the command only needs to be:
|
119
|
+
|
120
|
+
odisk ~/backup
|
121
|
+
|
122
|
+
## License:
|
123
|
+
|
124
|
+
Copyright (c) 2012, Peter Ohler
|
125
|
+
All rights reserved.
|
126
|
+
|
127
|
+
Redistribution and use in source and binary forms, with or without
|
128
|
+
modification, are permitted provided that the following conditions are met:
|
129
|
+
|
130
|
+
- Redistributions of source code must retain the above copyright notice, this
|
131
|
+
list of conditions and the following disclaimer.
|
132
|
+
|
133
|
+
- Redistributions in binary form must reproduce the above copyright notice,
|
134
|
+
this list of conditions and the following disclaimer in the documentation
|
135
|
+
and/or other materials provided with the distribution.
|
136
|
+
|
137
|
+
- Neither the name of Peter Ohler nor the names of its contributors may be
|
138
|
+
used to endorse or promote products derived from this software without
|
139
|
+
specific prior written permission.
|
140
|
+
|
141
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
142
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
143
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
144
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
145
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
146
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
147
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
148
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
149
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
150
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
data/bin/odisk
ADDED
@@ -0,0 +1,184 @@
|
|
1
|
+
#!/usr/bin/env ruby -wW1
|
2
|
+
# encoding: UTF-8
|
3
|
+
|
4
|
+
while (i = ARGV.index('-I'))
|
5
|
+
x,path = ARGV.slice!(i, 2)
|
6
|
+
$: << path
|
7
|
+
end
|
8
|
+
|
9
|
+
# TBD tmp for testing
|
10
|
+
$: << ::File.join(::File.dirname(__FILE__), "../../oj/lib")
|
11
|
+
$: << ::File.join(::File.dirname(__FILE__), "../../oj/ext")
|
12
|
+
$: << ::File.join(::File.dirname(__FILE__), "../../opee/lib")
|
13
|
+
$: << ::File.join(::File.dirname(__FILE__), "../lib")
|
14
|
+
|
15
|
+
require 'optparse'
|
16
|
+
begin
|
17
|
+
v = $VERBOSE
|
18
|
+
$VERBOSE = false
|
19
|
+
require 'net/ssh'
|
20
|
+
require 'net/sftp'
|
21
|
+
$VERBOSE = v
|
22
|
+
end
|
23
|
+
require 'opee'
|
24
|
+
require 'oj'
|
25
|
+
require 'odisk'
|
26
|
+
|
27
|
+
$verbose = Logger::WARN
|
28
|
+
$dir = '.'
|
29
|
+
$digests_only = false
|
30
|
+
$dry_run = false
|
31
|
+
$master = nil
|
32
|
+
$plain = false
|
33
|
+
$remote = ::ODisk::Remote.new()
|
34
|
+
$crypter_count = 2
|
35
|
+
$copier_count = 4
|
36
|
+
|
37
|
+
opts = OptionParser.new('Usage: odisk [options] <local_directory>')
|
38
|
+
opts.on('-s', 'decrease verbosity') { $verbose += 1 unless 5 == $verbose }
|
39
|
+
opts.on('-v', 'increase verbosity') { $verbose -= 1 unless 0 == $verbose }
|
40
|
+
opts.on('-d', 'generate digests only') { $digests_only = true }
|
41
|
+
opts.on('-u', 'unencrypted or plain remote files') { $plain = true }
|
42
|
+
opts.on('-n', 'dry run / no modifications') { $dry_run = true }
|
43
|
+
opts.on('-r', '--remote [user@host:dir:pass_file]',
|
44
|
+
String, 'remote user, host, directory, passphrase file for gpg') { |r|
|
45
|
+
$remote.update(r)
|
46
|
+
}
|
47
|
+
opts.on('-m', '--master [local | remote]',
|
48
|
+
String, 'force master to local or remote') { |m|
|
49
|
+
case m
|
50
|
+
when 'local'
|
51
|
+
$master = ::ODisk::Planner::Step::LOCAL
|
52
|
+
when 'remote'
|
53
|
+
$master = ::ODisk::Planner::Step::REMOTE
|
54
|
+
else
|
55
|
+
puts opts.help
|
56
|
+
Process.exit!(0)
|
57
|
+
end
|
58
|
+
}
|
59
|
+
opts.on('-e', '--encrypt-count Integer', Integer, 'number of encryption actors') { |i| $crypter_count = i }
|
60
|
+
opts.on('-c', '--copier-count Integer', Integer, 'number of copier actors') { |i| $copier_count = i }
|
61
|
+
opts.on('-h', '--help', 'Show this display') { puts opts.help; Process.exit!(0) }
|
62
|
+
dirs = opts.parse(ARGV)
|
63
|
+
|
64
|
+
if 1 != dirs.size
|
65
|
+
puts opts.help
|
66
|
+
Process.exit!(0)
|
67
|
+
end
|
68
|
+
$local_top = ::File.expand_path(dirs[0])
|
69
|
+
|
70
|
+
# Walk up the directory tree looking for .odisk directories and a remote.json
|
71
|
+
# in that directory. If the directory does not exist then stop the walk. If
|
72
|
+
# not found check the ~/.odisk/remotes.json file for a matching top.
|
73
|
+
top = $local_top
|
74
|
+
if $ruser.nil? || $rhost.nil? || $rtop.nil?
|
75
|
+
while ::File.directory?(::File.join(top, '.odisk'))
|
76
|
+
rfile = ::File.join(top, '.odisk', 'remote')
|
77
|
+
if ::File.file?(rfile)
|
78
|
+
rstr = ::File.read(rfile).strip()
|
79
|
+
#$remote.pass_file = ::File.expand_path($remote.pass_file) unless $remote.pass_file.nil? || $remote.pass_file.empty?
|
80
|
+
orig_pass_file = $remote.pass_file
|
81
|
+
$remote.update(rstr)
|
82
|
+
$remote.pass_file = ::File.expand_path($remote.pass_file) unless $remote.pass_file.nil? || $remote.pass_file.empty?
|
83
|
+
if !$remote.dir.nil? && !$remote.dir.empty? && top != $local_top
|
84
|
+
$remote.dir = $remote.dir + $local_top[top.size..-1]
|
85
|
+
end
|
86
|
+
if $remote.pass_file != orig_pass_file && !::File.file?($remote.pass_file)
|
87
|
+
$remote.pass_file = ::File.join(top, '.odisk', $remote.pass_file)
|
88
|
+
end
|
89
|
+
break
|
90
|
+
end
|
91
|
+
top = ::File.dirname(top)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
ODisk.info_from_remotes($local_top, $remote) unless $remote.complete?
|
96
|
+
$remote.user = ENV['USER'] if $remote.user.nil?
|
97
|
+
$remote.pass_file = nil if $plain
|
98
|
+
unless $remote.okay?
|
99
|
+
puts "*** user@host:top_dir not specified on command line, in local .odisk/remote file, or in ~/.odisk/remotes"
|
100
|
+
puts opts.help
|
101
|
+
Process.exit!(0)
|
102
|
+
end
|
103
|
+
|
104
|
+
Thread.current[:name] = 'main'
|
105
|
+
::Opee::Env.logger.formatter = proc { |s,t,p,m|
|
106
|
+
s = '' if s.nil?
|
107
|
+
"#{s[0]} [#{t.strftime('%Y-%m-%dT%H:%M:%S.%6N')} ##{p}]: #{m}\n"
|
108
|
+
}
|
109
|
+
::Opee::Env.logger.severity = $verbose
|
110
|
+
|
111
|
+
if Logger::INFO >= $verbose
|
112
|
+
if $digests_only
|
113
|
+
::Opee::Env.info(%{
|
114
|
+
Generate Local Digests
|
115
|
+
local directory: #{::File.expand_path($local_top)}
|
116
|
+
})
|
117
|
+
else
|
118
|
+
::Opee::Env.info(%{
|
119
|
+
Synchronize
|
120
|
+
remote host: #{$remote.host}
|
121
|
+
remote user: #{$remote.user}
|
122
|
+
remote directory: #{$remote.dir}
|
123
|
+
local: #{::File.expand_path($local_top)}
|
124
|
+
dry run: #{$dry_run}
|
125
|
+
master: #{$master.nil? ? 'NONE' : (::ODisk::Planner::Step::LOCAL == $master ? 'LOCAL' : 'REMOTE')}
|
126
|
+
})
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
# If $local_top/.odisk/remote does not exist or is different that what is in $remote, replace it.
|
131
|
+
remote_str = $remote.to_s
|
132
|
+
top_remote_path = ::File.join($local_top, '.odisk', 'remote')
|
133
|
+
if !::File.file?(top_remote_path) || ::File.read(top_remote_path).strip() != remote_str
|
134
|
+
::Opee::Env.info("Writing #{top_remote_path}")
|
135
|
+
unless $dry_run
|
136
|
+
`mkdir -p #{::File.join($local_top, '.odisk')}`
|
137
|
+
::File.open(top_remote_path, 'w') { |f| f.write(remote_str + "\n") }
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
fetcher = nil
|
142
|
+
planner_inputs = [:digester, :starter]
|
143
|
+
|
144
|
+
fixer = ::ODisk::StatFixer.new(:name => 'Fixer')
|
145
|
+
dir_wq = ::Opee::WorkQueue.new(:method => :start, :name => 'DirWorkQueue')
|
146
|
+
if $digests_only
|
147
|
+
copy_wq = nil
|
148
|
+
crypt_wq = nil
|
149
|
+
else
|
150
|
+
copy_wq = ::Opee::AskQueue.new(:name => 'CopyQueue')
|
151
|
+
crypt_wq = ::Opee::AskQueue.new(:name => 'CryptQueue')
|
152
|
+
$copier_count.times { |i|
|
153
|
+
::ODisk::Copier.new(:name => "Copier-#{i}",
|
154
|
+
:crypt_queue => crypt_wq,
|
155
|
+
:copy_queue => copy_wq,
|
156
|
+
:fixer => fixer)
|
157
|
+
}
|
158
|
+
$crypter_count.times { |i|
|
159
|
+
::ODisk::Crypter.new(:name => "Crypter-#{i}",
|
160
|
+
:crypt_queue => crypt_wq,
|
161
|
+
:copy_queue => copy_wq,
|
162
|
+
:fixer => fixer)
|
163
|
+
}
|
164
|
+
planner_inputs << :fetcher
|
165
|
+
end
|
166
|
+
planner = ::ODisk::Planner.new(:name => 'Planner',
|
167
|
+
:dir_queue => dir_wq,
|
168
|
+
:copy_queue => copy_wq,
|
169
|
+
:crypt_queue => crypt_wq,
|
170
|
+
:inputs => planner_inputs,
|
171
|
+
:fixer => fixer)
|
172
|
+
fetcher = ::ODisk::Fetcher.new(:name => 'Fetcher',
|
173
|
+
:collector => planner) unless $digests_only
|
174
|
+
digester = ::ODisk::Digester.new(:name => 'Digester',
|
175
|
+
:collector => planner)
|
176
|
+
starter = ::ODisk::SyncStarter.new(:name => 'Starter',
|
177
|
+
:dir_queue => dir_wq,
|
178
|
+
:digester => digester,
|
179
|
+
:fetcher => fetcher,
|
180
|
+
:collector => planner)
|
181
|
+
|
182
|
+
dir_wq.ask(:add, '')
|
183
|
+
|
184
|
+
::Opee::Env.wait_close()
|
data/lib/odisk.rb
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
|
2
|
+
module ODisk
|
3
|
+
|
4
|
+
def self.info_from_remotes(local_dir, remote)
|
5
|
+
orig_pass_file = remote.pass_file
|
6
|
+
local_dir = ::File.expand_path(local_dir)
|
7
|
+
remotes_file = ::File.join(::File.expand_path('~'), '.odisk', 'remotes')
|
8
|
+
if ::File.file?(remotes_file)
|
9
|
+
::File.readlines(remotes_file).each do |line|
|
10
|
+
l,r = line.split(':', 2)
|
11
|
+
l = ::File.expand_path(l)
|
12
|
+
if l == local_dir || local_dir.start_with?(l + '/')
|
13
|
+
remote.update(r)
|
14
|
+
remote.dir = remote.dir + remote.dir[l.size..-1] if l != local_dir && !remote.dir.nil?
|
15
|
+
if remote.pass_file != orig_pass_file && !::File.file?(remote.pass_file)
|
16
|
+
remote.pass_file = ::File.join(::File.expand_path('~'), '.odisk', remote.pass_file)
|
17
|
+
end
|
18
|
+
break
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
|
26
|
+
# data
|
27
|
+
require 'odisk/remote'
|
28
|
+
require 'odisk/info'
|
29
|
+
require 'odisk/dir'
|
30
|
+
require 'odisk/link'
|
31
|
+
require 'odisk/file'
|
32
|
+
require 'odisk/digest'
|
33
|
+
require 'odisk/diff'
|
34
|
+
# jobs
|
35
|
+
require 'odisk/dirsyncjob'
|
36
|
+
require 'odisk/statjob'
|
37
|
+
require 'odisk/syncjob'
|
38
|
+
# collectors
|
39
|
+
require 'odisk/statfixer'
|
40
|
+
require 'odisk/planner'
|
41
|
+
# actors
|
42
|
+
require 'odisk/copier'
|
43
|
+
require 'odisk/crypter'
|
44
|
+
require 'odisk/digester'
|
45
|
+
require 'odisk/fetcher'
|
46
|
+
require 'odisk/syncstarter'
|
data/lib/odisk/copier.rb
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
|
2
|
+
module ODisk
|
3
|
+
class Copier < ::Opee::Actor
|
4
|
+
|
5
|
+
def initialize(options={})
|
6
|
+
@ftp = nil
|
7
|
+
@ssh = nil
|
8
|
+
super(options)
|
9
|
+
@copy_queue.ask(:ready, self)
|
10
|
+
end
|
11
|
+
|
12
|
+
def set_options(options)
|
13
|
+
super(options)
|
14
|
+
@copy_queue = options[:copy_queue]
|
15
|
+
@crypt_queue = options[:crypt_queue]
|
16
|
+
@fixer = options[:fixer]
|
17
|
+
end
|
18
|
+
|
19
|
+
def close()
|
20
|
+
@ftp.close_channel() unless @ftp.nil?
|
21
|
+
@ftp = nil
|
22
|
+
@ssh.close() unless @ssh.nil?
|
23
|
+
super()
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def upload(local, remote, delete_after=false)
|
29
|
+
::Opee::Env.info("upload \"#{local}\" to \"#{remote}\"#{delete_after ? ' then delete' : ''}")
|
30
|
+
unless $dry_run
|
31
|
+
@ftp = Net::SFTP.start($remote.host, $remote.user) if @ftp.nil?
|
32
|
+
begin
|
33
|
+
@ftp.upload!(local, remote)
|
34
|
+
`rm "#{local}"` if delete_after
|
35
|
+
::Opee::Env.warn("Uploaded \"#{local}\"")
|
36
|
+
rescue Net::SFTP::StatusException => e
|
37
|
+
if Net::SFTP::Constants::StatusCodes::FX_NO_SUCH_FILE == e.code
|
38
|
+
assure_dirs_exist(::File.dirname(remote))
|
39
|
+
retry
|
40
|
+
else
|
41
|
+
::Opee::Env.error("Upload of \"#{local}\" failed: #{e.class}: (#{e.code}) #{e.description}\n #{e.text}\n #{e.response}")
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
@copy_queue.ask(:ready, self)
|
46
|
+
end
|
47
|
+
|
48
|
+
def download(remote, local, decrypt_path=nil)
|
49
|
+
::Opee::Env.info("download #{remote} to #{local}")
|
50
|
+
@ftp = Net::SFTP.start($remote.host, $remote.user) if @ftp.nil?
|
51
|
+
begin
|
52
|
+
@ftp.download!(remote, local)
|
53
|
+
if decrypt_path.nil?
|
54
|
+
@fixer.ask(:collect, local, :copier) unless @fixer.nil?
|
55
|
+
::Opee::Env.warn("Downloaded \"#{local}\"")
|
56
|
+
else
|
57
|
+
@crypt_queue.add_method(:decrypt, local, decrypt_path)
|
58
|
+
end
|
59
|
+
rescue Exception => e
|
60
|
+
::Opee::Env.error("Download of \"#{local}\" failed: #{e.class}: #{e.message}")
|
61
|
+
#::Opee::Env.rescue(e)
|
62
|
+
end
|
63
|
+
@copy_queue.ask(:ready, self)
|
64
|
+
end
|
65
|
+
|
66
|
+
def assure_dirs_exist(dir)
|
67
|
+
::Opee::Env.info("creating remote dir \"#{dir}\"")
|
68
|
+
@ssh = Net::SSH.start($remote.host, $remote.user) if @ssh.nil?
|
69
|
+
out = @ssh.exec!(%{mkdir -p "#{dir}"})
|
70
|
+
raise out unless out.nil? || out.strip().empty?
|
71
|
+
end
|
72
|
+
|
73
|
+
end # Copier
|
74
|
+
end # ODisk
|