cloudflock 0.7.0 → 0.7.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -13
- data/lib/cloudflock/app/common/rackspace.rb +18 -10
- data/lib/cloudflock/app/common/servers.rb +2 -2
- data/lib/cloudflock/app/files-migrate.rb +82 -61
- data/lib/cloudflock/app.rb +18 -0
- data/lib/cloudflock/remote/files.rb +5 -1
- data/lib/cloudflock.rb +1 -1
- metadata +18 -18
checksums.yaml
CHANGED
@@ -1,15 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
|
5
|
-
data.tar.gz: !binary |-
|
6
|
-
N2Q2M2Y4ZjMyNmE2MDYzNTA3MjRmYTYyNTQ0OWRlNDIyYWY5ODdhOQ==
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 421fa1e64759c72ef901ee7431000811e635e4d1
|
4
|
+
data.tar.gz: 570f428da5e21ae71020b3eaf6d9e93a5c5a9e9c
|
7
5
|
SHA512:
|
8
|
-
metadata.gz:
|
9
|
-
|
10
|
-
NDU1OTczYTBhNzI3OTYzNjhlY2QwNDI5NTQ0NTE4MzExM2NiMjkwNWZmZWUy
|
11
|
-
Y2U1ZjJkZjgyZmJkMTViMzMzMTBlZjljMGUxNzRjN2IxMjM2YTg=
|
12
|
-
data.tar.gz: !binary |-
|
13
|
-
MWY2ZGYyMDUwYTFkYzZmZDhkMDZiMmQ5NDU1ZDA3Y2U0MzhiYWFjNDQ0NTc4
|
14
|
-
NDYyZjM4ZjNhMWVlZTU1OTUxNTk2MzhlODgyMDRhY2UxYjRlYjhjZjgwNzMw
|
15
|
-
NTlhYmFlMjQ5YzQ0NzY3ZjA3MmJmNzJkNWFjMDc1M2VjZTEyZTE=
|
6
|
+
metadata.gz: 444ea7d4d45b7d614f41005f69e606d32d8a92a0e44d0f75d7291e14770acf75af8799cd45d88b216f6c13f6dcb3f67495844c85393ddf3c799c83aab032a76a
|
7
|
+
data.tar.gz: 30714d89645a257a8bc7481d05d5700a8d54f433d484335b7caabc24c3464a35d735e5d15d5f77fbd2e465c0bed6d00f60c2a5a4bdbe309911f8c9b0c0f2ed78
|
@@ -13,23 +13,29 @@ module CloudFlock; module App
|
|
13
13
|
# Returns a Hash containing information necessary to establish an API
|
14
14
|
# session.
|
15
15
|
def define_rackspace_api
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
region.gsub!(/$/, '_AUTH_ENDPOINT')
|
24
|
-
api[:rackspace_region] = Fog::Rackspace.const_get(region.upcase)
|
16
|
+
{
|
17
|
+
provider: 'rackspace',
|
18
|
+
rackspace_username: UI.prompt('Rackspace Cloud Username'),
|
19
|
+
rackspace_api_key: UI.prompt('Rackspace Cloud API key'),
|
20
|
+
rackspace_region: define_rackspace_region
|
21
|
+
}
|
22
|
+
end
|
25
23
|
|
26
|
-
|
24
|
+
# Public: Determine which Rackspace public endpoint should be used.
|
25
|
+
#
|
26
|
+
# Returns a String.
|
27
|
+
def define_rackspace_region
|
28
|
+
countries = [/^u[sk]$/i]
|
29
|
+
region = UI.prompt('Account Region (us, uk)', valid_answers: countries)
|
30
|
+
Fog::Rackspace.const_get(region.upcase + '_AUTH_ENDPOINT')
|
27
31
|
end
|
28
32
|
|
29
33
|
# Public: Wrap define_rackspace_service_region, specifying
|
30
34
|
# 'cloudServersOpenStack' as the service type.
|
31
35
|
#
|
32
36
|
# api - Authenticated Fog API instance.
|
37
|
+
#
|
38
|
+
# Returns a Hash.
|
33
39
|
def define_rackspace_cloudservers_region(api)
|
34
40
|
api.merge(define_rackspace_service_region(api, 'cloudServersOpenStack'))
|
35
41
|
end
|
@@ -38,6 +44,8 @@ module CloudFlock; module App
|
|
38
44
|
# the service type.
|
39
45
|
#
|
40
46
|
# api - Authenticated Fog API instance.
|
47
|
+
#
|
48
|
+
# Returns a Hash.
|
41
49
|
def define_rackspace_files_region(api)
|
42
50
|
api.merge(define_rackspace_service_region(api, 'cloudFiles'))
|
43
51
|
end
|
@@ -76,8 +76,8 @@ module CloudFlock; module App
|
|
76
76
|
|
77
77
|
key_path = File.join(Dir.home, '.ssh', 'id_rsa')
|
78
78
|
key_path = '' unless File.exists?(key_path)
|
79
|
-
|
80
|
-
|
79
|
+
check_option_fs(host, :ssh_key, "#{name} SSH Key",
|
80
|
+
default_answer: key_path, allow_empty: true)
|
81
81
|
|
82
82
|
# Using sudo is only applicable if the user isn't root
|
83
83
|
host[:sudo] = false if host[:username] == 'root'
|
@@ -52,30 +52,43 @@ module CloudFlock; module App
|
|
52
52
|
#
|
53
53
|
# Returns a CloudFlock::Remote::Files object.
|
54
54
|
def define_api(desc, create = false)
|
55
|
-
store = {}
|
56
55
|
query = "#{desc} provider (rackspace, aws, local)"
|
57
56
|
answers = [/^(?:rackspace|aws|local)$/i]
|
58
57
|
|
59
58
|
provider = UI.prompt(query, valid_answers: answers)
|
60
59
|
|
61
|
-
case provider
|
60
|
+
api = case provider
|
62
61
|
when /rackspace/i
|
63
|
-
|
64
|
-
store[:rackspace_username] = UI.prompt('Rackspace username')
|
65
|
-
store[:rackspace_api_key] = UI.prompt('Rackspace API key')
|
62
|
+
define_rackspace_api
|
66
63
|
when /aws/i
|
67
|
-
|
68
|
-
|
69
|
-
|
64
|
+
{
|
65
|
+
provider: 'AWS',
|
66
|
+
aws_access_key_id: UI.prompt('AWS Access Key ID'),
|
67
|
+
aws_secret_access_key: UI.prompt('AWS secret access key')
|
68
|
+
}
|
70
69
|
when /local/i
|
71
|
-
|
72
|
-
|
73
|
-
|
70
|
+
{
|
71
|
+
provider: 'local',
|
72
|
+
local_root: UI.prompt("#{desc} location")
|
73
|
+
}
|
74
74
|
end
|
75
75
|
|
76
|
-
|
76
|
+
store = api.merge(define_rackspace_files_region(api))
|
77
|
+
setup_object_store(CloudFlock::Remote::Files.new(store), desc, create)
|
78
|
+
end
|
77
79
|
|
78
|
-
|
80
|
+
# Internal: Connect to an object store and determine the directory on which
|
81
|
+
# to act.
|
82
|
+
#
|
83
|
+
# store - Fog::Remote::Files object.
|
84
|
+
# desc - Description of the data store for display purposes.
|
85
|
+
# create - Whether to create non-existing locations.
|
86
|
+
#
|
87
|
+
# Returns a CloudFlock::Remote::Files object.
|
88
|
+
def setup_object_store(store, desc, create)
|
89
|
+
return store if store.local?
|
90
|
+
|
91
|
+
options = store.directories.map do |dir|
|
79
92
|
{ name: dir.key, files: dir.count.to_s }
|
80
93
|
end
|
81
94
|
valid = options.reduce([]) { |c,e| c << e[:name] }
|
@@ -83,15 +96,15 @@ module CloudFlock; module App
|
|
83
96
|
puts UI.build_grid(options, name: "Directory name", files: "File count")
|
84
97
|
if create
|
85
98
|
selected = UI.prompt("#{desc} directory")
|
86
|
-
unless
|
87
|
-
|
99
|
+
unless store.directories.select { |dir| dir.key == selected }.any?
|
100
|
+
store.directories.create(key: selected)
|
88
101
|
end
|
89
102
|
else
|
90
103
|
selected = UI.prompt("#{desc} directory", valid_answers: valid)
|
91
104
|
end
|
92
|
-
|
105
|
+
store.directory = selected
|
93
106
|
|
94
|
-
|
107
|
+
store
|
95
108
|
end
|
96
109
|
|
97
110
|
# Internal: Set up queue and Mutexes, create threads to manage the transfer
|
@@ -111,34 +124,35 @@ module CloudFlock; module App
|
|
111
124
|
#
|
112
125
|
# Returns nothing.
|
113
126
|
def files_migrate(source_store, dest_store, options = {})
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
down_threads = options[:download_threads] || DOWNLOAD_THREADS
|
127
|
+
mutexes = { queue: Queue.new, ongoing: Mutex.new }
|
128
|
+
up_threads = options[:upload_threads] || UPLOAD_THREADS
|
129
|
+
down_threads = options[:download_threads] || DOWNLOAD_THREADS
|
118
130
|
|
119
|
-
|
120
|
-
manage_source(source_store, file_queue, mutexes, up_threads)
|
121
|
-
end
|
131
|
+
mutexes[:ongoing].lock
|
122
132
|
|
123
133
|
destination = Thread.new do
|
124
|
-
manage_destination(dest_store,
|
134
|
+
manage_destination(dest_store, mutexes, down_threads)
|
135
|
+
end
|
136
|
+
source = Thread.new do
|
137
|
+
manage_source(source_store, mutexes, up_threads)
|
125
138
|
end
|
126
139
|
|
127
|
-
[source
|
140
|
+
[source].each(&:join)
|
141
|
+
mutexes[:ongoing].unlock
|
142
|
+
|
143
|
+
[destination].each(&:join)
|
128
144
|
end
|
129
145
|
|
130
146
|
# Internal: Create and observe threads which download files from a
|
131
147
|
# non-local file store. If the files exist locally, simply generate a list
|
132
148
|
# of the files in queue.
|
133
149
|
#
|
134
|
-
#
|
150
|
+
# store - CloudFlock::Remote::Files object set up to pull files from
|
135
151
|
# a source directory.
|
136
|
-
# file_queue - Array in which details regarding files to be transferred
|
137
|
-
# will be stored.
|
138
152
|
# mutexes - Hash containing two Mutexes:
|
139
|
-
# :queue
|
140
|
-
# :
|
141
|
-
#
|
153
|
+
# :queue - Queue to coordinate file access.
|
154
|
+
# :ongoing - Indicates that the transfer is ongoing when
|
155
|
+
# locked.
|
142
156
|
# thread_count - Hash optionally containing overrides for the number of
|
143
157
|
# upload and download threads to use for transfer
|
144
158
|
# concurrency. (default: {})
|
@@ -148,38 +162,30 @@ module CloudFlock; module App
|
|
148
162
|
# Overrides DOWNLOAD_THREADS constant.
|
149
163
|
#
|
150
164
|
# Returns nothing.
|
151
|
-
def manage_source(
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
mutexes[:queue].synchronize do
|
157
|
-
file_queue << { path: "#{source_store.prefix}/#{file.key}",
|
158
|
-
name: file.key, temp: false }
|
159
|
-
end
|
165
|
+
def manage_source(store, mutexes, thread_count)
|
166
|
+
if store.local?
|
167
|
+
store.each_file do |file|
|
168
|
+
node = File.new("#{store.prefix}/#{file.key}")
|
169
|
+
mutexes[:queue] << { file: node, name: file.key, temp: false }
|
160
170
|
end
|
161
171
|
else
|
162
172
|
source_threads = []
|
163
173
|
file_list_mutex = Mutex.new
|
164
|
-
file_list =
|
174
|
+
file_list = store.file_list
|
165
175
|
|
166
176
|
thread_count.times do
|
167
177
|
source_threads << Thread.new do
|
168
178
|
while file = file_list_mutex.synchronize { file_list.pop } do
|
169
179
|
temp = Tempfile.new(file.gsub(/\//, ''))
|
170
|
-
temp.write(
|
180
|
+
temp.write(store.get_file(file))
|
171
181
|
temp.close
|
172
|
-
mutexes[:queue]
|
173
|
-
file_queue << { path: temp.path, name: file, temp: true }
|
174
|
-
end
|
182
|
+
mutexes[:queue] << { file: temp, name: file, temp: true }
|
175
183
|
end
|
176
184
|
end
|
177
185
|
end
|
178
186
|
|
179
187
|
source_threads.each(&:join)
|
180
188
|
end
|
181
|
-
|
182
|
-
mutexes[:finished].unlock
|
183
189
|
end
|
184
190
|
|
185
191
|
# Internal: Create and observe threads which download files from a
|
@@ -188,12 +194,10 @@ module CloudFlock; module App
|
|
188
194
|
#
|
189
195
|
# dest_store - CloudFlock::Remote::Files object set up to upload files to
|
190
196
|
# a destination directory.
|
191
|
-
# file_queue - Array from which to retrieve details regarding files to be
|
192
|
-
# transferred.
|
193
197
|
# mutexes - Hash containing two Mutexes:
|
194
|
-
# :queue
|
195
|
-
# :
|
196
|
-
#
|
198
|
+
# :queue - Queue to coordinate file access.
|
199
|
+
# :ongoing - Indicates that the transfer is ongoing when
|
200
|
+
# locked.
|
197
201
|
# thread_count - Hash optionally containing overrides for the number of
|
198
202
|
# upload and download threads to use for transfer
|
199
203
|
# concurrency. (default: {})
|
@@ -203,24 +207,41 @@ module CloudFlock; module App
|
|
203
207
|
# Overrides DOWNLOAD_THREADS constant.
|
204
208
|
#
|
205
209
|
# Returns nothing.
|
206
|
-
def manage_destination(dest_store,
|
210
|
+
def manage_destination(dest_store, mutexes, thread_count)
|
207
211
|
dest_threads = []
|
208
212
|
|
209
213
|
thread_count.times do
|
210
214
|
dest_threads << Thread.new do
|
211
|
-
|
212
|
-
while file = mutexes[:queue].synchronize { file_queue.pop }
|
213
|
-
content = File.read(file[:path])
|
214
|
-
dest_store.create(key: file[:name], body: content)
|
215
|
-
File.unlink(file[:path]) if file[:temp]
|
216
|
-
end
|
217
|
-
end
|
215
|
+
upload_thread(dest_store, mutexes)
|
218
216
|
end
|
219
217
|
end
|
220
218
|
|
221
219
|
dest_threads.each(&:join)
|
222
220
|
end
|
223
221
|
|
222
|
+
# Internal: Upload files from a Queue to a given object store.
|
223
|
+
#
|
224
|
+
# dest_store - CloudFlock::Remote::Files object set up to upload files to
|
225
|
+
# a destination directory.
|
226
|
+
# mutexes - Hash containing two Mutexes:
|
227
|
+
# :queue - Queue to coordinate file access.
|
228
|
+
# :ongoing - Indicates that the transfer is ongoing when
|
229
|
+
# locked.
|
230
|
+
#
|
231
|
+
# Returns nothing.
|
232
|
+
def upload_thread(dest_store, mutexes)
|
233
|
+
while mutexes[:ongoing].locked?
|
234
|
+
while file = mutexes[:queue].pop(true)
|
235
|
+
file[:file].open if file[:temp]
|
236
|
+
dest_store.create(key: file[:name], body: file[:file].read)
|
237
|
+
file[:file].close
|
238
|
+
end
|
239
|
+
end
|
240
|
+
rescue ThreadError
|
241
|
+
sleep 0.1
|
242
|
+
retry
|
243
|
+
end
|
244
|
+
|
224
245
|
# Internal: Set up an OptionParser object to recognize options specific to
|
225
246
|
# profiling a remote host.
|
226
247
|
#
|
data/lib/cloudflock/app.rb
CHANGED
@@ -26,6 +26,24 @@ module CloudFlock
|
|
26
26
|
options[name] = UI.prompt(prompt, prompt_options)
|
27
27
|
end
|
28
28
|
|
29
|
+
# Public: Wrap check_option, allowing for filesystem autocompletion in user
|
30
|
+
# response if the option is not set.
|
31
|
+
#
|
32
|
+
# options - Hash containing options to test against.
|
33
|
+
# name - The key in the options Hash expected to contain the
|
34
|
+
# response desired.
|
35
|
+
# prompt - Prompt to present to the user.
|
36
|
+
# prompt_options - Options to pass along to ConsoleGlitter::UI#prompt.
|
37
|
+
# (default: {})
|
38
|
+
#
|
39
|
+
# Returns the contents of the options[name] or else a String if
|
40
|
+
# options[name] is nil.
|
41
|
+
def check_option_fs(options, name, prompt, prompt_options = {})
|
42
|
+
return options[name] unless options[name].nil?
|
43
|
+
|
44
|
+
options[name] = UI.prompt_filesystem(prompt, prompt_options)
|
45
|
+
end
|
46
|
+
|
29
47
|
# Public: Check if an option is set; return the value if so, otherwise
|
30
48
|
# prompt the user for a response.
|
31
49
|
#
|
@@ -63,6 +63,9 @@ module CloudFlock; module Remote
|
|
63
63
|
# Returns a String.
|
64
64
|
def get_file(file)
|
65
65
|
@files.files.get(file).body
|
66
|
+
rescue Excon::Errors::Timeout, Fog::Storage::Rackspace::ServiceError
|
67
|
+
# Triggered by server and request timeouts respectively.
|
68
|
+
retry
|
66
69
|
end
|
67
70
|
|
68
71
|
# Public: Create a file in the current directory.
|
@@ -74,7 +77,8 @@ module CloudFlock; module Remote
|
|
74
77
|
# Returns nothing.
|
75
78
|
def create(file_spec)
|
76
79
|
@files.files.create(file_spec)
|
77
|
-
rescue Excon::Errors::Timeout
|
80
|
+
rescue Excon::Errors::Timeout, Fog::Storage::Rackspace::ServiceError
|
81
|
+
# Triggered by server and request timeouts respectively.
|
78
82
|
retry
|
79
83
|
end
|
80
84
|
|
data/lib/cloudflock.rb
CHANGED
metadata
CHANGED
@@ -1,69 +1,69 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cloudflock
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Wuest
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-05-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fog
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - ~>
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '1.21'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - ~>
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '1.21'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: cpe
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - ~>
|
31
|
+
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0.5'
|
34
|
-
- -
|
34
|
+
- - ">="
|
35
35
|
- !ruby/object:Gem::Version
|
36
36
|
version: 0.5.0
|
37
37
|
type: :runtime
|
38
38
|
prerelease: false
|
39
39
|
version_requirements: !ruby/object:Gem::Requirement
|
40
40
|
requirements:
|
41
|
-
- - ~>
|
41
|
+
- - "~>"
|
42
42
|
- !ruby/object:Gem::Version
|
43
43
|
version: '0.5'
|
44
|
-
- -
|
44
|
+
- - ">="
|
45
45
|
- !ruby/object:Gem::Version
|
46
46
|
version: 0.5.0
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: console-glitter
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
50
50
|
requirements:
|
51
|
-
- - ~>
|
51
|
+
- - "~>"
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: '0.
|
54
|
-
- -
|
53
|
+
version: '0.2'
|
54
|
+
- - ">="
|
55
55
|
- !ruby/object:Gem::Version
|
56
|
-
version: 0.1
|
56
|
+
version: 0.2.1
|
57
57
|
type: :runtime
|
58
58
|
prerelease: false
|
59
59
|
version_requirements: !ruby/object:Gem::Requirement
|
60
60
|
requirements:
|
61
|
-
- - ~>
|
61
|
+
- - "~>"
|
62
62
|
- !ruby/object:Gem::Version
|
63
|
-
version: '0.
|
64
|
-
- -
|
63
|
+
version: '0.2'
|
64
|
+
- - ">="
|
65
65
|
- !ruby/object:Gem::Version
|
66
|
-
version: 0.1
|
66
|
+
version: 0.2.1
|
67
67
|
description: CloudFlock is a library and toolchain focused on migration
|
68
68
|
email: chris@chriswuest.com
|
69
69
|
executables:
|
@@ -111,12 +111,12 @@ require_paths:
|
|
111
111
|
- lib
|
112
112
|
required_ruby_version: !ruby/object:Gem::Requirement
|
113
113
|
requirements:
|
114
|
-
- -
|
114
|
+
- - ">="
|
115
115
|
- !ruby/object:Gem::Version
|
116
116
|
version: '0'
|
117
117
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
118
118
|
requirements:
|
119
|
-
- -
|
119
|
+
- - ">="
|
120
120
|
- !ruby/object:Gem::Version
|
121
121
|
version: '0'
|
122
122
|
requirements: []
|