cloudflock 0.4.4 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,18 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $0 = 'cloudflock-files'
4
+
5
+ require 'cloudflock/interface/cli/app/files'
6
+ CLI = CloudFlock::Interface::CLI::Console
7
+ Opts = CloudFlock::Interface::CLI::Opts
8
+
9
+ # Trap C-c to kill the application in a friendly manner
10
+ trap 'INT' do
11
+ puts "\nCaught SIGINT, exiting..."
12
+ exit 1
13
+ end
14
+
15
+ options = Opts.parse
16
+ options[:config] = Opts.parse_config_file(options[:config_file])
17
+
18
+ CloudFlock::Interface::CLI::App::Files.new(options)
@@ -0,0 +1,179 @@
1
+ require 'cloudflock'
2
+ require 'cloudflock/interface/cli'
3
+ require 'cloudflock/remote/files'
4
+ require 'tempfile'
5
+ require 'thread'
6
+ require 'fog'
7
+
8
+ # Public: The Files app provides the interface to perform migrations of
9
+ # File/Object storage (e.g. Amazon S3, Local files and Rackspace Cloud Files).
10
+ class CloudFlock::Interface::CLI::App::Files
11
+ CLI = CloudFlock::Interface::CLI::Console
12
+
13
+ DOWNLOAD_THREAD_COUNT = 4
14
+ UPLOAD_THREAD_COUNT = 4
15
+
16
+ # Public: Begin Files migration on the command line
17
+ #
18
+ # opts - Hash containing options mappings.
19
+ def initialize(opts)
20
+ @options = opts
21
+ @download_finished = false
22
+ @download_mutex = Mutex.new
23
+ @upload_mutex = Mutex.new
24
+ @download_list = []
25
+ @upload_list = []
26
+
27
+ source_store = define_store("source")
28
+ destination_store = define_store("destination")
29
+
30
+ @source_container = define_container(source_store, "source")
31
+ @destination_container = define_container(destination_store, "destination",
32
+ true)
33
+
34
+ if perform_migration
35
+ puts "#{CLI.bold}#{CLI.blue}*** Migration complete#{CLI.reset}\a"
36
+ else
37
+ puts "#{CLI.bold}#{CLI.red}*** Migration failed#{CLI.reset}\a"
38
+ end
39
+ rescue Excon::Errors::Unauthorized => err
40
+ puts "A provider has returned an Unauthorized error."
41
+ puts err.inspect if @options[:verbose]
42
+ exit 1
43
+ end
44
+
45
+ # Internal: Migrate objects from the source store to the destination store.
46
+ #
47
+ # Returns a boolean value corresponding to whether the migration has
48
+ # completed successfully.
49
+ def perform_migration
50
+ download_threads = []
51
+ upload_threads = []
52
+
53
+ @source_container.files.each { |f| @download_list.push(f) }
54
+
55
+ DOWNLOAD_THREAD_COUNT.times do
56
+ download_threads << download_thread
57
+ end
58
+ UPLOAD_THREAD_COUNT.times do
59
+ upload_threads << upload_thread
60
+ end
61
+
62
+ download_threads.each { |t| t.join }
63
+ @download_finished = true
64
+ upload_threads.each { |t| t.join }
65
+ true
66
+ rescue => e
67
+ if @options[:verbose]
68
+ puts "#{CLI.bold}#{CLI.red}*** Error ***#{CLI.reset}"
69
+ puts e.inspect
70
+ puts e.backtrace
71
+ puts
72
+ end
73
+ false
74
+ end
75
+
76
+ # Internal: Create a new Thread to download objects from the source
77
+ # container.
78
+ #
79
+ # Returns a Thread.
80
+ def download_thread
81
+ Thread.new do
82
+ file = nil
83
+ until @download_list.empty?
84
+ @download_mutex.synchronize do
85
+ file = @download_list.pop
86
+ end
87
+ next if file.nil?
88
+ # AWS stores directories as their own object
89
+ next if file.content_length == 0 && file.key =~ /\/$/
90
+
91
+ tmp = Tempfile.new(file.object_id.to_s)
92
+ @source_container.files.get(file.key) do |data, rem, cl|
93
+ tmp.syswrite(data)
94
+ end
95
+ tmp.flush
96
+ tmp.rewind
97
+ @upload_mutex.synchronize do
98
+ @upload_list.push(body: tmp, key: file.key)
99
+ end
100
+ end
101
+ end
102
+ end
103
+
104
+ # Internal: Create a new Thread to upload objects to the desination
105
+ # container.
106
+ #
107
+ # Returns a Thread.
108
+ def upload_thread
109
+ Thread.new do
110
+ file = nil
111
+ until @upload_list.empty? && @download_finished
112
+ sleep 0.1
113
+ @upload_mutex.synchronize do
114
+ file = @upload_list.pop
115
+ end
116
+ next if file.nil?
117
+ @destination_container.files.create(file)
118
+ end
119
+ end
120
+ end
121
+
122
+ # Internal: Ascertain the location for a data store.
123
+ #
124
+ # desc - String containing a description for the file store.
125
+ #
126
+ # Returns a Fog object pointing to the data store.
127
+ # Raises ArgumentError if desc isn't a String.
128
+ def define_store(desc)
129
+ unless desc.kind_of?(String)
130
+ raise ArgumentError, "String expected"
131
+ end
132
+ store = {}
133
+ store[:provider] = CLI.prompt("#{desc} provider (aws, local, rax)",
134
+ valid_answers: ["rax", "aws", "local"])
135
+ case store[:provider]
136
+ when 'rax'
137
+ store[:provider] = 'Rackspace'
138
+ store[:rackspace_username] = CLI.prompt("Rackspace username")
139
+ store[:rackspace_api_key] = CLI.prompt("Rackspace API key")
140
+ when 'aws'
141
+ store[:provider] = 'AWS'
142
+ store[:aws_access_key_id] = CLI.prompt("AWS Access Key ID")
143
+ store[:aws_secret_access_key] = CLI.prompt("AWS secret access key")
144
+ when 'local'
145
+ store[:local_root] = CLI.prompt("#{desc} location")
146
+ end
147
+
148
+ CloudFlock::Remote::Files.connect(store)
149
+ end
150
+
151
+ # Internal: Obtain the name of a container.
152
+ #
153
+ # store - Fog object pointing to a Fog::Storage object.
154
+ # desc - String containing a description for the container.
155
+ # create - Boolean value indicating whether to create the container.
156
+ #
157
+ # Returns a Fog object pointing to the container.
158
+ # Raises ArgumentError if store isn't a Fog::Storage object.
159
+ # Raises ArgumentError if desc isn't a String.
160
+ def define_container(store, desc, create=false)
161
+ unless store.class.to_s =~ /^Fog::Storage/
162
+ raise ArgumentError, "Fog Storage object expected"
163
+ end
164
+ unless desc.kind_of?(String)
165
+ raise ArgumentError, "String expected"
166
+ end
167
+
168
+ if create
169
+ container = CLI.prompt("#{desc} container name")
170
+ return store.directories.create(key: container)
171
+ else
172
+ puts "Available containers:"
173
+ puts store.directories.map(&:key)
174
+ container = CLI.prompt("#{desc} container name",
175
+ valid_answers: store.directories.map(&:key))
176
+ return store.directories.select { |i| i.key == container }[0]
177
+ end
178
+ end
179
+ end
@@ -0,0 +1,28 @@
1
+ require 'cloudflock'
2
+
3
+ # Public: Provide an interface to instantiate Fog::Storage instances, perform
4
+ # basic sanity checking for the local provider.
5
+ module CloudFlock::Remote::Files extend self
6
+ # Public: Set up and verify a data source.
7
+ #
8
+ # target - Hash containing connection details per Fog requirements, a String
9
+ # containing the path to a local directory.
10
+ #
11
+ # Returns an instance of a Files subclass.
12
+ def connect(target)
13
+ target = { provider: 'local', local_root: target } if target.is_a?(String)
14
+ raise ArgumentError, "String or Hash expected" unless target.is_a?(Hash)
15
+
16
+ if target[:provider].downcase == 'local'
17
+ target_parent = File.expand_path(target[:local_root], '..')
18
+ unless File.exists?(target_parent)
19
+ raise Errno::ENOENT, "#{target_parent} does not exist"
20
+ end
21
+ unless File.directory?(target_parent)
22
+ raise Errno::ENOENT, "#{target_parent} is not a directory"
23
+ end
24
+ end
25
+
26
+ Fog::Storage.new(target)
27
+ end
28
+ end
@@ -0,0 +1,6 @@
1
+ /etc/yum.repos.d/
2
+ /usr/lib/yum-plugins
3
+ /etc/yum.conf
4
+ /etc/yum
5
+ /etc/yum.repos.d
6
+ /etc/sysconfig/iptables
@@ -3,3 +3,4 @@
3
3
  /etc/yum.conf
4
4
  /etc/yum
5
5
  /etc/yum.repos.d
6
+ /etc/sysconfig/iptables
@@ -69,7 +69,7 @@ module CloudFlock::Target::Servers::Migrate extend self
69
69
  preserve_files = ["passwd", "shadow", "group"]
70
70
  path = "/mnt/migration_target/etc"
71
71
  preserve_files.each do |file|
72
- copy_command = "[ -f #{path}/migration.#{file} ] || /bin/cp -an " +
72
+ copy_command = "[ -f #{path}/migration.#{file} ] || /bin/cp -a " +
73
73
  "#{path}/#{file} #{path}/migration.#{file}"
74
74
  host.puts(copy_command)
75
75
  host.prompt
@@ -1,3 +1,3 @@
1
1
  module CloudFlock
2
- VERSION = '0.4.4'
2
+ VERSION = '0.5.0'
3
3
  end
metadata CHANGED
@@ -1,55 +1,62 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cloudflock
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.4
4
+ version: 0.5.0
5
+ prerelease:
5
6
  platform: ruby
6
7
  authors:
7
8
  - Chris Wuest
8
9
  autorequire:
9
10
  bindir: bin
10
11
  cert_chain: []
11
- date: 2013-05-03 00:00:00.000000000 Z
12
+ date: 2013-05-22 00:00:00.000000000 Z
12
13
  dependencies:
13
14
  - !ruby/object:Gem::Dependency
14
15
  name: fog
15
16
  requirement: !ruby/object:Gem::Requirement
17
+ none: false
16
18
  requirements:
17
- - - '>='
19
+ - - ! '>='
18
20
  - !ruby/object:Gem::Version
19
- version: '0'
21
+ version: 1.11.1
20
22
  type: :runtime
21
23
  prerelease: false
22
24
  version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
23
26
  requirements:
24
- - - '>='
27
+ - - ! '>='
25
28
  - !ruby/object:Gem::Version
26
- version: '0'
29
+ version: 1.11.1
27
30
  - !ruby/object:Gem::Dependency
28
31
  name: expectr
29
32
  requirement: !ruby/object:Gem::Requirement
33
+ none: false
30
34
  requirements:
31
- - - '>='
35
+ - - ! '>='
32
36
  - !ruby/object:Gem::Version
33
37
  version: '0'
34
38
  type: :runtime
35
39
  prerelease: false
36
40
  version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
37
42
  requirements:
38
- - - '>='
43
+ - - ! '>='
39
44
  - !ruby/object:Gem::Version
40
45
  version: '0'
41
46
  - !ruby/object:Gem::Dependency
42
47
  name: cpe
43
48
  requirement: !ruby/object:Gem::Requirement
49
+ none: false
44
50
  requirements:
45
- - - '>='
51
+ - - ! '>='
46
52
  - !ruby/object:Gem::Version
47
53
  version: '0'
48
54
  type: :runtime
49
55
  prerelease: false
50
56
  version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
51
58
  requirements:
52
- - - '>='
59
+ - - ! '>='
53
60
  - !ruby/object:Gem::Version
54
61
  version: '0'
55
62
  description: CloudFlock is a library and toolchain focused on migration
@@ -65,17 +72,19 @@ files:
65
72
  - lib/cloudflock/error.rb
66
73
  - lib/cloudflock/interface/cli.rb
67
74
  - lib/cloudflock/interface/cli/app/common/servers.rb
75
+ - lib/cloudflock/interface/cli/app/files.rb
68
76
  - lib/cloudflock/interface/cli/app/servers.rb
69
77
  - lib/cloudflock/interface/cli/app/servers/migrate.rb
70
78
  - lib/cloudflock/interface/cli/app/servers/profile.rb
71
79
  - lib/cloudflock/interface/cli/console.rb
72
80
  - lib/cloudflock/interface/cli/opts.rb
73
81
  - lib/cloudflock/interface/cli/opts/servers.rb
74
- - lib/cloudflock/patch/fog.rb
82
+ - lib/cloudflock/remote/files.rb
75
83
  - lib/cloudflock/remote/ssh.rb
76
84
  - lib/cloudflock/target/servers.rb
77
85
  - lib/cloudflock/target/servers/data/exceptions/base.txt
78
86
  - lib/cloudflock/target/servers/data/exceptions/platform/amazon.txt
87
+ - lib/cloudflock/target/servers/data/exceptions/platform/centos.txt
79
88
  - lib/cloudflock/target/servers/data/exceptions/platform/debian.txt
80
89
  - lib/cloudflock/target/servers/data/exceptions/platform/redhat.txt
81
90
  - lib/cloudflock/target/servers/data/exceptions/platform/suse.txt
@@ -90,32 +99,33 @@ files:
90
99
  - lib/cloudflock/target/servers/profile.rb
91
100
  - lib/cloudflock/version.rb
92
101
  - bin/cloudflock
102
+ - bin/cloudflock-files
93
103
  - bin/cloudflock-profile
94
104
  - bin/cloudflock-servers
95
105
  - bin/cloudflock.default
96
106
  homepage: http://github.com/cwuest/cloudflock
97
107
  licenses:
98
108
  - Apache 2.0
99
- metadata: {}
100
109
  post_install_message:
101
110
  rdoc_options: []
102
111
  require_paths:
103
112
  - lib
104
113
  required_ruby_version: !ruby/object:Gem::Requirement
114
+ none: false
105
115
  requirements:
106
- - - '>='
116
+ - - ! '>='
107
117
  - !ruby/object:Gem::Version
108
118
  version: '0'
109
119
  required_rubygems_version: !ruby/object:Gem::Requirement
120
+ none: false
110
121
  requirements:
111
- - - '>='
122
+ - - ! '>='
112
123
  - !ruby/object:Gem::Version
113
124
  version: '0'
114
125
  requirements: []
115
126
  rubyforge_project:
116
- rubygems_version: 2.0.3
127
+ rubygems_version: 1.8.25
117
128
  signing_key:
118
- specification_version: 4
129
+ specification_version: 3
119
130
  summary: Server migration automation
120
131
  test_files: []
121
- has_rdoc:
checksums.yaml DELETED
@@ -1,7 +0,0 @@
1
- ---
2
- SHA1:
3
- metadata.gz: 8d1d382459026ce9c6cc5b6b2c340dde31b6b035
4
- data.tar.gz: 829d46918880f162a85babcd15217808c56d468f
5
- SHA512:
6
- metadata.gz: 6262061b988a8bd8a8a60722220942ce1ff8baa01a8bad37995ef38e91ed749b6f3c0132003e0db09a978e1844cadaef72fb43a8f1be38fd0080af2d5f2c3658
7
- data.tar.gz: f57c2a93299ea7bea12413d6c72d8ccb7d5f00642c746d5b2f6eccb0e0863b2415b4c39d8a1b15b026e6f600d79f97085c4cab496dc8d5506b3603f9443276f8
@@ -1,113 +0,0 @@
1
- require 'fog'
2
- module Fog
3
- module Compute
4
- class RackspaceV2
5
- class Server
6
- # Place existing server into rescue mode, allowing for offline editing of configuration. The original server's disk is attached to a new instance of the same base image for a period of time to facilitate working within rescue mode. The original server will be autom atically restored after 90 minutes.
7
- # @return [Boolean] returns true if call to put server in rescue mode returns success
8
- # @raise [Fog::Rackspace::Errors::NotFound] - HTTP 404
9
- # @raise [Fog::Rackspace::Errors::BadRequest] - HTTP 400
10
- # @raise [Fog::Rackspace::Errors::InternalServerError] - HTTP 500
11
- # @raise [Fog::Rackspace::Errors::ServiceError]
12
- # @note Rescue mode is only guaranteed to be active for 90 minutes.
13
- # @see http://docs.rackspace.com/servers/api/v2/cs-devguide/content/rescue_mode.html
14
- # @see #unrescue
15
- #
16
- # * Status Transition:
17
- # * ACTIVE -> PREP_RESCUE -> RESCUE
18
- def rescue
19
- requires :identity
20
- data = service.rescue_server(identity)
21
- merge_attributes(data.body)
22
- self.state = RESCUE
23
- true
24
- end
25
-
26
- # Remove existing server from rescue mode.
27
- # @return [Boolean] returns true if call to remove server from rescue mode returns success
28
- # @raise [Fog::Rackspace::Errors::NotFound] - HTTP 404
29
- # @raise [Fog::Rackspace::Errors::BadRequest] - HTTP 400
30
- # @raise [Fog::Rackspace::Errors::InternalServerError] - HTTP 500
31
- # @raise [Fog::Rackspace::Errors::ServiceError]
32
- # @note Rescue mode is only guaranteed to be active for 90 minutes.
33
- # @see http://docs.rackspace.com/servers/api/v2/cs-devguide/content/exit_rescue_mode.html
34
- # @see #rescue
35
- #
36
- # * Status Transition:
37
- # * RESCUE -> PREP_UNRESCUE -> ACTIVE
38
- def unrescue
39
- requires :identity
40
- service.unrescue_server(identity)
41
- self.state = ACTIVE
42
- true
43
- end
44
- end
45
-
46
- class Real
47
- # Puts server into rescue mode
48
- # @param [String] server_id id of server to rescue
49
- # @return [Excon::Response] response
50
- # @raise [Fog::Rackspace::Errors::NotFound] - HTTP 404
51
- # @raise [Fog::Rackspace::Errors::BadRequest] - HTTP 400
52
- # @raise [Fog::Rackspace::Errors::InternalServerError] - HTTP 500
53
- # @raise [Fog::Rackspace::Errors::ServiceError]
54
- # @note Rescue mode is only guaranteed to be active for 90 minutes.
55
- # @see http://docs.rackspace.com/servers/api/v2/cs-devguide/content/rescue_mode.html
56
- #
57
- # * Status Transition:
58
- # * ACTIVE -> PREP_RESCUE -> RESCUE
59
- def rescue_server(server_id)
60
- data = {
61
- 'rescue' => nil
62
- }
63
-
64
- request(
65
- :body => Fog::JSON.encode(data),
66
- :expects => [200],
67
- :method => 'POST',
68
- :path => "servers/#{server_id}/action"
69
- )
70
- end
71
-
72
- # Take server out of rescue mode
73
- # @param [String] server_id id of server
74
- # @return [Excon::Response] response
75
- # @raise [Fog::Rackspace::Errors::NotFound] - HTTP 404
76
- # @raise [Fog::Rackspace::Errors::BadRequest] - HTTP 400
77
- # @raise [Fog::Rackspace::Errors::InternalServerError] - HTTP 500
78
- # @raise [Fog::Rackspace::Errors::ServiceError]
79
- # @see http://docs.rackspace.com/servers/api/v2/cs-devguide/content/exit_rescue_mode.html
80
- #
81
- # * Status Transition:
82
- # * RESCUE -> PREP_UNRESCUE -> ACTIVE
83
- def unrescue_server(server_id)
84
- data = {
85
- 'unrescue' => nil
86
- }
87
-
88
- request(
89
- :body => Fog::JSON.encode(data),
90
- :expects => [202],
91
- :method => 'POST',
92
- :path => "servers/#{server_id}/action"
93
- )
94
- end
95
- end
96
-
97
- class Mock
98
- def rescue_server(server_id)
99
- server = self.data[:servers][server_id]
100
- server["status"] = "PREP_RESCUE"
101
- response(:status => 200)
102
- end
103
-
104
- def unrescue_server(server_id)
105
- server = self.data[:servers][server_id]
106
- server["status"] = "PREP_UNRESCUE"
107
- response(:status => 202)
108
- end
109
- end
110
- end
111
- end
112
- end
113
-