syncwrap 1.5.2 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (107) hide show
  1. checksums.yaml +7 -0
  2. data/History.rdoc +19 -0
  3. data/Manifest.txt +82 -34
  4. data/README.rdoc +96 -48
  5. data/Rakefile +0 -65
  6. data/bin/syncwrap +27 -0
  7. data/examples/LAYOUT.rdoc +70 -0
  8. data/examples/Rakefile +16 -0
  9. data/examples/ec2.rb +44 -0
  10. data/examples/hello.rb +14 -0
  11. data/examples/hello_binding.rb +27 -0
  12. data/examples/jruby.rb +11 -0
  13. data/examples/private/aws.json +4 -0
  14. data/examples/rput.rb +24 -0
  15. data/examples/sync/home/bob/.ssh/authorized_keys +1 -0
  16. data/examples/sync/tmp/sample.erb +3 -0
  17. data/lib/syncwrap/amazon_ec2.rb +236 -0
  18. data/lib/syncwrap/amazon_ws.rb +308 -0
  19. data/lib/syncwrap/base.rb +4 -2
  20. data/lib/syncwrap/cli.rb +328 -0
  21. data/lib/syncwrap/component.rb +443 -0
  22. data/lib/syncwrap/components/commercial_jdk.rb +76 -0
  23. data/lib/syncwrap/components/cruby_vm.rb +144 -0
  24. data/lib/syncwrap/components/etc_hosts.rb +44 -0
  25. data/lib/syncwrap/{geminabox.rb → components/geminabox.rb} +12 -17
  26. data/lib/syncwrap/components/hashdot.rb +97 -0
  27. data/lib/syncwrap/components/iyyov.rb +144 -0
  28. data/lib/syncwrap/components/iyyov_daemon.rb +125 -0
  29. data/lib/syncwrap/components/jruby_vm.rb +122 -0
  30. data/lib/syncwrap/components/mdraid.rb +204 -0
  31. data/lib/syncwrap/components/network.rb +99 -0
  32. data/lib/syncwrap/components/open_jdk.rb +70 -0
  33. data/lib/syncwrap/components/postgresql.rb +159 -0
  34. data/lib/syncwrap/components/qpid.rb +303 -0
  35. data/lib/syncwrap/components/rhel.rb +71 -0
  36. data/lib/syncwrap/components/run_user.rb +99 -0
  37. data/lib/syncwrap/components/ubuntu.rb +85 -0
  38. data/lib/syncwrap/components/users.rb +200 -0
  39. data/lib/syncwrap/context.rb +260 -0
  40. data/lib/syncwrap/distro.rb +53 -60
  41. data/lib/syncwrap/formatter.rb +149 -0
  42. data/lib/syncwrap/host.rb +134 -0
  43. data/lib/syncwrap/main.rb +62 -0
  44. data/lib/syncwrap/path_util.rb +55 -0
  45. data/lib/syncwrap/rsync.rb +227 -0
  46. data/lib/syncwrap/ruby_support.rb +110 -0
  47. data/lib/syncwrap/shell.rb +207 -0
  48. data/lib/syncwrap.rb +367 -1
  49. data/{etc → sync/etc}/gemrc +1 -3
  50. data/sync/etc/hosts.erb +8 -0
  51. data/{etc/init.d/iyyov → sync/etc/init.d/iyyov.erb} +35 -7
  52. data/sync/etc/sysconfig/pgsql/postgresql.erb +2 -0
  53. data/sync/src/hashdot/Makefile.erb +98 -0
  54. data/sync/src/hashdot/profiles/default.hdp.erb +25 -0
  55. data/sync/src/hashdot/profiles/jruby-common.hdp +28 -0
  56. data/sync/src/hashdot/profiles/jruby-shortlived.hdp +9 -0
  57. data/sync/src/hashdot/profiles/jruby.hdp.erb +13 -0
  58. data/sync/src/hashdot/profiles/shortlived.hdp +6 -0
  59. data/sync/var/iyyov/default/config.rb +1 -0
  60. data/sync/var/iyyov/default/daemon.rb.erb +15 -0
  61. data/sync/var/iyyov/jobs.rb.erb +4 -0
  62. data/test/muddled_sync.rb +13 -0
  63. data/test/setup.rb +39 -0
  64. data/test/sync/d1/bar +1 -0
  65. data/test/sync/d1/foo.erb +1 -0
  66. data/test/sync/d3/d2/bar +1 -0
  67. data/test/sync/d3/d2/foo.erb +1 -0
  68. data/test/test_components.rb +108 -0
  69. data/test/test_context.rb +107 -0
  70. data/test/test_context_rput.rb +289 -0
  71. data/test/test_rsync.rb +138 -0
  72. data/test/test_shell.rb +233 -0
  73. data/test/test_space.rb +218 -0
  74. data/test/test_space_main.rb +40 -0
  75. data/test/zfile +1 -0
  76. metadata +204 -71
  77. data/etc/sysconfig/pgsql/postgresql +0 -2
  78. data/lib/syncwrap/aws.rb +0 -448
  79. data/lib/syncwrap/common.rb +0 -161
  80. data/lib/syncwrap/ec2.rb +0 -59
  81. data/lib/syncwrap/hashdot.rb +0 -70
  82. data/lib/syncwrap/iyyov.rb +0 -139
  83. data/lib/syncwrap/java.rb +0 -61
  84. data/lib/syncwrap/jruby.rb +0 -118
  85. data/lib/syncwrap/postgresql.rb +0 -135
  86. data/lib/syncwrap/qpid.rb +0 -251
  87. data/lib/syncwrap/remote_task.rb +0 -199
  88. data/lib/syncwrap/rhel.rb +0 -67
  89. data/lib/syncwrap/ubuntu.rb +0 -78
  90. data/lib/syncwrap/user_run.rb +0 -102
  91. data/test/test_syncwrap.rb +0 -202
  92. data/var/iyyov/jobs.rb +0 -11
  93. /data/{etc → sync/etc}/corosync/corosync.conf +0 -0
  94. /data/{etc → sync/etc}/corosync/uidgid.d/qpid +0 -0
  95. /data/{etc → sync/etc}/init.d/qpidd +0 -0
  96. /data/{etc → sync/etc}/sysctl.d/61-postgresql-shm.conf +0 -0
  97. /data/{usr/local → sync/jruby}/bin/jgem +0 -0
  98. /data/{postgresql → sync/postgresql}/rhel/pg_hba.conf +0 -0
  99. /data/{postgresql → sync/postgresql}/rhel/pg_ident.conf +0 -0
  100. /data/{postgresql → sync/postgresql}/rhel/postgresql.conf +0 -0
  101. /data/{postgresql → sync/postgresql}/ubuntu/environment +0 -0
  102. /data/{postgresql → sync/postgresql}/ubuntu/pg_ctl.conf +0 -0
  103. /data/{postgresql → sync/postgresql}/ubuntu/pg_hba.conf +0 -0
  104. /data/{postgresql → sync/postgresql}/ubuntu/pg_ident.conf +0 -0
  105. /data/{postgresql → sync/postgresql}/ubuntu/postgresql.conf +0 -0
  106. /data/{postgresql → sync/postgresql}/ubuntu/start.conf +0 -0
  107. /data/{usr → sync/usr}/local/etc/qpidd.conf +0 -0
@@ -0,0 +1,328 @@
1
+ #--
2
+ # Copyright (c) 2011-2014 David Kellum
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License"); you
5
+ # may not use this file except in compliance with the License. You
6
+ # may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13
+ # implied. See the License for the specific language governing
14
+ # permissions and limitations under the License.
15
+ #++
16
+
17
+ require 'syncwrap/base'
18
+
19
+ require 'optparse'
20
+ require 'term/ansicolor'
21
+
22
+ module SyncWrap
23
+
24
+ class CLI
25
+
26
+ def initialize
27
+ @sw_file = './sync.rb'
28
+ @options = {}
29
+ @list_hosts = false
30
+ @list_roles = false
31
+ @list_components = false
32
+ @component_plan = []
33
+ @roles = []
34
+ @host_patterns = []
35
+ @create_plan = []
36
+ @import_regions = []
37
+ @terminate_hosts = []
38
+ @delete_attached_storage = false
39
+ @space = Space.new
40
+ end
41
+
42
+ attr_reader :space
43
+
44
+ def parse_cmd( args )
45
+ opts = OptionParser.new do |opts|
46
+ opts.banner = "Usage: syncwrap {options} [Component[.method]] ..."
47
+
48
+ opts.on( "-f", "--file FILE",
49
+ "Load FILE for role/host/component definitions",
50
+ "(default: './sync.rb')" ) do |f|
51
+ @sw_file = f
52
+ end
53
+
54
+ opts.on( "-h", "--hosts PATTERN",
55
+ "Constrain hosts by pattern (may use multiple)" ) do |p|
56
+ @host_patterns << Regexp.new( p )
57
+ end
58
+
59
+ opts.on( "-r", "--hosts-with-role ROLE",
60
+ "Constrain hosts by role (may use multiple)" ) do |r|
61
+ @roles << r.sub(/^:/,'').to_sym
62
+ end
63
+
64
+ opts.on( "-n", "--dryrun",
65
+ "Run in \"dry run\", or no changes/test mode",
66
+ "(typically combined with -v)" ) do
67
+ @options[ :dryrun ] = true
68
+ end
69
+
70
+ opts.on( "-t", "--threads N",
71
+ "Specify the number of hosts to process concurrently",
72
+ "(default: all hosts)",
73
+ Integer ) do |n|
74
+ @options[ :threads ] = n
75
+ end
76
+
77
+ opts.on( "-e", "--each-component",
78
+ "Flush shell commands after each component/method" ) do
79
+ @options[ :flush_component ] = true
80
+ end
81
+
82
+ opts.on( "-s", "--no-coalesce",
83
+ "Do not coalesce streams (as is the default)" ) do
84
+ @options[ :coalesce ] = false
85
+ end
86
+
87
+ opts.on( "--no-color",
88
+ "Do not colorize output (as is the default)" ) do
89
+ @options[ :colorize ] = false
90
+ end
91
+
92
+ opts.on( "-v", "--verbose",
93
+ "Show details of local/remote command execution" ) do
94
+ @options[ :verbose ] = true
95
+ end
96
+
97
+ opts.on( "-x", "--expand-shell",
98
+ "Use -x (expand) instead of -v shell verbose output",
99
+ "(sh_verbose: :x, typically combined with -v)" ) do
100
+ @options[ :sh_verbose ] = :x
101
+ end
102
+
103
+ opts.on( "--version",
104
+ "Show syncwrap version and exit" ) do
105
+ puts "syncwrap: #{SyncWrap::VERSION}"
106
+ exit 0
107
+ end
108
+
109
+ opts.on( "--list-components",
110
+ "List selected components and exit" ) do
111
+ @list_components = true
112
+ end
113
+
114
+ opts.on( "--list-roles",
115
+ "List relevent roles and exit" ) do
116
+ @list_roles = true
117
+ end
118
+
119
+ opts.on( "--list-hosts",
120
+ "List selected hosts and exit" ) do
121
+ @list_hosts = true
122
+ end
123
+
124
+ opts.on( "-l", "--list",
125
+ "List selected roles and hosts, then exit" ) do
126
+ @list_roles = true
127
+ @list_hosts = true
128
+ end
129
+
130
+ opts.on( "-C", "--create-host P",
131
+ "Create hosts where P has format: [N*]profile[:name]",
132
+ " N: number to create (default: 1)",
133
+ " profile: the profile name as setup in the sync file",
134
+ " name: Host name, or prefix in the case on N>1",
135
+ "Hosts are appended to the sync file and space" ) do |h|
136
+ first,rest = h.split('*')
137
+ if rest
138
+ count = first.to_i
139
+ else
140
+ count = 1
141
+ rest = first
142
+ end
143
+ profile,name = rest.split(':')
144
+ profile = profile.to_sym
145
+ @create_plan << [ count, profile, name ]
146
+ end
147
+
148
+ opts.on( "-I", "--import-hosts REGIONS",
149
+ "Import hosts form provider 'region' names, ",
150
+ "append to sync file and exit." ) do |rs|
151
+ @import_regions = rs.split( /[\s,]+/ )
152
+ end
153
+
154
+ opts.on( "--terminate-host NAME",
155
+ "Terminate the specified instance and data via provider",
156
+ "WARNING: potential for data loss!" ) do |name|
157
+ @terminate_hosts << name
158
+ end
159
+
160
+ opts.on( "--delete-attached-storage",
161
+ "When terminating hosts, also delete any attached storage",
162
+ "volumes which wouldn't otherwise be deleted.",
163
+ "WARNING: Data WILL be lost!" ) do
164
+ @delete_attached_storage = true
165
+ end
166
+
167
+ end
168
+
169
+ @component_plan = opts.parse!( args )
170
+
171
+ rescue OptionParser::ParseError => e
172
+ $stderr.puts e.message
173
+ $stderr.puts opts
174
+ exit 3
175
+ end
176
+
177
+ def run( args )
178
+ parse_cmd( args )
179
+ space.load_sync_file( @sw_file )
180
+
181
+ if !@import_regions.empty?
182
+ if space.provider
183
+ space.provider.import_hosts( @import_regions, @sw_file )
184
+ exit 0
185
+ else
186
+ raise "No provider set in sync file/registered with Space"
187
+ end
188
+ end
189
+
190
+ if !@terminate_hosts.empty?
191
+ space.provider.terminate_hosts( @terminate_hosts,
192
+ @delete_attached_storage,
193
+ @sw_file )
194
+ exit 0
195
+ end
196
+
197
+ @create_plan.each do |count, profile, name|
198
+ space.provider.create_hosts( count, profile, name, @sw_file )
199
+ end
200
+
201
+ resolve_hosts
202
+ lookup_component_plan
203
+ resolve_components
204
+
205
+ lists = [ @list_components, @list_roles, @list_hosts ].count( true )
206
+ list_components( component_classes, lists > 1 ) if @list_components
207
+ list_roles( @hosts, lists > 1) if @list_roles
208
+ list_hosts( @hosts, lists > 1) if @list_hosts
209
+
210
+ exit( 0 ) if lists > 0
211
+
212
+ success = space.execute( @hosts, @component_plan, @options )
213
+ exit( success ? 0 : 1 )
214
+ end
215
+
216
+ def list_components( comp_classes, multi )
217
+ puts "Selected Components:" if multi
218
+ puts short_class_names( comp_classes ).join( ' ')
219
+ puts if multi
220
+ end
221
+
222
+ def list_roles( hosts, multi )
223
+ puts "Included Roles:" if multi
224
+ roles = hosts.map { |h| h.roles }.inject([],:|)
225
+ table = roles.map do |role|
226
+ row = [ ':' + role.to_s ]
227
+ classes = space.role( role ).map { |c| c.class }
228
+ row << short_class_names( classes ).join(' ')
229
+ end
230
+ print_table( table )
231
+ puts if multi
232
+ end
233
+
234
+ def list_hosts( hosts, multi )
235
+ names = []
236
+ puts "Selected Hosts:" if multi
237
+ table = hosts.map do |host|
238
+ row = [ host.name ]
239
+ row += host.contents.map do |c|
240
+ if c.is_a?( Symbol )
241
+ ':' + c.to_s
242
+ else
243
+ short_class_names( [c.class], names ).first
244
+ end
245
+ end
246
+ end
247
+ print_table( table )
248
+ end
249
+
250
+ def print_table( table )
251
+ max_columns = table.map { |r| r.count }.max || 0
252
+ col_widths = max_columns.times.map do |i|
253
+ table.map { |row| row[i] && row[i].length }.compact.max
254
+ end
255
+ format = col_widths.inject("") do |f,w|
256
+ if f.empty? #first
257
+ f << "%-#{w}s "
258
+ else
259
+ f << "%-#{w}s "
260
+ end
261
+ end
262
+
263
+ table.each do |row|
264
+ row[ max_columns ] = nil
265
+ puts format % row
266
+ end
267
+ end
268
+
269
+ # Return a new Array of classes mapped to the shortest possible
270
+ # non-duplicate name.
271
+ def short_class_names( classes, names = [] )
272
+ classes.map do |cls|
273
+ segs = cls.name.split( '::' )
274
+ (0...segs.length).reverse_each do |s|
275
+ tn = segs[s..-1].join( '::' )
276
+ if !names.include?( tn )
277
+ break tn
278
+ end
279
+ end
280
+ end
281
+ end
282
+
283
+ def lookup_component_plan
284
+ @component_plan.map! do |comp|
285
+ name, meth = comp.split( '.' )
286
+ [ name, meth ]
287
+ end
288
+ @component_plan = space.resolve_component_plan( @component_plan )
289
+ end
290
+
291
+ def component_classes
292
+ if @component_plan.empty?
293
+ space.component_classes( @hosts )
294
+ else
295
+ @component_plan.map { |a| a[0] }
296
+ end
297
+ end
298
+
299
+ def resolve_components
300
+ if ! @component_plan.empty?
301
+ @hosts.select! do |host|
302
+ host.components.any? do |comp|
303
+ component_classes.any? { |cc| comp.is_a?( cc ) }
304
+ end
305
+ end
306
+ end
307
+ end
308
+
309
+ def resolve_hosts
310
+ @hosts = space.hosts
311
+ unless @host_patterns.empty?
312
+ @hosts.select! do |host|
313
+ @host_patterns.any? do |pat|
314
+ md = pat.match( host.name )
315
+ md && md[0] == host.name
316
+ end
317
+ end
318
+ end
319
+
320
+ unless @roles.empty?
321
+ @hosts.select! do |host|
322
+ host.roles.any? { |r| @roles.include?( r ) }
323
+ end
324
+ end
325
+ end
326
+
327
+ end
328
+ end