syncwrap 1.5.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. checksums.yaml +7 -0
  2. data/History.rdoc +19 -0
  3. data/Manifest.txt +82 -34
  4. data/README.rdoc +96 -48
  5. data/Rakefile +0 -65
  6. data/bin/syncwrap +27 -0
  7. data/examples/LAYOUT.rdoc +70 -0
  8. data/examples/Rakefile +16 -0
  9. data/examples/ec2.rb +44 -0
  10. data/examples/hello.rb +14 -0
  11. data/examples/hello_binding.rb +27 -0
  12. data/examples/jruby.rb +11 -0
  13. data/examples/private/aws.json +4 -0
  14. data/examples/rput.rb +24 -0
  15. data/examples/sync/home/bob/.ssh/authorized_keys +1 -0
  16. data/examples/sync/tmp/sample.erb +3 -0
  17. data/lib/syncwrap/amazon_ec2.rb +236 -0
  18. data/lib/syncwrap/amazon_ws.rb +308 -0
  19. data/lib/syncwrap/base.rb +4 -2
  20. data/lib/syncwrap/cli.rb +328 -0
  21. data/lib/syncwrap/component.rb +443 -0
  22. data/lib/syncwrap/components/commercial_jdk.rb +76 -0
  23. data/lib/syncwrap/components/cruby_vm.rb +144 -0
  24. data/lib/syncwrap/components/etc_hosts.rb +44 -0
  25. data/lib/syncwrap/{geminabox.rb → components/geminabox.rb} +12 -17
  26. data/lib/syncwrap/components/hashdot.rb +97 -0
  27. data/lib/syncwrap/components/iyyov.rb +144 -0
  28. data/lib/syncwrap/components/iyyov_daemon.rb +125 -0
  29. data/lib/syncwrap/components/jruby_vm.rb +122 -0
  30. data/lib/syncwrap/components/mdraid.rb +204 -0
  31. data/lib/syncwrap/components/network.rb +99 -0
  32. data/lib/syncwrap/components/open_jdk.rb +70 -0
  33. data/lib/syncwrap/components/postgresql.rb +159 -0
  34. data/lib/syncwrap/components/qpid.rb +303 -0
  35. data/lib/syncwrap/components/rhel.rb +71 -0
  36. data/lib/syncwrap/components/run_user.rb +99 -0
  37. data/lib/syncwrap/components/ubuntu.rb +85 -0
  38. data/lib/syncwrap/components/users.rb +200 -0
  39. data/lib/syncwrap/context.rb +260 -0
  40. data/lib/syncwrap/distro.rb +53 -60
  41. data/lib/syncwrap/formatter.rb +149 -0
  42. data/lib/syncwrap/host.rb +134 -0
  43. data/lib/syncwrap/main.rb +62 -0
  44. data/lib/syncwrap/path_util.rb +55 -0
  45. data/lib/syncwrap/rsync.rb +227 -0
  46. data/lib/syncwrap/ruby_support.rb +110 -0
  47. data/lib/syncwrap/shell.rb +207 -0
  48. data/lib/syncwrap.rb +367 -1
  49. data/{etc → sync/etc}/gemrc +1 -3
  50. data/sync/etc/hosts.erb +8 -0
  51. data/{etc/init.d/iyyov → sync/etc/init.d/iyyov.erb} +35 -7
  52. data/sync/etc/sysconfig/pgsql/postgresql.erb +2 -0
  53. data/sync/src/hashdot/Makefile.erb +98 -0
  54. data/sync/src/hashdot/profiles/default.hdp.erb +25 -0
  55. data/sync/src/hashdot/profiles/jruby-common.hdp +28 -0
  56. data/sync/src/hashdot/profiles/jruby-shortlived.hdp +9 -0
  57. data/sync/src/hashdot/profiles/jruby.hdp.erb +13 -0
  58. data/sync/src/hashdot/profiles/shortlived.hdp +6 -0
  59. data/sync/var/iyyov/default/config.rb +1 -0
  60. data/sync/var/iyyov/default/daemon.rb.erb +15 -0
  61. data/sync/var/iyyov/jobs.rb.erb +4 -0
  62. data/test/muddled_sync.rb +13 -0
  63. data/test/setup.rb +39 -0
  64. data/test/sync/d1/bar +1 -0
  65. data/test/sync/d1/foo.erb +1 -0
  66. data/test/sync/d3/d2/bar +1 -0
  67. data/test/sync/d3/d2/foo.erb +1 -0
  68. data/test/test_components.rb +108 -0
  69. data/test/test_context.rb +107 -0
  70. data/test/test_context_rput.rb +289 -0
  71. data/test/test_rsync.rb +138 -0
  72. data/test/test_shell.rb +233 -0
  73. data/test/test_space.rb +218 -0
  74. data/test/test_space_main.rb +40 -0
  75. data/test/zfile +1 -0
  76. metadata +204 -71
  77. data/etc/sysconfig/pgsql/postgresql +0 -2
  78. data/lib/syncwrap/aws.rb +0 -448
  79. data/lib/syncwrap/common.rb +0 -161
  80. data/lib/syncwrap/ec2.rb +0 -59
  81. data/lib/syncwrap/hashdot.rb +0 -70
  82. data/lib/syncwrap/iyyov.rb +0 -139
  83. data/lib/syncwrap/java.rb +0 -61
  84. data/lib/syncwrap/jruby.rb +0 -118
  85. data/lib/syncwrap/postgresql.rb +0 -135
  86. data/lib/syncwrap/qpid.rb +0 -251
  87. data/lib/syncwrap/remote_task.rb +0 -199
  88. data/lib/syncwrap/rhel.rb +0 -67
  89. data/lib/syncwrap/ubuntu.rb +0 -78
  90. data/lib/syncwrap/user_run.rb +0 -102
  91. data/test/test_syncwrap.rb +0 -202
  92. data/var/iyyov/jobs.rb +0 -11
  93. /data/{etc → sync/etc}/corosync/corosync.conf +0 -0
  94. /data/{etc → sync/etc}/corosync/uidgid.d/qpid +0 -0
  95. /data/{etc → sync/etc}/init.d/qpidd +0 -0
  96. /data/{etc → sync/etc}/sysctl.d/61-postgresql-shm.conf +0 -0
  97. /data/{usr/local → sync/jruby}/bin/jgem +0 -0
  98. /data/{postgresql → sync/postgresql}/rhel/pg_hba.conf +0 -0
  99. /data/{postgresql → sync/postgresql}/rhel/pg_ident.conf +0 -0
  100. /data/{postgresql → sync/postgresql}/rhel/postgresql.conf +0 -0
  101. /data/{postgresql → sync/postgresql}/ubuntu/environment +0 -0
  102. /data/{postgresql → sync/postgresql}/ubuntu/pg_ctl.conf +0 -0
  103. /data/{postgresql → sync/postgresql}/ubuntu/pg_hba.conf +0 -0
  104. /data/{postgresql → sync/postgresql}/ubuntu/pg_ident.conf +0 -0
  105. /data/{postgresql → sync/postgresql}/ubuntu/postgresql.conf +0 -0
  106. /data/{postgresql → sync/postgresql}/ubuntu/start.conf +0 -0
  107. /data/{usr → sync/usr}/local/etc/qpidd.conf +0 -0
@@ -0,0 +1,328 @@
1
+ #--
2
+ # Copyright (c) 2011-2014 David Kellum
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License"); you
5
+ # may not use this file except in compliance with the License. You
6
+ # may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13
+ # implied. See the License for the specific language governing
14
+ # permissions and limitations under the License.
15
+ #++
16
+
17
+ require 'syncwrap/base'
18
+
19
+ require 'optparse'
20
+ require 'term/ansicolor'
21
+
22
+ module SyncWrap
23
+
24
+ class CLI
25
+
26
+ def initialize
27
+ @sw_file = './sync.rb'
28
+ @options = {}
29
+ @list_hosts = false
30
+ @list_roles = false
31
+ @list_components = false
32
+ @component_plan = []
33
+ @roles = []
34
+ @host_patterns = []
35
+ @create_plan = []
36
+ @import_regions = []
37
+ @terminate_hosts = []
38
+ @delete_attached_storage = false
39
+ @space = Space.new
40
+ end
41
+
42
+ attr_reader :space
43
+
44
+ def parse_cmd( args )
45
+ opts = OptionParser.new do |opts|
46
+ opts.banner = "Usage: syncwrap {options} [Component[.method]] ..."
47
+
48
+ opts.on( "-f", "--file FILE",
49
+ "Load FILE for role/host/component definitions",
50
+ "(default: './sync.rb')" ) do |f|
51
+ @sw_file = f
52
+ end
53
+
54
+ opts.on( "-h", "--hosts PATTERN",
55
+ "Constrain hosts by pattern (may use multiple)" ) do |p|
56
+ @host_patterns << Regexp.new( p )
57
+ end
58
+
59
+ opts.on( "-r", "--hosts-with-role ROLE",
60
+ "Constrain hosts by role (may use multiple)" ) do |r|
61
+ @roles << r.sub(/^:/,'').to_sym
62
+ end
63
+
64
+ opts.on( "-n", "--dryrun",
65
+ "Run in \"dry run\", or no changes/test mode",
66
+ "(typically combined with -v)" ) do
67
+ @options[ :dryrun ] = true
68
+ end
69
+
70
+ opts.on( "-t", "--threads N",
71
+ "Specify the number of hosts to process concurrently",
72
+ "(default: all hosts)",
73
+ Integer ) do |n|
74
+ @options[ :threads ] = n
75
+ end
76
+
77
+ opts.on( "-e", "--each-component",
78
+ "Flush shell commands after each component/method" ) do
79
+ @options[ :flush_component ] = true
80
+ end
81
+
82
+ opts.on( "-s", "--no-coalesce",
83
+ "Do not coalesce streams (as is the default)" ) do
84
+ @options[ :coalesce ] = false
85
+ end
86
+
87
+ opts.on( "--no-color",
88
+ "Do not colorize output (as is the default)" ) do
89
+ @options[ :colorize ] = false
90
+ end
91
+
92
+ opts.on( "-v", "--verbose",
93
+ "Show details of local/remote command execution" ) do
94
+ @options[ :verbose ] = true
95
+ end
96
+
97
+ opts.on( "-x", "--expand-shell",
98
+ "Use -x (expand) instead of -v shell verbose output",
99
+ "(sh_verbose: :x, typically combined with -v)" ) do
100
+ @options[ :sh_verbose ] = :x
101
+ end
102
+
103
+ opts.on( "--version",
104
+ "Show syncwrap version and exit" ) do
105
+ puts "syncwrap: #{SyncWrap::VERSION}"
106
+ exit 0
107
+ end
108
+
109
+ opts.on( "--list-components",
110
+ "List selected components and exit" ) do
111
+ @list_components = true
112
+ end
113
+
114
+ opts.on( "--list-roles",
115
+ "List relevent roles and exit" ) do
116
+ @list_roles = true
117
+ end
118
+
119
+ opts.on( "--list-hosts",
120
+ "List selected hosts and exit" ) do
121
+ @list_hosts = true
122
+ end
123
+
124
+ opts.on( "-l", "--list",
125
+ "List selected roles and hosts, then exit" ) do
126
+ @list_roles = true
127
+ @list_hosts = true
128
+ end
129
+
130
+ opts.on( "-C", "--create-host P",
131
+ "Create hosts where P has format: [N*]profile[:name]",
132
+ " N: number to create (default: 1)",
133
+ " profile: the profile name as setup in the sync file",
134
+ " name: Host name, or prefix in the case on N>1",
135
+ "Hosts are appended to the sync file and space" ) do |h|
136
+ first,rest = h.split('*')
137
+ if rest
138
+ count = first.to_i
139
+ else
140
+ count = 1
141
+ rest = first
142
+ end
143
+ profile,name = rest.split(':')
144
+ profile = profile.to_sym
145
+ @create_plan << [ count, profile, name ]
146
+ end
147
+
148
+ opts.on( "-I", "--import-hosts REGIONS",
149
+ "Import hosts form provider 'region' names, ",
150
+ "append to sync file and exit." ) do |rs|
151
+ @import_regions = rs.split( /[\s,]+/ )
152
+ end
153
+
154
+ opts.on( "--terminate-host NAME",
155
+ "Terminate the specified instance and data via provider",
156
+ "WARNING: potential for data loss!" ) do |name|
157
+ @terminate_hosts << name
158
+ end
159
+
160
+ opts.on( "--delete-attached-storage",
161
+ "When terminating hosts, also delete any attached storage",
162
+ "volumes which wouldn't otherwise be deleted.",
163
+ "WARNING: Data WILL be lost!" ) do
164
+ @delete_attached_storage = true
165
+ end
166
+
167
+ end
168
+
169
+ @component_plan = opts.parse!( args )
170
+
171
+ rescue OptionParser::ParseError => e
172
+ $stderr.puts e.message
173
+ $stderr.puts opts
174
+ exit 3
175
+ end
176
+
177
+ def run( args )
178
+ parse_cmd( args )
179
+ space.load_sync_file( @sw_file )
180
+
181
+ if !@import_regions.empty?
182
+ if space.provider
183
+ space.provider.import_hosts( @import_regions, @sw_file )
184
+ exit 0
185
+ else
186
+ raise "No provider set in sync file/registered with Space"
187
+ end
188
+ end
189
+
190
+ if !@terminate_hosts.empty?
191
+ space.provider.terminate_hosts( @terminate_hosts,
192
+ @delete_attached_storage,
193
+ @sw_file )
194
+ exit 0
195
+ end
196
+
197
+ @create_plan.each do |count, profile, name|
198
+ space.provider.create_hosts( count, profile, name, @sw_file )
199
+ end
200
+
201
+ resolve_hosts
202
+ lookup_component_plan
203
+ resolve_components
204
+
205
+ lists = [ @list_components, @list_roles, @list_hosts ].count( true )
206
+ list_components( component_classes, lists > 1 ) if @list_components
207
+ list_roles( @hosts, lists > 1) if @list_roles
208
+ list_hosts( @hosts, lists > 1) if @list_hosts
209
+
210
+ exit( 0 ) if lists > 0
211
+
212
+ success = space.execute( @hosts, @component_plan, @options )
213
+ exit( success ? 0 : 1 )
214
+ end
215
+
216
+ def list_components( comp_classes, multi )
217
+ puts "Selected Components:" if multi
218
+ puts short_class_names( comp_classes ).join( ' ')
219
+ puts if multi
220
+ end
221
+
222
+ def list_roles( hosts, multi )
223
+ puts "Included Roles:" if multi
224
+ roles = hosts.map { |h| h.roles }.inject([],:|)
225
+ table = roles.map do |role|
226
+ row = [ ':' + role.to_s ]
227
+ classes = space.role( role ).map { |c| c.class }
228
+ row << short_class_names( classes ).join(' ')
229
+ end
230
+ print_table( table )
231
+ puts if multi
232
+ end
233
+
234
+ def list_hosts( hosts, multi )
235
+ names = []
236
+ puts "Selected Hosts:" if multi
237
+ table = hosts.map do |host|
238
+ row = [ host.name ]
239
+ row += host.contents.map do |c|
240
+ if c.is_a?( Symbol )
241
+ ':' + c.to_s
242
+ else
243
+ short_class_names( [c.class], names ).first
244
+ end
245
+ end
246
+ end
247
+ print_table( table )
248
+ end
249
+
250
+ def print_table( table )
251
+ max_columns = table.map { |r| r.count }.max || 0
252
+ col_widths = max_columns.times.map do |i|
253
+ table.map { |row| row[i] && row[i].length }.compact.max
254
+ end
255
+ format = col_widths.inject("") do |f,w|
256
+ if f.empty? #first
257
+ f << "%-#{w}s "
258
+ else
259
+ f << "%-#{w}s "
260
+ end
261
+ end
262
+
263
+ table.each do |row|
264
+ row[ max_columns ] = nil
265
+ puts format % row
266
+ end
267
+ end
268
+
269
+ # Return a new Array of classes mapped to the shortest possible
270
+ # non-duplicate name.
271
+ def short_class_names( classes, names = [] )
272
+ classes.map do |cls|
273
+ segs = cls.name.split( '::' )
274
+ (0...segs.length).reverse_each do |s|
275
+ tn = segs[s..-1].join( '::' )
276
+ if !names.include?( tn )
277
+ break tn
278
+ end
279
+ end
280
+ end
281
+ end
282
+
283
+ def lookup_component_plan
284
+ @component_plan.map! do |comp|
285
+ name, meth = comp.split( '.' )
286
+ [ name, meth ]
287
+ end
288
+ @component_plan = space.resolve_component_plan( @component_plan )
289
+ end
290
+
291
+ def component_classes
292
+ if @component_plan.empty?
293
+ space.component_classes( @hosts )
294
+ else
295
+ @component_plan.map { |a| a[0] }
296
+ end
297
+ end
298
+
299
+ def resolve_components
300
+ if ! @component_plan.empty?
301
+ @hosts.select! do |host|
302
+ host.components.any? do |comp|
303
+ component_classes.any? { |cc| comp.is_a?( cc ) }
304
+ end
305
+ end
306
+ end
307
+ end
308
+
309
+ def resolve_hosts
310
+ @hosts = space.hosts
311
+ unless @host_patterns.empty?
312
+ @hosts.select! do |host|
313
+ @host_patterns.any? do |pat|
314
+ md = pat.match( host.name )
315
+ md && md[0] == host.name
316
+ end
317
+ end
318
+ end
319
+
320
+ unless @roles.empty?
321
+ @hosts.select! do |host|
322
+ host.roles.any? { |r| @roles.include?( r ) }
323
+ end
324
+ end
325
+ end
326
+
327
+ end
328
+ end