bitgirder-platform 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. data/LICENSE.txt +176 -0
  2. data/bin/ensure-test-db +117 -0
  3. data/bin/install-mysql +375 -0
  4. data/bin/tomcat7 +569 -0
  5. data/lib/bitgirder/concurrent.rb +400 -0
  6. data/lib/bitgirder/core.rb +1235 -0
  7. data/lib/bitgirder/etl.rb +58 -0
  8. data/lib/bitgirder/event/file.rb +485 -0
  9. data/lib/bitgirder/event/logger/testing.rb +140 -0
  10. data/lib/bitgirder/event/logger.rb +137 -0
  11. data/lib/bitgirder/event/testing.rb +88 -0
  12. data/lib/bitgirder/http.rb +255 -0
  13. data/lib/bitgirder/io/testing.rb +33 -0
  14. data/lib/bitgirder/io.rb +959 -0
  15. data/lib/bitgirder/irb.rb +35 -0
  16. data/lib/bitgirder/mysql.rb +60 -0
  17. data/lib/bitgirder/ops/java.rb +117 -0
  18. data/lib/bitgirder/ops/ruby.rb +235 -0
  19. data/lib/bitgirder/testing.rb +152 -0
  20. data/lib/doc-gen0.rb +0 -0
  21. data/lib/doc-gen1.rb +0 -0
  22. data/lib/doc-gen10.rb +0 -0
  23. data/lib/doc-gen11.rb +0 -0
  24. data/lib/doc-gen12.rb +0 -0
  25. data/lib/doc-gen13.rb +0 -0
  26. data/lib/doc-gen14.rb +0 -0
  27. data/lib/doc-gen15.rb +0 -0
  28. data/lib/doc-gen16.rb +0 -0
  29. data/lib/doc-gen17.rb +14 -0
  30. data/lib/doc-gen18.rb +0 -0
  31. data/lib/doc-gen19.rb +0 -0
  32. data/lib/doc-gen2.rb +0 -0
  33. data/lib/doc-gen20.rb +182 -0
  34. data/lib/doc-gen21.rb +0 -0
  35. data/lib/doc-gen3.rb +0 -0
  36. data/lib/doc-gen4.rb +0 -0
  37. data/lib/doc-gen5.rb +0 -0
  38. data/lib/doc-gen6.rb +0 -0
  39. data/lib/doc-gen7.rb +0 -0
  40. data/lib/doc-gen8.rb +0 -0
  41. data/lib/doc-gen9.rb +0 -0
  42. data/lib/mingle/bincodec.rb +512 -0
  43. data/lib/mingle/codec.rb +54 -0
  44. data/lib/mingle/http.rb +156 -0
  45. data/lib/mingle/io/stream.rb +142 -0
  46. data/lib/mingle/io.rb +160 -0
  47. data/lib/mingle/json.rb +257 -0
  48. data/lib/mingle/service.rb +110 -0
  49. data/lib/mingle-em.rb +92 -0
  50. data/lib/mingle.rb +2917 -0
  51. metadata +100 -0
data/bin/tomcat7 ADDED
@@ -0,0 +1,569 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bitgirder/core'
4
+ include BitGirder::Core
5
+
6
+ require 'bitgirder/io'
7
+
8
+ require 'bitgirder/ops/java'
9
+ include BitGirder::Ops::Java
10
+
11
+ require 'mingle'
12
+ include Mingle
13
+
14
+ require 'mingle/json'
15
+
16
+ require 'rexml/document'
17
+
18
+ require 'socket'
19
+
20
+ ENV_TOMCAT7_HOME = "TOMCAT7_HOME"
21
+
22
+ class Command < BitGirderClass
23
+
24
+ include BitGirder::Io
25
+
26
+ private
27
+ def load_struct( f )
28
+
29
+ File.open( f ) do |io|
30
+ Mingle::Json::JsonMingleCodec.new.from_buffer( io.read( io.size ) )
31
+ end
32
+ end
33
+
34
+ private
35
+ def save_struct( f, s )
36
+
37
+ File.open( ensure_parent( f ), "w" ) do |io|
38
+ io.print Mingle::Json::JsonMingleCodec.new.as_buffer( s )
39
+ end
40
+ end
41
+
42
+ private
43
+ def load_config( f )
44
+
45
+ flds = load_struct( f ).fields
46
+
47
+ TomcatConfig.new(
48
+ control_port: flds.expect_int( :control_port ),
49
+ http_port: flds.expect_int( :http_port ),
50
+ https_port: flds.expect_int( :https_port ),
51
+ shutdown_string: flds.expect_string( :shutdown_string ),
52
+ tomcat_home: flds.expect_string( :tomcat_home ),
53
+ java_home: flds.expect_string( :java_home )
54
+ )
55
+ end
56
+
57
+ private
58
+ def load_runtime_config( rt )
59
+ load_config( "#{rt}/config.json" )
60
+ end
61
+ end
62
+
63
+ class Help < Command
64
+
65
+ public
66
+ def run( run_ctx )
67
+
68
+ print "\ncommands:\n\n"
69
+ COMMANDS.keys.sort.each { |s| puts "#{ " " * 4 }#{s}" }
70
+ puts
71
+
72
+ end
73
+ end
74
+
75
+ class TomcatConfig < BitGirderClass
76
+
77
+ bg_attr :control_port, validation: :positive
78
+ bg_attr :http_port, validation: :positive
79
+ bg_attr :https_port, validation: :positive
80
+
81
+ bg_attr :shutdown_string
82
+
83
+ bg_attr :tomcat_home, validation: :file_exists
84
+
85
+ bg_attr :java_home,
86
+ validation: :file_exists,
87
+ default: lambda { JavaEnvironments.get_java_home }
88
+
89
+ public
90
+ def to_mingle_struct
91
+
92
+ MingleStruct.new(
93
+ type: :"bitgirder:ops:tomcat@v1/TomcatConfig",
94
+ fields: {
95
+ control_port: @control_port,
96
+ http_port: @http_port,
97
+ https_port: @https_port,
98
+ shutdown_string: @shutdown_string,
99
+ tomcat_home: @tomcat_home,
100
+ java_home: @java_home
101
+ }
102
+ )
103
+ end
104
+ end
105
+
106
+ PORT_OPTS = { validation: :positive, processor: :integer }
107
+
108
+ class Install < Command
109
+
110
+ PROTO_HTTP11 = "HTTP/1.1"
111
+
112
+ bg_attr :install_to
113
+
114
+ bg_attr :tomcat_home, validation: :file_exists
115
+
116
+ bg_attr :control_port, PORT_OPTS
117
+
118
+ bg_attr :http_port, PORT_OPTS
119
+
120
+ bg_attr :https_port, PORT_OPTS.merge( required: false )
121
+
122
+ bg_attr :ssl_keystore, validation: :file_exists, required: false
123
+
124
+ bg_attr :ssl_keystore_pass, required: false
125
+
126
+ bg_attr :ssl_keystore_type, required: false
127
+
128
+ bg_attr :ssl_key_alias, required: false
129
+
130
+ bg_attr :ssl_key_pass, required: false
131
+
132
+ bg_attr :wipe_install, required: false, processor: :boolean
133
+
134
+ bg_attr :webapps,
135
+ is_list: true,
136
+ processor: lambda { |s|
137
+ idx = s.index( "=" ) || s.size
138
+ dest = s[ 0, idx ]
139
+ if idx < s.size - 1
140
+ [ dest, s[ idx + 1 .. -1 ] ]
141
+ else
142
+ raise "Need a name for webapp #{dest}"
143
+ end
144
+ },
145
+ list_validation: :not_empty
146
+
147
+ bg_attr :java_home,
148
+ default: lambda { JavaEnvironments.get_java_home }
149
+
150
+ private
151
+ def impl_initialize
152
+
153
+ if @https_port && ( ! @ssl_keystore )
154
+ raise "Cannot configure SSL without at least a keystore"
155
+ end
156
+
157
+ if @ssl_keystore && ( ! @https_port )
158
+ raise "Need a port for SSL listener"
159
+ end
160
+ end
161
+
162
+ private
163
+ def base_dir( dir )
164
+ "#@install_to/catalina/#{dir}"
165
+ end
166
+
167
+ private
168
+ def init_config
169
+
170
+ @config = TomcatConfig.new(
171
+ control_port: @control_port,
172
+ http_port: @http_port,
173
+ https_port: @https_port,
174
+ shutdown_string: rand( 2 ** 128 ).to_s( 16 ),
175
+ tomcat_home: @tomcat_home,
176
+ java_home: @java_home
177
+ )
178
+ end
179
+
180
+ private
181
+ def setup_install_dir
182
+
183
+ if File.exist?( @install_to )
184
+ if @wipe_install
185
+ fu().rm_rf( @install_to )
186
+ else
187
+ raise "Install dir #@install_to exists (--wipe-install?)"
188
+ end
189
+ end
190
+
191
+ %w{ conf logs webapps work temp }.each do |dir|
192
+ ensure_dir( base_dir( dir ) )
193
+ end
194
+
195
+ fu().cp( "#@tomcat_home/conf/web.xml", base_dir( "conf" ) )
196
+ end
197
+
198
+ private
199
+ def new_elt( name, attrs )
200
+
201
+ res = REXML::Element.new( name.to_s )
202
+
203
+ attrs.each_pair do |k, v|
204
+ res.add_attribute( k.to_s, v.to_s ) unless v.nil?
205
+ end
206
+
207
+ res
208
+ end
209
+
210
+ private
211
+ def add_listener_elts( doc )
212
+
213
+ %w{ org.apache.catalina.core.JasperListener
214
+ org.apache.catalina.core.JreMemoryLeakPreventionListener
215
+ org.apache.catalina.mbeans.GlobalResourcesLifecycleListener
216
+ org.apache.catalina.core.ThreadLocalLeakPreventionListener
217
+ }.
218
+ each { |s| doc.root << new_elt( :Listener, :className => s ) }
219
+ end
220
+
221
+ private
222
+ def add_http_connector( svc )
223
+
224
+ svc << new_elt( :Connector,
225
+ :port => @config.http_port,
226
+ :protocol => PROTO_HTTP11,
227
+ :connectionTimeout => 20000,
228
+ :redirectPort => @config.https_port
229
+ )
230
+ end
231
+
232
+ private
233
+ def add_https_connector( svc )
234
+
235
+ svc << new_elt( :Connector,
236
+ :port => @config.https_port,
237
+ :protocol => PROTO_HTTP11,
238
+ :SSLEnabled => true,
239
+ :maxThreads => 150,
240
+ :scheme => "https",
241
+ :secure => true,
242
+ :clientAuth => false,
243
+ :sslProtocol => :TLS,
244
+ :keystoreFile => @ssl_keystore,
245
+ :keystorePass => @ssl_keystore_pass,
246
+ :keystoreType => @ssl_keystore_type,
247
+ :keyAlias => @ssl_key_alias,
248
+ :keyPass => @ssl_key_pass
249
+ )
250
+ end
251
+
252
+ private
253
+ def add_engine( svc )
254
+
255
+ eng = new_elt( :Engine, name: "Catalina", defaultHost: "localhost" )
256
+
257
+ valve = new_elt( :Valve,
258
+ className: "org.apache.catalina.valves.AccessLogValve",
259
+ directory: "logs",
260
+ prefix: "localhost_access_log.",
261
+ suffix: ".txt",
262
+ pattern: '%h %l %u %t "%r" %s %b'
263
+ )
264
+
265
+ host = new_elt( :Host,
266
+ name: "localhost",
267
+ appBase: "webapps",
268
+ unpackWARs: true,
269
+ autoDeply: true
270
+ )
271
+
272
+ host << valve
273
+ eng << host
274
+ svc << eng
275
+ end
276
+
277
+ private
278
+ def add_service_elt( doc )
279
+
280
+ doc.root << ( svc = new_elt( :Service, :name => "Catalina" ) )
281
+
282
+ add_http_connector( svc )
283
+ add_https_connector( svc )
284
+
285
+ add_engine( svc )
286
+ end
287
+
288
+ private
289
+ def build_server_conf
290
+
291
+ doc = REXML::Document.new
292
+ doc << REXML::XMLDecl.new( "1.0", "utf-8" )
293
+
294
+ doc << new_elt( :Server,
295
+ port: @config.control_port,
296
+ shutdown: @config.shutdown_string
297
+ )
298
+
299
+ add_listener_elts( doc )
300
+ add_service_elt( doc )
301
+
302
+ doc
303
+ end
304
+
305
+ private
306
+ def write_server_conf
307
+
308
+ doc = build_server_conf
309
+
310
+ File.open( "#{base_dir( :conf )}/server.xml", "w" ) do |io|
311
+ doc.write( io, 4 )
312
+ io.print "\n"
313
+ end
314
+ end
315
+
316
+ private
317
+ def install_webapps
318
+
319
+ @webapps.each do |app|
320
+
321
+ dest, src = *app
322
+ dest = ensure_parent( dest )
323
+ fu().ln_s( src, "#{base_dir( :webapps )}/#{dest}" )
324
+ end
325
+ end
326
+
327
+ private
328
+ def write_config
329
+ save_struct( "#@install_to/config.json", @config.to_mingle_struct )
330
+ end
331
+
332
+ public
333
+ def run
334
+
335
+ init_config
336
+ setup_install_dir
337
+ write_server_conf
338
+ install_webapps
339
+ write_config
340
+ end
341
+ end
342
+
343
+ class TomcatProcessBuilder < BitGirderClass
344
+
345
+ bg_attr :runtime
346
+ bg_attr :config
347
+
348
+ bg_attr :proc_opts, default: lambda { {} }
349
+
350
+ include BitGirder::Io
351
+
352
+ private
353
+ def get_run_classpath
354
+
355
+ %w{ bootstrap.jar tomcat-juli.jar }.map do |f|
356
+ file_exists( "#{@config.tomcat_home}/bin/#{f}" )
357
+ end
358
+ end
359
+
360
+ private
361
+ def get_run_sysprops
362
+
363
+ cat_home = @config.tomcat_home
364
+
365
+ res = {
366
+
367
+ "java.util.logging.config.file" =>
368
+ file_exists( "#{cat_home}/conf/logging.properties" ),
369
+
370
+ "java.util.logging.manager" =>
371
+ "org.apache.juli.ClassLoaderLogManager",
372
+
373
+ "catalina.base" => "#@runtime/catalina",
374
+ "catalina.home" => cat_home,
375
+
376
+ "java.io.tmpdir" => file_exists( "#@runtime/catalina/temp" )
377
+ }
378
+ end
379
+
380
+ private
381
+ def get_run_argv
382
+ %w{ start }
383
+ end
384
+
385
+ public
386
+ def build
387
+
388
+ JavaRunner.create_application_runner(
389
+ java_env: JavaEnvironment.new( java_home: @config.java_home ),
390
+ command: "java",
391
+ classpath: get_run_classpath,
392
+ sys_props: get_run_sysprops,
393
+ main: "org.apache.catalina.startup.Bootstrap",
394
+ argv: get_run_argv,
395
+ proc_opts: @proc_opts
396
+ ).
397
+ process_builder
398
+ end
399
+ end
400
+
401
+ class RuntimeCommand < Command
402
+
403
+ bg_attr :runtime,
404
+ validation: :file_exists,
405
+ default: ENV[ ENV_TOMCAT7_HOME ]
406
+
407
+ include BitGirder::Io
408
+
409
+ private
410
+ def impl_initialize
411
+ @config = load_config( file_exists( "#@runtime/config.json" ) )
412
+ end
413
+
414
+ private
415
+ def pid_file
416
+ "#@runtime/tomcat.pid"
417
+ end
418
+ end
419
+
420
+ class Run < RuntimeCommand
421
+
422
+ public
423
+ def run
424
+
425
+ TomcatProcessBuilder.
426
+ new( config: @config, runtime: @runtime ).
427
+ build.
428
+ exec
429
+ end
430
+ end
431
+
432
+ class Daemon < RuntimeCommand
433
+
434
+ private
435
+ def do_start
436
+
437
+ b = TomcatProcessBuilder.new( config: @config, runtime: @runtime )
438
+
439
+ Process.daemon
440
+ File.open( pid_file, "w" ) { |io| io.puts $$ }
441
+
442
+ out = "#@runtime/catalina/logs/catalina.out"
443
+ b.proc_opts[ [ STDOUT, STDERR ] ] = File.open( out, "a" )
444
+
445
+ b.build.exec
446
+ end
447
+
448
+ private
449
+ def get_current_pid
450
+
451
+ if File.exist?( f = pid_file )
452
+ File.open( f ) { |io| io.gets.to_i }
453
+ else
454
+ raise "No pid file (#{f}) present. Tomcat not running?"
455
+ end
456
+ end
457
+
458
+ private
459
+ def send_shutdown
460
+
461
+ begin
462
+ sock = TCPSocket::new( "localhost", @config.control_port )
463
+
464
+ begin
465
+ sock.puts @config.shutdown_string
466
+ ensure
467
+ sock.close
468
+ end
469
+
470
+ rescue Errno::ECONNREFUSED
471
+ end
472
+ end
473
+
474
+ private
475
+ def await_port_close( port )
476
+
477
+ 3.times do |i|
478
+ begin
479
+ TCPSocket.new( "localhost", port ).close
480
+ code( "Still got a connection at :#{port}; will retry" )
481
+ sleep( 2 )
482
+ rescue Errno::ECONNREFUSED
483
+ return true
484
+ end
485
+ end
486
+
487
+ return false
488
+ end
489
+
490
+ private
491
+ def await_srv_close
492
+
493
+ # Returns true if and only if all ports are seen closed
494
+ ! [ :control_port, :http_port, :https_port ].find do |port|
495
+ ! await_port_close( @config.send( port ) )
496
+ end
497
+ end
498
+
499
+ private
500
+ def do_stop
501
+
502
+ pid = get_current_pid
503
+
504
+ send_shutdown
505
+
506
+ unless await_srv_close
507
+ code( "Server appears to still be running; sending KILL to #{pid}" )
508
+ Process.kill( "KILL", pid )
509
+ end
510
+
511
+ fu().rm( pid_file )
512
+ end
513
+
514
+ private
515
+ def do_restart
516
+
517
+ do_stop
518
+ do_start
519
+ end
520
+
521
+ public
522
+ def run( run_ctx )
523
+
524
+ cmd = run_ctx[ :argv_remain ].shift or raise "Need a daemon command"
525
+
526
+ if respond_to?( meth = :"do_#{cmd}", true )
527
+ send( meth )
528
+ else
529
+ raise "Unknown daemon command: #{cmd}"
530
+ end
531
+ end
532
+ end
533
+
534
+ class Tailf < RuntimeCommand
535
+
536
+ public
537
+ def run
538
+
539
+ UnixProcessBuilder.new(
540
+ cmd: "tail",
541
+ argv: [ "-f" ] + Dir.glob( "#@runtime/catalina/logs/*" )
542
+ ).
543
+ exec
544
+ end
545
+ end
546
+
547
+ COMMANDS = {
548
+ install: Install,
549
+ run: Run,
550
+ daemon: Daemon,
551
+ tailf: Tailf,
552
+ help: Help
553
+ }
554
+
555
+ def get_command_cls
556
+
557
+ idx = ARGV.find_index { |arg| ! arg.start_with?( "-" ) }
558
+
559
+ cmd_sym = idx ? ARGV.delete_at( idx ).to_sym : :help
560
+
561
+ unless res = COMMANDS[ cmd_sym ]
562
+ res = Help
563
+ STDERR.puts( "Unrecognized command: #{cmd_sym}" )
564
+ end
565
+
566
+ res
567
+ end
568
+
569
+ BitGirderCliApplication.run( get_command_cls )