hq-log-monitor-server 0.0.0 → 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,8 +1,8 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
- require "hq/systools/monitoring/log-monitor-server-script"
3
+ require "hq/log-monitor-server/script"
4
4
 
5
- script = HQ::SysTools::Monitoring::LogMonitorServerScript.new
5
+ script = HQ::LogMonitorServer::Script.new
6
6
  script.args = ARGV
7
7
  script.main
8
8
  exit script.status
@@ -2,7 +2,7 @@ require "capybara/cucumber"
2
2
  require "mongo"
3
3
  require "net/http"
4
4
 
5
- require "hq/systools/monitoring/log-monitor-server-script"
5
+ require "hq/log-monitor-server/script"
6
6
 
7
7
  $token = (?a..?z).to_a.sample(10).join
8
8
 
@@ -19,7 +19,7 @@ Given /^the log monitor server config:$/ do
19
19
  @log_monitor_server_config.flush
20
20
 
21
21
  @log_monitor_server_script =
22
- HQ::SysTools::Monitoring::LogMonitorServerScript.new
22
+ HQ::LogMonitorServer::Script.new
23
23
 
24
24
  @log_monitor_server_script.args = [
25
25
  "--config",
@@ -0,0 +1,687 @@
1
+ require "mongo"
2
+ require "multi_json"
3
+ require "rack"
4
+ require "webrick"
5
+ require "xml"
6
+
7
+ require "hq/tools/escape"
8
+ require "hq/tools/getopt"
9
+
10
+ module HQ
11
+ module LogMonitorServer
12
+ class Script
13
+
14
+ include Tools::Escape
15
+
16
+ attr_accessor :args
17
+ attr_accessor :status
18
+
19
+ def initialize
20
+ @status = 0
21
+ end
22
+
23
+ def main
24
+ setup
25
+ trap "INT" do
26
+ @web_server.shutdown
27
+ end
28
+ run
29
+ end
30
+
31
+ def start
32
+ setup
33
+ Thread.new { run }
34
+ end
35
+
36
+ def stop
37
+ @web_server.shutdown
38
+ end
39
+
40
+ def setup
41
+ process_args
42
+ read_config
43
+ connect_db
44
+ init_server
45
+ end
46
+
47
+ def run
48
+ @web_server.start
49
+ end
50
+
51
+ def process_args
52
+
53
+ @opts, @args =
54
+ Tools::Getopt.process @args, [
55
+
56
+ { :name => :config,
57
+ :required => true },
58
+
59
+ { :name => :quiet,
60
+ :boolean => true },
61
+
62
+ ]
63
+
64
+ @args.empty? \
65
+ or raise "Extra args on command line"
66
+
67
+ end
68
+
69
+ def read_config
70
+
71
+ config_doc =
72
+ XML::Document.file @opts[:config]
73
+
74
+ @config_elem =
75
+ config_doc.root
76
+
77
+ @server_elem =
78
+ @config_elem.find_first("server")
79
+
80
+ @db_elem =
81
+ @config_elem.find_first("db")
82
+
83
+ end
84
+
85
+ def connect_db
86
+
87
+ @mongo =
88
+ Mongo::MongoClient.new \
89
+ @db_elem["host"],
90
+ @db_elem["port"].to_i
91
+
92
+ @db =
93
+ @mongo[@db_elem["name"]]
94
+
95
+ end
96
+
97
+ def init_server
98
+
99
+ @web_config = {
100
+ :Port => @server_elem["port"].to_i,
101
+ :AccessLog => [],
102
+ }
103
+
104
+ if @opts[:quiet]
105
+ @web_config.merge!({
106
+ :Logger => WEBrick::Log::new("/dev/null", 7),
107
+ :DoNotReverseLookup => true,
108
+ })
109
+ end
110
+
111
+ @web_server =
112
+ WEBrick::HTTPServer.new \
113
+ @web_config
114
+
115
+ @web_server.mount "/", Rack::Handler::WEBrick, self
116
+
117
+ end
118
+
119
+ def call env
120
+
121
+ case env["PATH_INFO"]
122
+
123
+ when "/submit-log-event"
124
+ submit_log_event env
125
+
126
+ when "/"
127
+ overview_page env
128
+
129
+ when /^\/service\/([^\/]+)$/
130
+ service_page env, :service => $1
131
+
132
+ when /^\/service\/([^\/]+)\/host\/([^\/]+)$/
133
+ service_host_page env, :service => $1, :host => $2
134
+
135
+ when /^\/event\/([^\/]+)$/
136
+ event_page env, :event_id => $1
137
+
138
+ when "/favicon.ico"
139
+ [ 404, {}, [] ]
140
+
141
+ else
142
+ raise "Not found: #{env["PATH_INFO"]}"
143
+
144
+ end
145
+
146
+ end
147
+
148
+ def submit_log_event env
149
+
150
+ # decode it
151
+
152
+ event = MultiJson.load env["rack.input"].read
153
+
154
+ # add a timestamp
155
+
156
+ event["timestamp"] = Time.now
157
+
158
+ # insert it
159
+
160
+ @db["events"].insert event
161
+
162
+ # update summary
163
+
164
+ summary =
165
+ @db["summaries"].find({
166
+ "_id" => event["source"],
167
+ }).first
168
+
169
+ summary ||= {
170
+ "_id" => event["source"],
171
+ "combined" => { "new" => 0, "total" => 0 },
172
+ "types" => {},
173
+ }
174
+
175
+ summary["types"][event["type"]] ||=
176
+ { "new" => 0, "total" => 0 }
177
+
178
+ summary["types"][event["type"]]["new"] += 1
179
+ summary["types"][event["type"]]["total"] += 1
180
+
181
+ summary["combined"]["new"] += 1
182
+ summary["combined"]["total"] += 1
183
+
184
+ @db["summaries"].save summary
185
+
186
+ # respond successfully
187
+
188
+ return 202, {}, []
189
+
190
+ end
191
+
192
+ def overview_page env
193
+
194
+ summaries_by_service = {}
195
+
196
+ @db["summaries"].find.each do
197
+ |summary|
198
+
199
+ service =
200
+ summary["_id"]["service"]
201
+
202
+ summary_by_service =
203
+ summaries_by_service[service] ||= {
204
+ "service" => service,
205
+ "combined" => { "new" => 0, "total" => 0 },
206
+ "types" => {},
207
+ }
208
+
209
+ summary_by_service["combined"]["new"] +=
210
+ summary["combined"]["new"]
211
+
212
+ summary_by_service["combined"]["total"] +=
213
+ summary["combined"]["total"]
214
+
215
+ summary["types"].each do
216
+ |type, type_summary|
217
+
218
+ type_summary_by_service =
219
+ summary_by_service["types"][type] ||= {
220
+ "new" => 0,
221
+ "total" => 0,
222
+ }
223
+
224
+ type_summary_by_service["new"] +=
225
+ type_summary["new"]
226
+
227
+ type_summary_by_service["total"] +=
228
+ type_summary["total"]
229
+
230
+ end
231
+
232
+ end
233
+
234
+ headers = {}
235
+ html = []
236
+
237
+ headers["content-type"] = "text/html; charset=utf-8"
238
+
239
+ html << "<! DOCTYPE html>\n"
240
+ html << "<html>\n"
241
+ html << "<head>\n"
242
+
243
+ title =
244
+ "Overview \u2014 Log monitor"
245
+
246
+ html << "<title>%s</title>\n" % [
247
+ esc_ht(title),
248
+ ]
249
+
250
+ html << "</head>\n"
251
+ html << "<body>\n"
252
+
253
+ html << "<h1>%s</h1>\n" % [
254
+ esc_ht(title),
255
+ ]
256
+
257
+ if summaries_by_service.empty?
258
+ html << "<p>No events have been logged</p>\n"
259
+ else
260
+
261
+ html << "<table id=\"summaries\">\n"
262
+ html << "<thead>\n"
263
+
264
+ html << "<tr>\n"
265
+ html << "<th>Service</th>\n"
266
+ html << "<th>Alerts</th>\n"
267
+ html << "<th>Details</th>\n"
268
+ html << "<th>View</th>\n"
269
+ html << "</tr>\n"
270
+
271
+ html << "</thead>\n"
272
+ html << "<tbody>\n"
273
+
274
+ summaries_by_service.each do
275
+ |service, summary|
276
+
277
+ html << "<tr class=\"summary\">\n"
278
+
279
+ html << "<td class=\"service\">%s</td>\n" % [
280
+ esc_ht(summary["service"]),
281
+ ]
282
+
283
+ html << "<td class=\"alerts\">%s</td>\n" % [
284
+ esc_ht(summary["combined"]["new"].to_s),
285
+ ]
286
+
287
+ html << "<td class=\"detail\">%s</td>\n" % [
288
+ esc_ht(
289
+ summary["types"].map {
290
+ |type, counts|
291
+ "%s %s" % [ counts["new"], type ]
292
+ }.join ", "
293
+ ),
294
+ ]
295
+
296
+ html << "<td class=\"view\">%s</td>\n" % [
297
+ "<a href=\"%s\">view</a>" % [
298
+ "/service/%s" % [
299
+ esc_ue(summary["service"]),
300
+ ],
301
+ ],
302
+ ]
303
+
304
+ html << "</tr>\n"
305
+
306
+ end
307
+
308
+ html << "</tbody>\n"
309
+ html << "</table>\n"
310
+
311
+ end
312
+
313
+ html << "</body>\n"
314
+ html << "</html>\n"
315
+
316
+ return 200, headers, html
317
+
318
+ end
319
+
320
+ def service_page env, context
321
+
322
+ summaries =
323
+ @db["summaries"]
324
+ .find({
325
+ "_id.service" => context[:service]
326
+ })
327
+ .to_a
328
+
329
+ title =
330
+ "%s - Log monitor" % [
331
+ context[:service],
332
+ ]
333
+
334
+ headers = {}
335
+ html = []
336
+
337
+ headers["content-type"] = "text/html; charset=utf-8"
338
+
339
+ html << "<!DOCTYPE html>\n"
340
+ html << "<html>\n"
341
+ html << "<head>\n"
342
+
343
+ html << "<title>%s</title>\n" % [
344
+ esc_ht(title),
345
+ ]
346
+
347
+ html << "</head>\n"
348
+ html << "<body>\n"
349
+
350
+ html << "<h1>%s</h1>\n" % [
351
+ esc_ht(title),
352
+ ]
353
+
354
+ if summaries.empty?
355
+ html << "<p>No events have been logged for this service</p>\n"
356
+ else
357
+
358
+ html << "<table id=\"summaries\">\n"
359
+ html << "<thead>\n"
360
+
361
+ html << "<tr>\n"
362
+ html << "<th>Host</th>\n"
363
+ html << "<th>Class</th>\n"
364
+ html << "<th>Alerts</th>\n"
365
+ html << "<th>Details</th>\n"
366
+ html << "<th>View</th>\n"
367
+ html << "</tr>\n"
368
+
369
+ html << "</thead>\n"
370
+ html << "<tbody>\n"
371
+
372
+ summaries.each do
373
+ |summary|
374
+
375
+ html << "<tr class=\"summary\">\n"
376
+
377
+ html << "<td class=\"host\">%s</td>\n" % [
378
+ esc_ht(summary["_id"]["host"]),
379
+ ]
380
+
381
+ html << "<td class=\"service\">%s</td>\n" % [
382
+ esc_ht(summary["_id"]["class"]),
383
+ ]
384
+
385
+ html << "<td class=\"alerts\">%s</td>\n" % [
386
+ esc_ht(summary["combined"]["new"].to_s),
387
+ ]
388
+
389
+ html << "<td class=\"detail\">%s</td>\n" % [
390
+ esc_ht(
391
+ summary["types"].map {
392
+ |type, counts|
393
+ "%s %s" % [ counts["new"], type ]
394
+ }.join ", "
395
+ ),
396
+ ]
397
+
398
+ html << "<td class=\"view\">%s</td>\n" % [
399
+ "<a href=\"%s\">view</a>" % [
400
+ "/service/%s/host/%s" % [
401
+ esc_ue(summary["_id"]["service"]),
402
+ esc_ue(summary["_id"]["host"]),
403
+ ],
404
+ ],
405
+ ]
406
+
407
+ html << "</tr>\n"
408
+
409
+ end
410
+
411
+ html << "</tbody>\n"
412
+ html << "</table>\n"
413
+
414
+ end
415
+
416
+ html << "</body>\n"
417
+ html << "</html>\n"
418
+
419
+ return 200, headers, html
420
+
421
+ end
422
+
423
+ def service_host_page env, context
424
+
425
+ events =
426
+ @db["events"]
427
+ .find({
428
+ "source.service" => context[:service],
429
+ "source.host" => context[:host],
430
+ })
431
+ .to_a
432
+
433
+ title =
434
+ "%s %s \u2014 Log monitor" % [
435
+ context[:host],
436
+ context[:service],
437
+ ]
438
+
439
+ headers = {}
440
+ html = []
441
+
442
+ headers["content-type"] = "text/html; charset=utf-8"
443
+
444
+ html << "<!DOCTYPE html>\n"
445
+ html << "<html>\n"
446
+ html << "<head>\n"
447
+
448
+ html << "<title>%s</title>\n" % [
449
+ esc_ht(title),
450
+ ]
451
+
452
+ html << "</head>\n"
453
+ html << "<body>\n"
454
+
455
+ html << "<h1>%s</h1>\n" % [
456
+ esc_ht(title),
457
+ ]
458
+
459
+ if events.empty?
460
+ html << "<p>No events have been logged for this service on this " +
461
+ "host</p>\n"
462
+ else
463
+
464
+ html << "<table id=\"events\">\n"
465
+ html << "<thead>\n"
466
+
467
+ html << "<tr>\n"
468
+ html << "<th>Timestamp</th>\n"
469
+ html << "<th>File</th>\n"
470
+ html << "<th>Line</th>\n"
471
+ html << "<th>Type</th>\n"
472
+ html << "<th>View</th>\n"
473
+ html << "</tr>\n"
474
+
475
+ html << "</thead>\n"
476
+ html << "<tbody>\n"
477
+
478
+ events.each do
479
+ |event|
480
+
481
+ html << "<tr class=\"event\">\n"
482
+
483
+ html << "<td class=\"timestamp\">%s</td>\n" % [
484
+ esc_ht(event["timestamp"].to_s),
485
+ ]
486
+
487
+ html << "<td class=\"file\">%s</td>\n" % [
488
+ esc_ht(event["location"]["file"]),
489
+ ]
490
+
491
+ html << "<td class=\"line\">%s</td>\n" % [
492
+ esc_ht(event["location"]["line"].to_s),
493
+ ]
494
+
495
+ html << "<td class=\"type\">%s</td>\n" % [
496
+ esc_ht(event["type"]),
497
+ ]
498
+
499
+ html << "<td class=\"view\">%s</td>\n" % [
500
+ "<a href=\"%s\">view</a>" % [
501
+ "/event/%s" % [
502
+ esc_ue(event["_id"].to_s),
503
+ ],
504
+ ],
505
+ ]
506
+
507
+ html << "</tr>\n"
508
+
509
+ end
510
+
511
+ html << "</tbody>\n"
512
+ html << "</table>\n"
513
+
514
+ end
515
+
516
+ html << "</body>\n"
517
+ html << "</html>\n"
518
+
519
+ return 200, headers, html
520
+
521
+ end
522
+
523
+ def event_page env, context
524
+
525
+ event =
526
+ @db["events"]
527
+ .find_one({
528
+ "_id" => BSON::ObjectId.from_string(context[:event_id]),
529
+ })
530
+
531
+ title =
532
+ "Event %s \u2014 Log monitor" % [
533
+ context[:event_id],
534
+ ]
535
+
536
+ headers = {}
537
+ html = []
538
+
539
+ headers["content-type"] = "text/html; charset=utf-8"
540
+
541
+ html << "<!DOCTYPE html>\n"
542
+ html << "<html>\n"
543
+ html << "<head>\n"
544
+
545
+ html << "<title>%s</title>\n" % [
546
+ esc_ht(title),
547
+ ]
548
+
549
+ html << "</head>\n"
550
+ html << "<body>\n"
551
+
552
+ html << "<h1>%s</h1>\n" % [
553
+ esc_ht(title),
554
+ ]
555
+
556
+ unless event
557
+
558
+ html << "<p>Event id not recognised</p>\n"
559
+
560
+ else
561
+
562
+ html << "<table id=\"event\">\n"
563
+ html << "<tbody>\n"
564
+
565
+ html << "<tr>\n"
566
+ html << "<th>ID</th>\n"
567
+ html << "<td>%s</td>\n" % [
568
+ esc_ht(event["_id"].to_s),
569
+ ]
570
+ html << "</tr>\n"
571
+
572
+ html << "<tr>\n"
573
+ html << "<th>Timestamp</th>\n"
574
+ html << "<td>%s</td>\n" % [
575
+ esc_ht(event["timestamp"].to_s),
576
+ ]
577
+ html << "</tr>\n"
578
+
579
+ html << "<tr>\n"
580
+ html << "<th>Service</th>\n"
581
+ html << "<td>%s</td>\n" % [
582
+ esc_ht(event["source"]["service"]),
583
+ ]
584
+ html << "</tr>\n"
585
+
586
+ html << "<tr>\n"
587
+ html << "<th>Host</th>\n"
588
+ html << "<td>%s</td>\n" % [
589
+ esc_ht(event["source"]["host"]),
590
+ ]
591
+ html << "</tr>\n"
592
+
593
+ html << "<tr>\n"
594
+ html << "<th>Class</th>\n"
595
+ html << "<td>%s</td>\n" % [
596
+ esc_ht(event["source"]["class"]),
597
+ ]
598
+ html << "</tr>\n"
599
+
600
+ html << "<tr>\n"
601
+ html << "<th>Filename</th>\n"
602
+ html << "<td>%s</td>\n" % [
603
+ esc_ht(event["location"]["file"]),
604
+ ]
605
+ html << "</tr>\n"
606
+
607
+ html << "<tr>\n"
608
+ html << "<th>Line number</th>\n"
609
+ html << "<td>%s</td>\n" % [
610
+ esc_ht((event["location"]["line"] + 1).to_s),
611
+ ]
612
+ html << "</tr>\n"
613
+
614
+ unless event["lines"]["before"].empty?
615
+
616
+ html << "<tr>\n"
617
+ html << "<th>Before</th>\n"
618
+ html << "<td>%s</td>\n" % [
619
+ event["lines"]["before"]
620
+ .map { |line| esc_ht(line) }
621
+ .join("<br>")
622
+ ]
623
+
624
+ end
625
+
626
+ html << "<tr>\n"
627
+ html << "<th>Matching</th>\n"
628
+ html << "<td>%s</td>\n" % [
629
+ esc_ht(event["lines"]["matching"]),
630
+ ]
631
+
632
+ unless event["lines"]["after"].empty?
633
+
634
+ html << "<tr>\n"
635
+ html << "<th>Before</th>\n"
636
+ html << "<td>%s</td>\n" % [
637
+ event["lines"]["after"]
638
+ .map { |line| esc_ht(line) }
639
+ .join("<br>")
640
+ ]
641
+
642
+ end
643
+
644
+ html << "</tbody>\n"
645
+ html << "</table>\n"
646
+
647
+ end
648
+
649
+ html << "</body>\n"
650
+ html << "</html>\n"
651
+
652
+ return 200, headers, html
653
+
654
+ end
655
+
656
+ def sf format, *args
657
+
658
+ ret = []
659
+
660
+ format.scan(/%.|%%|[^%]+|%/).each do
661
+ |match|
662
+
663
+ case match
664
+
665
+ when ?%
666
+ raise "Error"
667
+
668
+ when "%%"
669
+ ret << ?%
670
+
671
+ when /^%(.)$/
672
+ ret << send("format_#{$1}", args.shift)
673
+
674
+ else
675
+ ret << match
676
+
677
+ end
678
+
679
+ end
680
+
681
+ return ret.join
682
+
683
+ end
684
+
685
+ end
686
+ end
687
+ end