bind_log_analyzer 0.1.0 → 0.1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (28) hide show
  1. data/.gitignore +1 -0
  2. data/README.md +10 -0
  3. data/doc/BindLogAnalyzer.html +166 -0
  4. data/doc/BindLogAnalyzer/Base.html +1016 -0
  5. data/doc/BindLogAnalyzer/Connector.html +558 -0
  6. data/doc/BindLogAnalyzer/DatabaseConfsNotValid.html +127 -0
  7. data/doc/Log.html +127 -0
  8. data/doc/_index.html +156 -0
  9. data/doc/class_list.html +47 -0
  10. data/doc/css/common.css +1 -0
  11. data/doc/css/full_list.css +55 -0
  12. data/doc/css/style.css +322 -0
  13. data/doc/file.README.html +215 -0
  14. data/doc/file_list.html +49 -0
  15. data/doc/frames.html +13 -0
  16. data/doc/index.html +215 -0
  17. data/doc/js/app.js +205 -0
  18. data/doc/js/full_list.js +173 -0
  19. data/doc/js/jquery.js +16 -0
  20. data/doc/method_list.html +150 -0
  21. data/doc/top-level-namespace.html +107 -0
  22. data/lib/bind_log_analyzer.rb +2 -0
  23. data/lib/bind_log_analyzer/base.rb +25 -1
  24. data/lib/bind_log_analyzer/connector.rb +12 -0
  25. data/lib/bind_log_analyzer/exceptions.rb +1 -0
  26. data/lib/bind_log_analyzer/version.rb +2 -1
  27. data/lib/models/log.rb +1 -0
  28. metadata +27 -7
data/doc/file_list.html ADDED
@@ -0,0 +1,49 @@
1
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
2
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
3
+ <html>
4
+ <head>
5
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
6
+
7
+ <link rel="stylesheet" href="css/full_list.css" type="text/css" media="screen" charset="utf-8" />
8
+
9
+ <link rel="stylesheet" href="css/common.css" type="text/css" media="screen" charset="utf-8" />
10
+
11
+
12
+
13
+ <script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
14
+
15
+ <script type="text/javascript" charset="utf-8" src="js/full_list.js"></script>
16
+
17
+
18
+ <base id="base_target" target="_parent" />
19
+ </head>
20
+ <body>
21
+ <script type="text/javascript" charset="utf-8">
22
+ if (window.top.frames.main) {
23
+ document.getElementById('base_target').target = 'main';
24
+ document.body.className = 'frames';
25
+ }
26
+ </script>
27
+ <div id="content">
28
+ <h1 id="full_list_header">File List</h1>
29
+ <div id="nav">
30
+
31
+ <a target="_self" href="class_list.html">Classes</a>
32
+
33
+ <a target="_self" href="method_list.html">Methods</a>
34
+
35
+ <a target="_self" href="file_list.html">Files</a>
36
+
37
+ </div>
38
+ <div id="search">Search: <input type="text" /></div>
39
+
40
+ <ul id="full_list" class="files">
41
+
42
+
43
+ <li class="r1"><a href="index.html" title="README">README</a></li>
44
+
45
+
46
+ </ul>
47
+ </div>
48
+ </body>
49
+ </html>
data/doc/frames.html ADDED
@@ -0,0 +1,13 @@
1
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Frameset//EN"
2
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-frameset.dtd">
3
+
4
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
5
+ <head>
6
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
7
+ <title>Documentation by YARD 0.7.5</title>
8
+ </head>
9
+ <frameset cols="20%,*">
10
+ <frame name="list" src="class_list.html" />
11
+ <frame name="main" src="index.html" />
12
+ </frameset>
13
+ </html>
data/doc/index.html ADDED
@@ -0,0 +1,215 @@
1
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
2
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
3
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
4
+ <head>
5
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
6
+ <title>
7
+ File: README
8
+
9
+ &mdash; Documentation by YARD 0.7.5
10
+
11
+ </title>
12
+
13
+ <link rel="stylesheet" href="css/style.css" type="text/css" media="screen" charset="utf-8" />
14
+
15
+ <link rel="stylesheet" href="css/common.css" type="text/css" media="screen" charset="utf-8" />
16
+
17
+ <script type="text/javascript" charset="utf-8">
18
+ relpath = '';
19
+ if (relpath != '') relpath += '/';
20
+ </script>
21
+
22
+ <script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
23
+
24
+ <script type="text/javascript" charset="utf-8" src="js/app.js"></script>
25
+
26
+
27
+ </head>
28
+ <body>
29
+ <script type="text/javascript" charset="utf-8">
30
+ if (window.top.frames.main) document.body.className = 'frames';
31
+ </script>
32
+
33
+ <div id="header">
34
+ <div id="menu">
35
+
36
+ <a href="_index.html" title="Index">Index</a> &raquo;
37
+ <span class="title">File: README</span>
38
+
39
+
40
+ <div class="noframes"><span class="title">(</span><a href="." target="_top">no frames</a><span class="title">)</span></div>
41
+ </div>
42
+
43
+ <div id="search">
44
+
45
+ <a id="class_list_link" href="#">Class List</a>
46
+
47
+ <a id="method_list_link" href="#">Method List</a>
48
+
49
+ <a id="file_list_link" href="#">File List</a>
50
+
51
+ </div>
52
+ <div class="clear"></div>
53
+ </div>
54
+
55
+ <iframe id="search_frame"></iframe>
56
+
57
+ <div id="content"><div id='filecontents'><h1>Bind Log Analyzer</h1>
58
+
59
+ <p>Simple analysis and SQL storage for Bind DNS server&#39;s logs</p>
60
+
61
+ <h2>Requirements</h2>
62
+
63
+ <p>This gem was tested with:</p>
64
+
65
+ <ul>
66
+ <li>ruby-1.9.3-p125</li>
67
+ <li>rubygem (1.8.15)</li>
68
+ <li>bundler (1.0.21)</li>
69
+ <li>activerecord (3.2.2)</li>
70
+ </ul>
71
+
72
+ <h2>Installation</h2>
73
+
74
+ <p>Just install the gem:</p>
75
+
76
+ <pre class="code ruby"><code><span class='id identifier rubyid_gem'>gem</span> <span class='id identifier rubyid_install'>install</span> <span class='id identifier rubyid_bind_log_analyzer'>bind_log_analyzer</span>
77
+ </code></pre>
78
+
79
+ <p>The gem requires <strong>active_record</strong> but you probably need to install the right adapter. As example, if you&#39;ll use MySQL, install the <strong>mysql2</strong> gem.</p>
80
+
81
+ <h2>Configuration</h2>
82
+
83
+ <h3>Bind</h3>
84
+
85
+ <p>To configure <strong>Bind</strong> add these lines to <em>/etc/bind/named.conf.options</em> (or whatever your s.o. and bind installation require)</p>
86
+
87
+ <pre class="code ruby"><code>logging{
88
+ channel &quot;querylog&quot; {
89
+ file &quot;/var/log/bind/query.log&quot;;
90
+ print-time yes;
91
+ };
92
+
93
+ category queries { querylog; };
94
+ };
95
+ </code></pre>
96
+
97
+ <p>Restart bind and make sure than the <em>query.log</em> file contains lines as this:</p>
98
+
99
+ <pre class="code ruby"><code>28-Mar-2012 16:48:19.694 client 192.168.10.38#58767: query: www.github.com IN A + (192.168.10.1)
100
+ </code></pre>
101
+
102
+ <p>or the regexp will fail :(</p>
103
+
104
+ <h3>Database</h3>
105
+
106
+ <p>To store the logs you can use every database supported by ActiveRecord. Just create a database and a user with the right privileges. You can provide the -s flag to <em>BindLogAnalyzer</em> to make it create the table. Otherwise create it by yourself.
107
+ This is the MySQL CREATE TABLE syntax:</p>
108
+
109
+ <pre class="code ruby"><code>CREATE TABLE `logs` (
110
+ `id` int(11) NOT NULL AUTO_INCREMENT,
111
+ `date` datetime NOT NULL,
112
+ `client` varchar(255) NOT NULL,
113
+ `query` varchar(255) NOT NULL,
114
+ `q_type` varchar(255) NOT NULL,
115
+ `server` varchar(255) NOT NULL,
116
+ PRIMARY KEY (`id`)
117
+ ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=latin1;
118
+ </code></pre>
119
+
120
+ <h2>Usage</h2>
121
+
122
+ <p>Use the provided --help to get various options available. This is the default help:</p>
123
+
124
+ <pre class="code ruby"><code>-h, --help Display this screen
125
+ -v, --verbose LEVEL Enables verbose output. Use level 1 for WARN, 2 for INFO and 3 for DEBUG
126
+ -s, --setup Creates the needed tables in the database.
127
+ -f, --file FILE Indicates the log file to parse. It's mandatory.
128
+ -c, --config CONFIG A yaml file containing the database configurations under the &quot;database&quot; entry
129
+ -a, --adapter ADAPTER The database name to save the logs
130
+ -d, --database DATABASE The database name to save the logs
131
+ -H, --host HOST The address (IP, hostname or path) of the database
132
+ -P, --port PORT The port of the database
133
+ -u, --user USER The username to be used to connect to the database
134
+ -p, --password PASSWORD The password of the user
135
+ </code></pre>
136
+
137
+ <p>There&#39;s only one mandatory argument which is <strong>--file FILE</strong>. With this flag you pass the Bind log file to analyze to <em>BindLogAnalyzer</em>.</p>
138
+
139
+ <p>The first time you launch <em>BindLogAnalyzer</em> you can use the <strong>-s|--setup</strong> flag to make it create the table (using ActiveRecord::Migration).
140
+ The database credentials can be provided using the needed flags or creating a YAML file containing all the informations under the <strong>database</strong> key. This is an example:</p>
141
+
142
+ <pre class="code ruby"><code>database:
143
+ adapter: mysql2
144
+ database: bindloganalyzer
145
+ host: localhost
146
+ port: 3306
147
+ username: root
148
+ password:
149
+ </code></pre>
150
+
151
+ <h2>Automatization</h2>
152
+
153
+ <p>A good way to use this script is to let it be launched by <strong>logrotate</strong> so create the <em>/etc/logrotate.d/bind</em> file with this content:</p>
154
+
155
+ <pre class="code ruby"><code>/var/log/named/query.log {
156
+ weekly
157
+ missingok
158
+ rotate 8
159
+ compress
160
+ delaycompress
161
+ notifempty
162
+ create 644 bind bind
163
+ postrotate
164
+ if [ -e /var/log/named/query.log.1 ]; then
165
+ exec su - YOUR_USER -c '/usr/local/bin/update_bind_log_analyzer.sh /var/log/named/query.log.1'
166
+ fi
167
+ endscript
168
+ }
169
+ </code></pre>
170
+
171
+ <p>The script <strong>/usr/local/bin/update<em>bind</em>log_analyzer.sh</strong> can be wherever you prefer. Its typical content if you use RVM and a dedicated gemset for <em>BindLogAnalyzer</em>, can be:</p>
172
+
173
+ <pre class="code ruby"><code>#!/bin/bash
174
+
175
+ # *************************** #
176
+ # EDIT THESE VARS #
177
+ # *************************** #
178
+ BLA_RVM_GEMSET=&quot;1.9.3-p125@bind_log_analyzer&quot;
179
+ BLA_USER=&quot;my_username&quot;
180
+ BLA_DB_FILE=&quot;/etc/bind_log_analyzer/database.yml&quot;
181
+
182
+ # *************************** #
183
+ # DO NOT EDIT BELOW THIS LINE #
184
+ # *************************** #
185
+ . /home/$BLA_USER/.rvm/scripts/rvm &amp;&amp; source &quot;/home/$BLA_USER/.rvm/scripts/rvm&quot;
186
+ rvm use $BLA_RVM_GEMSET
187
+ bind_log_analyzer --config $BLA_DB_FILE --file $1
188
+ </code></pre>
189
+
190
+ <h2>Performance</h2>
191
+
192
+ <p>On a 1.6 Ghz Intel Core i5 with SSD SATA2 disk, using Ruby-1.9.3-p125 and MySQL 5.5.15, this is the performance:</p>
193
+
194
+ <pre class="code ruby"><code>~$ time bind_log_analyzer -f query.log -c database.yml
195
+ Analyzed 319758 lines and correctly stored 319758 logs
196
+ bind_log_analyzer -f query.log -c database.yml 322,44s user 22,90s system 76% cpu 7:33,17 total
197
+ </code></pre>
198
+
199
+ <p>which is equivalent to ±706 query/sec.</p>
200
+
201
+ <h2>To do</h2>
202
+
203
+ <ul>
204
+ <li>Add a web interface to show the queries (with awesome graphs, obviously :)</li>
205
+ </ul>
206
+ </div></div>
207
+
208
+ <div id="footer">
209
+ Generated on Tue Apr 3 17:01:09 2012 by
210
+ <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
211
+ 0.7.5 (ruby-1.9.3).
212
+ </div>
213
+
214
+ </body>
215
+ </html>
data/doc/js/app.js ADDED
@@ -0,0 +1,205 @@
1
+ function createSourceLinks() {
2
+ $('.method_details_list .source_code').
3
+ before("<span class='showSource'>[<a href='#' class='toggleSource'>View source</a>]</span>");
4
+ $('.toggleSource').toggle(function() {
5
+ $(this).parent().nextAll('.source_code').slideDown(100);
6
+ $(this).text("Hide source");
7
+ },
8
+ function() {
9
+ $(this).parent().nextAll('.source_code').slideUp(100);
10
+ $(this).text("View source");
11
+ });
12
+ }
13
+
14
+ function createDefineLinks() {
15
+ var tHeight = 0;
16
+ $('.defines').after(" <a href='#' class='toggleDefines'>more...</a>");
17
+ $('.toggleDefines').toggle(function() {
18
+ tHeight = $(this).parent().prev().height();
19
+ $(this).prev().show();
20
+ $(this).parent().prev().height($(this).parent().height());
21
+ $(this).text("(less)");
22
+ },
23
+ function() {
24
+ $(this).prev().hide();
25
+ $(this).parent().prev().height(tHeight);
26
+ $(this).text("more...");
27
+ });
28
+ }
29
+
30
+ function createFullTreeLinks() {
31
+ var tHeight = 0;
32
+ $('.inheritanceTree').toggle(function() {
33
+ tHeight = $(this).parent().prev().height();
34
+ $(this).parent().toggleClass('showAll');
35
+ $(this).text("(hide)");
36
+ $(this).parent().prev().height($(this).parent().height());
37
+ },
38
+ function() {
39
+ $(this).parent().toggleClass('showAll');
40
+ $(this).parent().prev().height(tHeight);
41
+ $(this).text("show all");
42
+ });
43
+ }
44
+
45
+ function fixBoxInfoHeights() {
46
+ $('dl.box dd.r1, dl.box dd.r2').each(function() {
47
+ $(this).prev().height($(this).height());
48
+ });
49
+ }
50
+
51
+ function searchFrameLinks() {
52
+ $('#method_list_link').click(function() {
53
+ toggleSearchFrame(this, relpath + 'method_list.html');
54
+ });
55
+
56
+ $('#class_list_link').click(function() {
57
+ toggleSearchFrame(this, relpath + 'class_list.html');
58
+ });
59
+
60
+ $('#file_list_link').click(function() {
61
+ toggleSearchFrame(this, relpath + 'file_list.html');
62
+ });
63
+ }
64
+
65
+ function toggleSearchFrame(id, link) {
66
+ var frame = $('#search_frame');
67
+ $('#search a').removeClass('active').addClass('inactive');
68
+ if (frame.attr('src') == link && frame.css('display') != "none") {
69
+ frame.slideUp(100);
70
+ $('#search a').removeClass('active inactive');
71
+ }
72
+ else {
73
+ $(id).addClass('active').removeClass('inactive');
74
+ frame.attr('src', link).slideDown(100);
75
+ }
76
+ }
77
+
78
+ function linkSummaries() {
79
+ $('.summary_signature').click(function() {
80
+ document.location = $(this).find('a').attr('href');
81
+ });
82
+ }
83
+
84
+ function framesInit() {
85
+ if (window.top.frames.main) {
86
+ document.body.className = 'frames';
87
+ $('#menu .noframes a').attr('href', document.location);
88
+ $('html head title', window.parent.document).text($('html head title').text());
89
+ }
90
+ }
91
+
92
+ function keyboardShortcuts() {
93
+ if (window.top.frames.main) return;
94
+ $(document).keypress(function(evt) {
95
+ if (evt.altKey || evt.ctrlKey || evt.metaKey || evt.shiftKey) return;
96
+ if (typeof evt.target !== "undefined" &&
97
+ (evt.target.nodeName == "INPUT" ||
98
+ evt.target.nodeName == "TEXTAREA")) return;
99
+ switch (evt.charCode) {
100
+ case 67: case 99: $('#class_list_link').click(); break; // 'c'
101
+ case 77: case 109: $('#method_list_link').click(); break; // 'm'
102
+ case 70: case 102: $('#file_list_link').click(); break; // 'f'
103
+ default: break;
104
+ }
105
+ });
106
+ }
107
+
108
+ function summaryToggle() {
109
+ $('.summary_toggle').click(function() {
110
+ localStorage.summaryCollapsed = $(this).text();
111
+ $(this).text($(this).text() == "collapse" ? "expand" : "collapse");
112
+ var next = $(this).parent().parent().nextAll('ul.summary').first();
113
+ if (next.hasClass('compact')) {
114
+ next.toggle();
115
+ next.nextAll('ul.summary').first().toggle();
116
+ }
117
+ else if (next.hasClass('summary')) {
118
+ var list = $('<ul class="summary compact" />');
119
+ list.html(next.html());
120
+ list.find('.summary_desc, .note').remove();
121
+ list.find('a').each(function() {
122
+ $(this).html($(this).find('strong').html());
123
+ $(this).parent().html($(this)[0].outerHTML);
124
+ });
125
+ next.before(list);
126
+ next.toggle();
127
+ }
128
+ return false;
129
+ });
130
+ if (localStorage) {
131
+ if (localStorage.summaryCollapsed == "collapse") $('.summary_toggle').click();
132
+ else localStorage.summaryCollapsed = "expand";
133
+ }
134
+ }
135
+
136
+ function fixOutsideWorldLinks() {
137
+ $('a').each(function() {
138
+ if (window.location.host != this.host) this.target = '_parent';
139
+ });
140
+ }
141
+
142
+ function generateTOC() {
143
+ if ($('#filecontents').length === 0) return;
144
+ var _toc = $('<ol class="top"></ol>');
145
+ var show = false;
146
+ var toc = _toc;
147
+ var counter = 0;
148
+ var tags = ['h2', 'h3', 'h4', 'h5', 'h6'];
149
+ var i;
150
+ if ($('#filecontents h1').length > 1) tags.unshift('h1');
151
+ for (i = 0; i < tags.length; i++) { tags[i] = '#filecontents ' + tags[i]; }
152
+ var lastTag = parseInt(tags[0][1], 10);
153
+ $(tags.join(', ')).each(function() {
154
+ if (this.id == "filecontents") return;
155
+ show = true;
156
+ var thisTag = parseInt(this.tagName[1], 10);
157
+ if (this.id.length === 0) {
158
+ var proposedId = $(this).text().replace(/[^a-z0-9-]/ig, '_');
159
+ if ($('#' + proposedId).length > 0) { proposedId += counter; counter++; }
160
+ this.id = proposedId;
161
+ }
162
+ if (thisTag > lastTag) {
163
+ for (i = 0; i < thisTag - lastTag; i++) {
164
+ var tmp = $('<ol/>'); toc.append(tmp); toc = tmp;
165
+ }
166
+ }
167
+ if (thisTag < lastTag) {
168
+ for (i = 0; i < lastTag - thisTag; i++) toc = toc.parent();
169
+ }
170
+ toc.append('<li><a href="#' + this.id + '">' + $(this).text() + '</a></li>');
171
+ lastTag = thisTag;
172
+ });
173
+ if (!show) return;
174
+ html = '<div id="toc"><p class="title"><a class="hide_toc" href="#"><strong>Table of Contents</strong></a> <small>(<a href="#" class="float_toc">left</a>)</small></p></div>';
175
+ $('#content').prepend(html);
176
+ $('#toc').append(_toc);
177
+ $('#toc .hide_toc').toggle(function() {
178
+ $('#toc .top').slideUp('fast');
179
+ $('#toc').toggleClass('hidden');
180
+ $('#toc .title small').toggle();
181
+ }, function() {
182
+ $('#toc .top').slideDown('fast');
183
+ $('#toc').toggleClass('hidden');
184
+ $('#toc .title small').toggle();
185
+ });
186
+ $('#toc .float_toc').toggle(function() {
187
+ $(this).text('float');
188
+ $('#toc').toggleClass('nofloat');
189
+ }, function() {
190
+ $(this).text('left');
191
+ $('#toc').toggleClass('nofloat');
192
+ });
193
+ }
194
+
195
+ $(framesInit);
196
+ $(createSourceLinks);
197
+ $(createDefineLinks);
198
+ $(createFullTreeLinks);
199
+ $(fixBoxInfoHeights);
200
+ $(searchFrameLinks);
201
+ $(linkSummaries);
202
+ $(keyboardShortcuts);
203
+ $(summaryToggle);
204
+ $(fixOutsideWorldLinks);
205
+ $(generateTOC);