apache_log_report 1.1.6 → 1.1.10

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 742584032338664834cd5c1b669289980e083e68d172827f6517ce040543aa4d
4
- data.tar.gz: 1bf08e255feb434ded62589d5c70ea5a6eb83dc1f6a69e60a31cb520eea6b48f
3
+ metadata.gz: c046afd634406fbeb8bd2384d8cb918b444c984f5c75c2b77ae61f87237429a4
4
+ data.tar.gz: 9b7aa0b18006c5e903504f6287cbf6470a6bbf99ee0aef500b73168695b624f1
5
5
  SHA512:
6
- metadata.gz: 955dbe90321c68f5694aaa892cae97926172ca47b684a0eee1b9f7644dc26a13f853b3b3d20e35cf06872b29db0c06bc16c4c86e69c740ea86fe8768ae92b2a1
7
- data.tar.gz: e6d9ebc88435442bfb77faa3cb2eccb94a355e9ca7af44548a7b443531c9fae923e41f5d97201044f71414f2d9894162b593cd15170e255030ab49bd9992e3f0
6
+ metadata.gz: ad5802eb53e71b2302cc50447c9e2b05478ab98be23b0d1db02227c81b58c707bc2900f69f5abd712d11f6f43d67312c765b404de913d7ed495bcc4de9fe5ea1
7
+ data.tar.gz: 041c31954502ae3f989e52f3dd6adfc0b465a431ee152797fa11e5cbb9c37d6ade95e1ccd4f7166d2eee9ca9a3d4f8d1f300b472c1d0b9a2cf0be1a0bfb4468f
data/Gemfile.lock CHANGED
@@ -1,9 +1,10 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- apache_log_report (1.1.2)
4
+ apache_log_report (1.1.8)
5
5
  apache_log-parser
6
6
  browser
7
+ ipaddr
7
8
  sqlite3
8
9
  terminal-table
9
10
 
@@ -12,6 +13,7 @@ GEM
12
13
  specs:
13
14
  apache_log-parser (3.1.2)
14
15
  browser (5.3.1)
16
+ ipaddr (1.2.3)
15
17
  rake (12.3.3)
16
18
  sqlite3 (1.4.2)
17
19
  terminal-table (3.0.2)
data/README.org CHANGED
@@ -2,40 +2,6 @@
2
2
  #+AUTHOR: Adolfo Villafiorita
3
3
  #+STARTUP: showall
4
4
 
5
- * Introduction
5
+ This Gem has been superseded by [[https://rubygems.org/gems/log_sense][Log Sense]], which produces better
6
+ outputs and parses also rails logs.
6
7
 
7
- ApacheLogReport is an Apache log analyzer.
8
-
9
- * Installation
10
-
11
- * Usage
12
-
13
- * Change Log
14
-
15
- See the [[file:CHANGELOG.org][CHANGELOG]] file.
16
-
17
- * Todo
18
-
19
- ** TODO Referers should only include the hostname?
20
- ** TODO Graphs in HTML output
21
- ** TODO Countries
22
- ** TODO Light and Dark Theme
23
- ** TODO Embed CSS
24
- ** TODO Declare datatypes in table outputs, so that we can format data
25
-
26
- * Compatibility
27
-
28
-
29
- * Author and Contributors
30
-
31
- [[http://ict4g.net/adolfo][Adolfo Villafiorita]].
32
-
33
- * Known Bugs
34
-
35
- Some known bugs and an unknown number of unknown bugs.
36
-
37
- (See the open issues for the known bugs.)
38
-
39
- * License
40
-
41
- Distributed under the terms of the [[http://opensource.org/licenses/MIT][MIT License]].
data/Rakefile CHANGED
@@ -1,2 +1,15 @@
1
1
  require "bundler/gem_tasks"
2
2
  task :default => :spec
3
+
4
+ require 'rake/testtask'
5
+ Rake::TestTask.new do |t|
6
+ t.libs << 'test'
7
+ end
8
+
9
+ require_relative './lib/apache_log_report/ip_locator.rb'
10
+
11
+ desc "Convert Geolocation DB to sqlite"
12
+ task :dbip_to_sqlite3, [:filename] do |tasks, args|
13
+ filename = args[:filename]
14
+ ApacheLogReport::IpLocator::dbip_to_sqlite filename
15
+ end
@@ -6,8 +6,8 @@ Gem::Specification.new do |spec|
6
6
  spec.authors = ["Adolfo Villafiorita"]
7
7
  spec.email = ["adolfo.villafiorita@ict4g.net"]
8
8
 
9
- spec.summary = %q{Generate analytics from an Apache log file.}
10
- spec.description = %q{Generate requests reports in HTML, OrgMode, and SQLite format from an Apache log file.}
9
+ spec.summary = %q{Generate analytics from an Apache log file. Superseded by https://rubygems.org/gems/log_senseLog Sense.}
10
+ spec.description = %q{Generate requests reports in HTML, OrgMode, and SQLite format from an Apache log file. Superseded by https://rubygems.org/gems/log_senseLog Sense.}
11
11
  spec.homepage = "https://www.ict4g.net/gitea/adolfo/apache_log_report"
12
12
  spec.license = "MIT"
13
13
  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
@@ -29,6 +29,18 @@ Gem::Specification.new do |spec|
29
29
 
30
30
  spec.add_dependency "apache_log-parser"
31
31
  spec.add_dependency "browser"
32
+ spec.add_dependency "ipaddr"
33
+ spec.add_dependency "iso_country_codes"
32
34
  spec.add_dependency "sqlite3"
33
35
  spec.add_dependency "terminal-table"
36
+
37
+ spec.add_development_dependency "minitest"
38
+
39
+ spec.post_install_message = <<-MESSAGE
40
+ ! The 'apache_log_report' gem has been deprecated and has been replaced 'log_sense'.
41
+ ! See: https://rubygems.org/gems/log_sense
42
+ ! And: https://www.ict4g.net/gitea/adolfo/log_sense
43
+ MESSAGE
44
+
45
+
34
46
  end
@@ -31,6 +31,7 @@ if @options[:output_format] == "sqlite"
31
31
  b.finish
32
32
  else
33
33
  @data = ApacheLogReport::DataCruncher.crunch @db, @options
34
+ @data = ApacheLogReport::DataCruncher.geolocate @data
34
35
 
35
36
  @ended_at = Time.now
36
37
  @duration = @ended_at - @started_at
@@ -1,7 +1,5 @@
1
-
2
1
  module ApacheLogReport
3
2
  module DataCruncher
4
-
5
3
  #
6
4
  # take a sqlite3 database and analyze data
7
5
  #
@@ -103,12 +101,13 @@ module ApacheLogReport
103
101
 
104
102
  @browsers = db.execute "SELECT browser, count(browser), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by browser order by count(browser) desc"
105
103
  @platforms = db.execute "SELECT platform, count(platform), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by platform order by count(platform) desc"
106
- @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
107
104
  @referers = db.execute "SELECT referer, count(referer), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by referer order by count(referer) desc limit #{options[:limit]}"
108
105
 
109
- @streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
106
+ @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
110
107
 
108
+ @streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
111
109
  data = {}
110
+
112
111
  self.instance_variables.each do |variable|
113
112
  var_as_symbol = variable.to_s[1..-1].to_sym
114
113
  data[var_as_symbol] = eval(variable.to_s)
@@ -116,6 +115,16 @@ module ApacheLogReport
116
115
  data
117
116
  end
118
117
 
118
+ # add country code to data[:ips]
119
+ def self.geolocate data
120
+ @location_db = IpLocator::load_db
121
+ data[:ips].each do |ip|
122
+ country_code = IpLocator::locate_ip ip[0], @location_db
123
+ ip << country_code
124
+ end
125
+ data
126
+ end
127
+
119
128
  private
120
129
 
121
130
  def self.date_intersect date1, date2, method
@@ -127,8 +136,6 @@ module ApacheLogReport
127
136
  date2
128
137
  end
129
138
  end
130
-
131
-
132
139
  end
133
140
  end
134
141
 
@@ -0,0 +1,42 @@
1
+ require 'csv'
2
+ require 'sqlite3'
3
+ require 'ipaddr'
4
+ require 'iso_country_codes'
5
+
6
+ module ApacheLogReport
7
+ module IpLocator
8
+ DB_FILE = "ip_locations/dbip-country-lite.sqlite3"
9
+
10
+ def self.dbip_to_sqlite db_location
11
+ db = SQLite3::Database.new ":memory:"
12
+ db.execute "CREATE TABLE ip_location (
13
+ from_ip_n INTEGER,
14
+ from_ip TEXT,
15
+ to_ip TEXT,
16
+ country_code TEXT
17
+ )"
18
+
19
+ ins = db.prepare "INSERT INTO ip_location(from_ip_n, from_ip, to_ip, country_code) values (?, ?, ?, ?)"
20
+ CSV.foreach(db_location) do |row|
21
+ ip = IPAddr.new row[0]
22
+ ins.execute(ip.to_i, row[0], row[1], row[2])
23
+ end
24
+
25
+ # persist to file
26
+ ddb = SQLite3::Database.new(DB_FILE)
27
+ b = SQLite3::Backup.new(ddb, 'main', db, 'main')
28
+ b.step(-1) #=> DONE
29
+ b.finish
30
+ end
31
+
32
+ def self.load_db
33
+ SQLite3::Database.new DB_FILE
34
+ end
35
+
36
+ def self.locate_ip ip, db
37
+ ip_n = IPAddr.new(ip).to_i
38
+ res = db.execute "SELECT * FROM ip_location where from_ip_n <= #{ip_n} order by from_ip_n desc limit 1"
39
+ IsoCountryCodes.find(res[0][3]).name
40
+ end
41
+ end
42
+ end
@@ -32,6 +32,8 @@
32
32
  "Platforms",
33
33
  "Referers",
34
34
  "IPs",
35
+ "Geolocation",
36
+ "Streaks",
35
37
  "Command Invocation",
36
38
  "Performance"
37
39
  ].each do |item| %>
@@ -142,7 +144,7 @@
142
144
  { title: "Browsers", header: ["Browser", "Hits", "Visitors", "Size"], rows: data[:browsers] },
143
145
  { title: "Platforms", header: ["Platform", "Hits", "Visitors", "Size"], rows: data[:platforms] },
144
146
  { title: "Referers", header: ["Referers", "Hits", "Visitors", "Size"], rows: data[:referers], col: "col-12" },
145
- { title: "IPs", header: ["IPs", "Hits", "Visitors", "Size"], rows: data[:ips] },
147
+ { title: "IPs", header: ["IPs", "Hits", "Visitors", "Size", "Country"], rows: data[:ips] },
146
148
  { },
147
149
  ]
148
150
  %>
@@ -161,6 +163,31 @@
161
163
  <% end %>
162
164
  </div>
163
165
 
166
+ <article>
167
+ <h2 id="geolocation">Geolocation</h2>
168
+ <table class="table">
169
+ <thead>
170
+ <tr>
171
+ <th>Country Code</th>
172
+ <th>Total Hits</th>
173
+ <th>Total Visitors</th>
174
+ <th>IPs</th>
175
+ </tr>
176
+ </thead>
177
+ <tbody>
178
+ <%# IP, Hits, Visitors Size, Country%>
179
+ <% data[:ips].group_by { |x| x[4] }.each do |k, v| %>
180
+ <tr>
181
+ <td><%= k %></td>
182
+ <td><%= v.map { |x| x[1] }.inject(&:+) %></td>
183
+ <td><%= v.map { |x| x[2] }.inject(&:+) %></td>
184
+ <td><%= v.map { |x| x[0] }.join(", ") %></td>
185
+ </tr>
186
+ <% end %>
187
+ </tbody>
188
+ </table>
189
+ </article>
190
+
164
191
  <article>
165
192
  <h2 id="streaks">Streaks</h2>
166
193
 
@@ -168,7 +195,16 @@
168
195
  <thead>
169
196
  <tr>
170
197
  <th>IP</th>
171
- <th>Day and URL</th>
198
+ <th>
199
+ <div class="columns">
200
+ <div class="col-2 column">
201
+ Day
202
+ </div>
203
+ <div class="col-10 column">
204
+ Resources
205
+ </div>
206
+ </div>
207
+ </th>
172
208
  </tr>
173
209
  </thead>
174
210
  <tbody>
@@ -176,11 +212,28 @@
176
212
  <tr>
177
213
  <td class="ip"><%= ip %></td>
178
214
  <td class="streaks">
179
- <% date_urls.group_by(&:first).each do |date, urls| %>
180
- <% urls.each do |url| %>
181
- <b><%= url[1] %>:</b> <%= url[2] %> <br />
182
- <% end %>
215
+ <div class="columns">
216
+ <% date_urls.group_by { |x| x[1] }.each do |date, urls| %>
217
+ <div class="col-2 column">
218
+ <%= date %>
219
+ </div>
220
+ <div class="col-10 column">
221
+ <span class="res-title">HTML:</span>
222
+ <ul>
223
+ <% urls.map { |x| x[2] }.select { |x| x.match /.*\.html?/ }.each do |url| %>
224
+ <li><%= url %></li>
225
+ <% end %>
226
+ </ul>
227
+
228
+ <span class="res-title">Other Resources:</span>
229
+ <ul>
230
+ <% urls.map { |x| x[2] }.sort.select { |x| not x.match /.*\.html?/ }.each do |url| %>
231
+ <li><%= url %></li>
232
+ <% end %>
233
+ </ul>
234
+ </div>
183
235
  <% end %>
236
+ </div>
184
237
  </td>
185
238
  </tr>
186
239
  <% end %>
@@ -1,3 +1,3 @@
1
1
  module ApacheLogReport
2
- VERSION = "1.1.6"
2
+ VERSION = "1.1.10"
3
3
  end
@@ -2,4 +2,5 @@ require 'apache_log_report/version'
2
2
  require 'apache_log_report/options_parser'
3
3
  require 'apache_log_report/log_parser'
4
4
  require 'apache_log_report/data_cruncher'
5
+ require 'apache_log_report/ip_locator'
5
6
  require 'apache_log_report/emitter'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: apache_log_report
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.6
4
+ version: 1.1.10
5
5
  platform: ruby
6
6
  authors:
7
7
  - Adolfo Villafiorita
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-11-01 00:00:00.000000000 Z
11
+ date: 2022-01-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: apache_log-parser
@@ -38,6 +38,34 @@ dependencies:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: ipaddr
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: iso_country_codes
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
41
69
  - !ruby/object:Gem::Dependency
42
70
  name: sqlite3
43
71
  requirement: !ruby/object:Gem::Requirement
@@ -66,8 +94,22 @@ dependencies:
66
94
  - - ">="
67
95
  - !ruby/object:Gem::Version
68
96
  version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: minitest
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
69
111
  description: Generate requests reports in HTML, OrgMode, and SQLite format from an
70
- Apache log file.
112
+ Apache log file. Superseded by https://rubygems.org/gems/log_senseLog Sense.
71
113
  email:
72
114
  - adolfo.villafiorita@ict4g.net
73
115
  executables:
@@ -90,6 +132,7 @@ files:
90
132
  - lib/apache_log_report/apache_log_report.rb
91
133
  - lib/apache_log_report/data_cruncher.rb
92
134
  - lib/apache_log_report/emitter.rb
135
+ - lib/apache_log_report/ip_locator.rb
93
136
  - lib/apache_log_report/log_parser.rb
94
137
  - lib/apache_log_report/options_parser.rb
95
138
  - lib/apache_log_report/templates/_output_table.html.erb
@@ -104,7 +147,10 @@ metadata:
104
147
  homepage_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report
105
148
  source_code_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report
106
149
  changelog_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report/CHANGELOG.org
107
- post_install_message:
150
+ post_install_message: |2
151
+ ! The 'apache_log_report' gem has been deprecated and has been replaced 'log_sense'.
152
+ ! See: https://rubygems.org/gems/log_sense
153
+ ! And: https://www.ict4g.net/gitea/adolfo/log_sense
108
154
  rdoc_options: []
109
155
  require_paths:
110
156
  - lib
@@ -119,8 +165,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
119
165
  - !ruby/object:Gem::Version
120
166
  version: '0'
121
167
  requirements: []
122
- rubygems_version: 3.2.22
168
+ rubygems_version: 3.0.3.1
123
169
  signing_key:
124
170
  specification_version: 4
125
- summary: Generate analytics from an Apache log file.
171
+ summary: Generate analytics from an Apache log file. Superseded by https://rubygems.org/gems/log_senseLog
172
+ Sense.
126
173
  test_files: []