apache_log_report 1.1.6 → 1.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 742584032338664834cd5c1b669289980e083e68d172827f6517ce040543aa4d
-  data.tar.gz: 1bf08e255feb434ded62589d5c70ea5a6eb83dc1f6a69e60a31cb520eea6b48f
+  metadata.gz: c046afd634406fbeb8bd2384d8cb918b444c984f5c75c2b77ae61f87237429a4
+  data.tar.gz: 9b7aa0b18006c5e903504f6287cbf6470a6bbf99ee0aef500b73168695b624f1
 SHA512:
-  metadata.gz: 955dbe90321c68f5694aaa892cae97926172ca47b684a0eee1b9f7644dc26a13f853b3b3d20e35cf06872b29db0c06bc16c4c86e69c740ea86fe8768ae92b2a1
-  data.tar.gz: e6d9ebc88435442bfb77faa3cb2eccb94a355e9ca7af44548a7b443531c9fae923e41f5d97201044f71414f2d9894162b593cd15170e255030ab49bd9992e3f0
+  metadata.gz: ad5802eb53e71b2302cc50447c9e2b05478ab98be23b0d1db02227c81b58c707bc2900f69f5abd712d11f6f43d67312c765b404de913d7ed495bcc4de9fe5ea1
+  data.tar.gz: 041c31954502ae3f989e52f3dd6adfc0b465a431ee152797fa11e5cbb9c37d6ade95e1ccd4f7166d2eee9ca9a3d4f8d1f300b472c1d0b9a2cf0be1a0bfb4468f

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    apache_log_report (1.1.2)
+    apache_log_report (1.1.8)
       apache_log-parser
       browser
+      ipaddr
       sqlite3
       terminal-table
 
@@ -12,6 +13,7 @@ GEM
   specs:
     apache_log-parser (3.1.2)
     browser (5.3.1)
+    ipaddr (1.2.3)
     rake (12.3.3)
     sqlite3 (1.4.2)
     terminal-table (3.0.2)

data/README.org

@@ -2,40 +2,6 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
-* Introduction
+This Gem has been superseded by [[https://rubygems.org/gems/log_sense][Log Sense]], which produces better
+outputs and parses also rails logs.
 
-ApacheLogReport is an Apache log analyzer.
-
-* Installation
-
-* Usage
-
-* Change Log
-
-See the [[file:CHANGELOG.org][CHANGELOG]] file.
-
-* Todo
-
-** TODO Referers should only include the hostname?
-** TODO Graphs in HTML output
-** TODO Countries
-** TODO Light and Dark Theme
-** TODO Embed CSS
-** TODO Declare datatypes in table outputs, so that we can format data
-
-* Compatibility
-
-
-* Author and Contributors
-
-[[http://ict4g.net/adolfo][Adolfo Villafiorita]].
-
-* Known Bugs
-
-Some known bugs and an unknown number of unknown bugs.
-
-(See the open issues for the known bugs.)
-
-* License
-
-Distributed under the terms of the [[http://opensource.org/licenses/MIT][MIT License]].

data/Rakefile

@@ -1,2 +1,15 @@
 require "bundler/gem_tasks"
 task :default => :spec
+
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+require_relative './lib/apache_log_report/ip_locator.rb'
+
+desc "Convert Geolocation DB to sqlite"
+task :dbip_to_sqlite3, [:filename] do |tasks, args|
+  filename = args[:filename]
+  ApacheLogReport::IpLocator::dbip_to_sqlite filename
+end

data/apache_log_report.gemspec

@@ -6,8 +6,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Adolfo Villafiorita"]
   spec.email         = ["adolfo.villafiorita@ict4g.net"]
 
-  spec.summary       = %q{Generate analytics from an Apache log file.}
-  spec.description   = %q{Generate requests reports in HTML, OrgMode, and SQLite format from an Apache log file.}
+  spec.summary       = %q{Generate analytics from an Apache log file.  Superseded by https://rubygems.org/gems/log_senseLog Sense.}
+  spec.description   = %q{Generate requests reports in HTML, OrgMode, and SQLite format from an Apache log file. Superseded by https://rubygems.org/gems/log_senseLog Sense.}
   spec.homepage      = "https://www.ict4g.net/gitea/adolfo/apache_log_report"
   spec.license       = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
@@ -29,6 +29,18 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "apache_log-parser"
   spec.add_dependency "browser"
+  spec.add_dependency "ipaddr"
+  spec.add_dependency "iso_country_codes"
   spec.add_dependency "sqlite3"
   spec.add_dependency "terminal-table"
+
+  spec.add_development_dependency "minitest"
+
+  spec.post_install_message = <<-MESSAGE
+  !    The 'apache_log_report' gem has been deprecated and has been replaced 'log_sense'.
+  !    See: https://rubygems.org/gems/log_sense
+  !    And: https://www.ict4g.net/gitea/adolfo/log_sense
+  MESSAGE
+
+
 end

data/exe/apache_log_report

@@ -31,6 +31,7 @@ if @options[:output_format]  == "sqlite"
   b.finish
 else
   @data = ApacheLogReport::DataCruncher.crunch @db, @options
+  @data = ApacheLogReport::DataCruncher.geolocate @data
 
   @ended_at = Time.now
   @duration = @ended_at - @started_at

data/lib/apache_log_report/data_cruncher.rb

@@ -1,7 +1,5 @@
-
 module ApacheLogReport
   module DataCruncher
-
     #
     # take a sqlite3 database and analyze data
     #
@@ -103,12 +101,13 @@ module ApacheLogReport
 
       @browsers = db.execute "SELECT browser, count(browser), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by browser order by count(browser) desc"
       @platforms = db.execute "SELECT platform, count(platform), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by platform order by count(platform) desc"
-      @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
       @referers = db.execute "SELECT referer, count(referer), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by referer order by count(referer) desc limit #{options[:limit]}"
       
-      @streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
+      @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
 
+      @streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
       data = {}
+
       self.instance_variables.each do |variable|
         var_as_symbol = variable.to_s[1..-1].to_sym
         data[var_as_symbol] = eval(variable.to_s)
@@ -116,6 +115,16 @@ module ApacheLogReport
       data
     end
 
+    # add country code to data[:ips]
+    def self.geolocate data
+      @location_db = IpLocator::load_db
+      data[:ips].each do |ip|
+        country_code = IpLocator::locate_ip ip[0], @location_db
+        ip << country_code
+      end
+      data
+    end
+
     private
 
     def self.date_intersect date1, date2, method
@@ -127,8 +136,6 @@ module ApacheLogReport
         date2
       end
     end
-
-
   end
 end
 

data/lib/apache_log_report/ip_locator.rb

@@ -0,0 +1,42 @@
+require 'csv'
+require 'sqlite3'
+require 'ipaddr'
+require 'iso_country_codes'
+
+module ApacheLogReport
+  module IpLocator
+    DB_FILE = "ip_locations/dbip-country-lite.sqlite3"
+
+    def self.dbip_to_sqlite db_location
+      db = SQLite3::Database.new ":memory:"
+      db.execute "CREATE TABLE ip_location (
+        from_ip_n INTEGER,
+        from_ip TEXT,
+        to_ip TEXT,
+        country_code TEXT
+      )"
+
+      ins = db.prepare "INSERT INTO ip_location(from_ip_n, from_ip, to_ip, country_code) values (?, ?, ?, ?)"
+      CSV.foreach(db_location) do |row|
+        ip = IPAddr.new row[0]
+        ins.execute(ip.to_i, row[0], row[1], row[2])
+      end
+
+      # persist to file
+      ddb = SQLite3::Database.new(DB_FILE)
+      b = SQLite3::Backup.new(ddb, 'main', db, 'main')
+      b.step(-1) #=> DONE
+      b.finish
+    end
+
+    def self.load_db
+      SQLite3::Database.new DB_FILE
+    end
+
+    def self.locate_ip ip, db
+      ip_n = IPAddr.new(ip).to_i
+      res = db.execute "SELECT * FROM ip_location where from_ip_n <= #{ip_n} order by from_ip_n desc limit 1"
+      IsoCountryCodes.find(res[0][3]).name
+    end
+  end
+end

data/lib/apache_log_report/templates/template.html.erb

@@ -32,6 +32,8 @@
                 "Platforms",
                 "Referers",
                 "IPs",
+                "Geolocation",
+                "Streaks",
                 "Command Invocation",
                 "Performance"
               ].each do |item| %>
@@ -142,7 +144,7 @@
           { title: "Browsers", header: ["Browser", "Hits", "Visitors", "Size"], rows: data[:browsers] },
           { title: "Platforms", header: ["Platform", "Hits", "Visitors", "Size"], rows: data[:platforms] },
           { title: "Referers", header: ["Referers", "Hits", "Visitors", "Size"], rows: data[:referers], col: "col-12" },
-          { title: "IPs", header: ["IPs", "Hits", "Visitors", "Size"], rows: data[:ips] },
+          { title: "IPs", header: ["IPs", "Hits", "Visitors", "Size", "Country"], rows: data[:ips] },
           { },
         ]
         %>
@@ -161,6 +163,31 @@
           <% end %>
         </div>
 
+        <article>
+          <h2 id="geolocation">Geolocation</h2>
+          <table class="table">
+            <thead>
+              <tr>
+                <th>Country Code</th>
+                <th>Total Hits</th>
+                <th>Total Visitors</th>
+                <th>IPs</th>
+              </tr>              
+            </thead>
+            <tbody>
+              <%# IP, Hits, Visitors Size, Country%>
+              <% data[:ips].group_by { |x| x[4] }.each do |k, v| %>
+                <tr>
+                  <td><%= k %></td>
+                  <td><%= v.map { |x| x[1] }.inject(&:+) %></td>
+                  <td><%= v.map { |x| x[2] }.inject(&:+) %></td>
+                  <td><%= v.map { |x| x[0] }.join(", ") %></td>
+                </tr>
+              <% end %>
+            </tbody>
+          </table>
+        </article>
+
         <article>
           <h2 id="streaks">Streaks</h2>
 
@@ -168,7 +195,16 @@
             <thead>
               <tr>
                 <th>IP</th>
-                <th>Day and URL</th>
+                <th>
+                  <div class="columns">
+                    <div class="col-2 column">
+                      Day
+                    </div>
+                    <div class="col-10 column">
+                      Resources
+                    </div>
+                  </div>
+                </th>
               </tr>
             </thead>
             <tbody>
@@ -176,11 +212,28 @@
                 <tr>
                   <td class="ip"><%= ip %></td>
                   <td class="streaks">
-                    <% date_urls.group_by(&:first).each do |date, urls| %>
-                      <% urls.each do |url| %>
-                        <b><%= url[1] %>:</b> <%= url[2] %> <br />
-                      <% end %>
+                    <div class="columns">
+                    <% date_urls.group_by { |x| x[1] }.each do |date, urls| %>
+                      <div class="col-2 column">
+                        <%= date %>
+                      </div>
+                      <div class="col-10 column">
+                        <span class="res-title">HTML:</span>
+                        <ul>
+                          <% urls.map { |x| x[2] }.select { |x| x.match /.*\.html?/ }.each do |url| %>
+                            <li><%= url %></li>
+                          <% end %>
+                        </ul>
+                        
+                        <span class="res-title">Other Resources:</span>
+                        <ul>
+                          <% urls.map { |x| x[2] }.sort.select { |x| not x.match /.*\.html?/ }.each do |url| %>
+                            <li><%= url %></li>
+                          <% end %>
+                        </ul>
+                      </div>
                     <% end %>
+                    </div>
                   </td>
                 </tr>
               <% end %>

data/lib/apache_log_report/version.rb

@@ -1,3 +1,3 @@
 module ApacheLogReport
-  VERSION = "1.1.6"
+  VERSION = "1.1.10"
 end

data/lib/apache_log_report.rb

@@ -2,4 +2,5 @@ require 'apache_log_report/version'
 require 'apache_log_report/options_parser'
 require 'apache_log_report/log_parser'
 require 'apache_log_report/data_cruncher'
+require 'apache_log_report/ip_locator'
 require 'apache_log_report/emitter'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: apache_log_report
 version: !ruby/object:Gem::Version
-  version: 1.1.6
+  version: 1.1.10
 platform: ruby
 authors:
 - Adolfo Villafiorita
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-11-01 00:00:00.000000000 Z
+date: 2022-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: apache_log-parser
@@ -38,6 +38,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ipaddr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: iso_country_codes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -66,8 +94,22 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Generate requests reports in HTML, OrgMode, and SQLite format from an
-  Apache log file.
+  Apache log file. Superseded by https://rubygems.org/gems/log_senseLog Sense.
 email:
 - adolfo.villafiorita@ict4g.net
 executables:
@@ -90,6 +132,7 @@ files:
 - lib/apache_log_report/apache_log_report.rb
 - lib/apache_log_report/data_cruncher.rb
 - lib/apache_log_report/emitter.rb
+- lib/apache_log_report/ip_locator.rb
 - lib/apache_log_report/log_parser.rb
 - lib/apache_log_report/options_parser.rb
 - lib/apache_log_report/templates/_output_table.html.erb
@@ -104,7 +147,10 @@ metadata:
   homepage_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report
   source_code_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report
   changelog_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report/CHANGELOG.org
-post_install_message:
+post_install_message: |2
+    !    The 'apache_log_report' gem has been deprecated and has been replaced 'log_sense'.
+    !    See: https://rubygems.org/gems/log_sense
+    !    And: https://www.ict4g.net/gitea/adolfo/log_sense
 rdoc_options: []
 require_paths:
 - lib
@@ -119,8 +165,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.0.3.1
 signing_key:
 specification_version: 4
-summary: Generate analytics from an Apache log file.
+summary: Generate analytics from an Apache log file.  Superseded by https://rubygems.org/gems/log_senseLog
+  Sense.
 test_files: []